diff --git a/il_supermarket_scarper/engines/apsx.py b/il_supermarket_scarper/engines/apsx.py index 53a6eca..34970df 100644 --- a/il_supermarket_scarper/engines/apsx.py +++ b/il_supermarket_scarper/engines/apsx.py @@ -44,7 +44,7 @@ def _build_query_url(self, query_params): res.append(base + self.aspx_page + query_params) return res - def _get_all_possible_query_string_params( + def _get_all_possible_query_string_params( # pylint: disable=unused-argument self, files_types=None, store_id=None, when_date=None ): """get the arguments need to add to the url""" @@ -52,35 +52,38 @@ def _get_all_possible_query_string_params( res = [] for c_id in self.chain_id: res.append(f"?code={c_id}") - return res + chains_urls = res + chains_urls = [f"?code={self.chain_id}"] - # add file types to url - if files_types: - chains_urls_with_types = [] - for files_type in files_types: - file_type_id = self.file_type_id(files_type) - chains_urls_with_types.extend( - [ - f"{chain_url}&WFileType={file_type_id}" - for chain_url in chains_urls - ] - ) - chains_urls = chains_urls_with_types - - # add store id - if store_id: - for chain_url in chains_urls: - chain_url += f"&WStore={store_id}" - - # posting date - if when_date: - for chain_url in chains_urls: - chain_url += ( - f"&WDate={when_date.strftime('%d/%m/%Y').reaplce('/','%2F')}" - ) return chains_urls + # # add file types to url + # if files_types: + # chains_urls_with_types = [] + # for files_type in files_types: + # file_type_id = self.file_type_id(files_type) + # chains_urls_with_types.extend( + # [ + # f"{chain_url}&WFileType={file_type_id}" + # for chain_url in chains_urls + # ] + # ) + # chains_urls = chains_urls_with_types + + # # add store id + # if store_id: + # for chain_url in chains_urls: + # chain_url += f"&WStore={store_id}" + + # # posting date + # if when_date: + # for chain_url in chains_urls: + # chain_url += ( + # f"&WDate={when_date.strftime('%d/%m/%Y').reaplce('/','%2F')}" + # ) + # return chains_urls + def get_request_url(self, files_types=None, store_id=None, when_date=None): result = [] for query_params in self._get_all_possible_query_string_params( diff --git a/il_supermarket_scarper/engines/cerberus.py b/il_supermarket_scarper/engines/cerberus.py index 302de4b..9f447d1 100644 --- a/il_supermarket_scarper/engines/cerberus.py +++ b/il_supermarket_scarper/engines/cerberus.py @@ -41,7 +41,7 @@ def scrape( limit=None, files_types=None, store_id=None, - only_latest=False, + when_date=None, files_names_to_scrape=None, filter_null=False, filter_zero=False, @@ -52,7 +52,7 @@ def scrape( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) files = self.collect_files_details_from_site( limit=limit, @@ -60,7 +60,7 @@ def scrape( filter_null=filter_null, filter_zero=filter_zero, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=files_names_to_scrape, ) self.on_collected_details(files) @@ -73,6 +73,7 @@ def scrape( return results except Exception as e: # pylint: disable=broad-except self.on_download_fail(e, files=files) + Logger.error_execption(e) return [] def collect_files_details_from_site( @@ -82,7 +83,7 @@ def collect_files_details_from_site( filter_null=False, filter_zero=False, store_id=None, - only_latest=False, + when_date=None, files_names_to_scrape=None, ): """collect all files to download from the site""" @@ -117,7 +118,7 @@ def collect_files_details_from_site( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=files_names_to_scrape, ) Logger.info(f"After applying limit: Found {len(files)} files") diff --git a/il_supermarket_scarper/engines/engine.py b/il_supermarket_scarper/engines/engine.py index b36ecf8..a55741f 100644 --- a/il_supermarket_scarper/engines/engine.py +++ b/il_supermarket_scarper/engines/engine.py @@ -1,7 +1,7 @@ from abc import ABC import os import re - +import datetime from il_supermarket_scarper.utils import ( get_output_folder, @@ -38,16 +38,16 @@ def get_storage_path(self): return self.storage_path def _is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): Logger.info( f"check if fail is allowd with, limit={limit}," - f"files_types={files_types},store_id={store_id},only_latest={only_latest}" + f"files_types={files_types},store_id={store_id},when_date={when_date}" ) return False def is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): """return true if its ok the scarper reuturn no enrty""" @@ -58,7 +58,7 @@ def is_validate_scraper_found_no_files( for file_type in files_types: if file_type in FileTypesFilters.all_full_files(): request_only_update_file = False - Logger.info(f"the value of {only_latest} should not affect.") + Logger.info(f"the value of {when_date} should not affect.") return ( limit == 0 or files_types == [] @@ -68,7 +68,7 @@ def is_validate_scraper_found_no_files( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) ) @@ -83,13 +83,13 @@ def apply_limit( files_types=None, by_function=lambda x: x, store_id=None, - only_latest=False, + when_date=None, files_names_to_scrape=None, ): """filter the list according to condition""" - assert ( - not only_latest or limit is None - ), "only_latest flag can't be applied with limit." + # assert ( + # when_date is not None or limit is None + # ), "when_date flag can't be applied with limit." # filter files already downloaded intreable_ = self.filter_already_downloaded( @@ -124,13 +124,17 @@ def apply_limit( ) Logger.info(f"Number of entry after filter file type id is {len(intreable_)}") - if only_latest: + if isinstance(when_date, datetime.datetime): + intreable_ = self.get_by_date(when_date, by_function, intreable_) + elif isinstance(when_date, str) and when_date == "latest": intreable_ = self.get_only_latest(by_function, intreable_) + elif when_date is not None: + raise ValueError(f"when_date should be datetime or bool, got {when_date}") Logger.info(f"Number of entry after filter keeping latast is {len(intreable_)}") # filter by limit if the 'files_types' filter is not on. - if limit and files_types is None: + if limit: assert limit > 0, "Limit must be greater than 0" Logger.info(f"Limit: {limit}") intreable_ = intreable_[: min(limit, len(list(intreable_)))] @@ -144,12 +148,12 @@ def apply_limit( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) ): raise ValueError( f"No files to download for file files_types={files_types}," - f"limit={limit},store_id={store_id},only_latest={only_latest}" + f"limit={limit},store_id={store_id},when_date={when_date}" ) return intreable_ @@ -182,6 +186,21 @@ def get_only_latest(self, by_function, intreable_): groups_value[store_info] = file return list(groups_value.values()) + def get_by_date(self, requested_date, by_function, intreable_): + """get by date""" + # + date_format = requested_date.strftime("%Y%m%d") + # + groups_value = [] + for file in intreable_: + name_split = by_function(file).split("-", maxsplit=2) + date_info = name_split[-1].rsplit(".", maxsplit=1)[0] + + if date_info.startswith(date_format): + groups_value.append(file) + + return groups_value + @classmethod def unique(cls, iterable, by_function=lambda x: x): """Returns the type of the file.""" @@ -210,7 +229,7 @@ def scrape( limit=None, files_types=None, store_id=None, - only_latest=False, + when_date=None, files_names_to_scrape=None, filter_null=False, filter_zero=False, @@ -222,7 +241,7 @@ def scrape( files_types=files_types, store_id=store_id, files_names_to_scrape=files_names_to_scrape, - only_latest=only_latest, + when_date=when_date, filter_null=filter_null, filter_zero=filter_zero, ) diff --git a/il_supermarket_scarper/engines/multipage_web.py b/il_supermarket_scarper/engines/multipage_web.py index 9b6742b..8145b81 100644 --- a/il_supermarket_scarper/engines/multipage_web.py +++ b/il_supermarket_scarper/engines/multipage_web.py @@ -77,7 +77,6 @@ def collect_files_details_from_site( files_types=None, store_id=None, when_date=None, - only_latest=False, files_names_to_scrape=None, ): self.post_scraping() @@ -109,7 +108,7 @@ def collect_files_details_from_site( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=files_names_to_scrape, ) @@ -125,7 +124,7 @@ def collect_files_details_from_page(self, html): return links, filenames def process_links_before_download( - self, page, limit=None, files_types=None, store_id=None, only_latest=None + self, page, limit=None, files_types=None, store_id=None, when_date=None ): """additional processing to the links before download""" response = self.session_with_cookies_by_chain(page) @@ -141,7 +140,7 @@ def process_links_before_download( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) Logger.info( diff --git a/il_supermarket_scarper/engines/publishprice.py b/il_supermarket_scarper/engines/publishprice.py index c8acd80..d87e7ab 100644 --- a/il_supermarket_scarper/engines/publishprice.py +++ b/il_supermarket_scarper/engines/publishprice.py @@ -88,14 +88,14 @@ def get_name_from_herf(x): return download_urls, file_names def _is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): return ( super()._is_validate_scraper_found_no_files( # what fails the rest limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) or ( # if we are looking for one store file in a weekend or holiday store_id and (_is_weekend_in_israel() or _is_holiday_in_israel()) diff --git a/il_supermarket_scarper/engines/web.py b/il_supermarket_scarper/engines/web.py index c232b56..476ed89 100644 --- a/il_supermarket_scarper/engines/web.py +++ b/il_supermarket_scarper/engines/web.py @@ -45,7 +45,7 @@ def apply_limit_zip( files_types=None, by_function=lambda x: x[0], store_id=None, - only_latest=False, + when_date=None, files_names_to_scrape=None, ): """apply limit to zip""" @@ -55,7 +55,7 @@ def apply_limit_zip( files_types=files_types, by_function=by_function, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=files_names_to_scrape, ) if len(ziped) == 0: @@ -69,7 +69,6 @@ def collect_files_details_from_site( files_types=None, store_id=None, when_date=None, - only_latest=False, files_names_to_scrape=None, ): """collect all enteris to download from site""" @@ -95,7 +94,7 @@ def collect_files_details_from_site( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=files_names_to_scrape, ) @@ -109,7 +108,7 @@ def scrape( limit=None, files_types=None, store_id=None, - only_latest=False, + when_date=None, files_names_to_scrape=None, filter_null=False, filter_zero=False, @@ -121,7 +120,7 @@ def scrape( limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, filter_null=filter_null, filter_zero=filter_zero, ) @@ -130,7 +129,7 @@ def scrape( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=files_names_to_scrape, ) @@ -153,4 +152,5 @@ def scrape( return results except Exception as e: # pylint: disable=broad-except self.on_download_fail(e, download_urls=download_urls, file_names=file_names) + Logger.error_execption(e) return [] diff --git a/il_supermarket_scarper/main.py b/il_supermarket_scarper/main.py index b688941..3136359 100644 --- a/il_supermarket_scarper/main.py +++ b/il_supermarket_scarper/main.py @@ -10,7 +10,7 @@ def __init__( size_estimation_mode=False, enabled_scrapers=None, limit=None, - only_latest=False, + when_date=None, files_types=FileTypesFilters.all_types(), dump_folder_name=None, lookup_in_db=True, @@ -27,7 +27,7 @@ def __init__( self.dump_folder_name = dump_folder_name self.limit = limit self.files_types = files_types - self.only_latest = only_latest + self.when_date = when_date def get_dump_folder_name(self): """get the dump folder name""" @@ -36,5 +36,5 @@ def get_dump_folder_name(self): def start(self): """run the scraping""" return self.runner.run( - limit=self.limit, files_types=self.files_types, only_latest=self.only_latest + limit=self.limit, files_types=self.files_types, when_date=self.when_date ) diff --git a/il_supermarket_scarper/scrapper_runner.py b/il_supermarket_scarper/scrapper_runner.py index 3e6978b..141759c 100644 --- a/il_supermarket_scarper/scrapper_runner.py +++ b/il_supermarket_scarper/scrapper_runner.py @@ -38,7 +38,7 @@ def __init__( self.multiprocessing = multiprocessing self.lookup_in_db = lookup_in_db - def run(self, limit=None, files_types=None, only_latest=False): + def run(self, limit=None, files_types=None, when_date=False): """run the scraper""" Logger.info(f"Limit is {limit}") Logger.info(f"files_types is {files_types}") @@ -54,7 +54,7 @@ def run(self, limit=None, files_types=None, only_latest=False): { "limit": limit, "files_types": files_types, - "only_latest": only_latest, + "when_date": when_date, }, ), self.enabled_scrapers, @@ -77,7 +77,7 @@ def scrape_one( limit=None, files_types=None, store_id=None, - only_latest=False, + when_date=None, ): """scrape one""" chain_scrapper_constractor = ScraperFactory.get(chain_scrapper_class) @@ -94,7 +94,7 @@ def scrape_one( limit=limit, files_types=files_types, store_id=store_id, - only_latest=only_latest, + when_date=when_date, files_names_to_scrape=None, filter_null=False, filter_zero=False, diff --git a/il_supermarket_scarper/scrappers/bareket.py b/il_supermarket_scarper/scrappers/bareket.py index 2f2bcd9..88a7456 100644 --- a/il_supermarket_scarper/scrappers/bareket.py +++ b/il_supermarket_scarper/scrappers/bareket.py @@ -16,7 +16,7 @@ def __init__(self, folder_name=None): ) def _is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): # no data on shabat if you test a single store file. return _is_saturday_in_israel() or _is_holiday_in_israel() and store_id diff --git a/il_supermarket_scarper/scrappers/meshnat_yosef.py b/il_supermarket_scarper/scrappers/meshnat_yosef.py index 91a1c9a..b82d942 100644 --- a/il_supermarket_scarper/scrappers/meshnat_yosef.py +++ b/il_supermarket_scarper/scrappers/meshnat_yosef.py @@ -34,7 +34,7 @@ def extract_task_from_entry(self, all_trs): return download_urls, file_names def _is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): # no data on shabat return _is_saturday_in_israel() or _is_holiday_in_israel() diff --git a/il_supermarket_scarper/scrappers/nativ_hashed.py b/il_supermarket_scarper/scrappers/nativ_hashed.py index 81c9773..5a7f6fa 100644 --- a/il_supermarket_scarper/scrappers/nativ_hashed.py +++ b/il_supermarket_scarper/scrappers/nativ_hashed.py @@ -19,7 +19,7 @@ def __init__(self, folder_name=None): ) def _is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): # no data on shabat return _is_saturday_in_israel() or _is_holiday_in_israel() diff --git a/il_supermarket_scarper/scrappers/polizer.py b/il_supermarket_scarper/scrappers/polizer.py index f20d171..6f8819c 100644 --- a/il_supermarket_scarper/scrappers/polizer.py +++ b/il_supermarket_scarper/scrappers/polizer.py @@ -14,7 +14,7 @@ def __init__(self, folder_name=None): ) def _is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): # no data on shabat return ( diff --git a/il_supermarket_scarper/scrappers/tests/test_cases.py b/il_supermarket_scarper/scrappers/tests/test_cases.py index 79064ce..9bd0dd7 100644 --- a/il_supermarket_scarper/scrappers/tests/test_cases.py +++ b/il_supermarket_scarper/scrappers/tests/test_cases.py @@ -4,7 +4,7 @@ import os import uuid import xml.etree.ElementTree as ET -from il_supermarket_scarper.utils import FileTypesFilters, Logger, DumpFolderNames +from il_supermarket_scarper.utils import FileTypesFilters, Logger, DumpFolderNames, _now from il_supermarket_scarper.scrappers_factory import ScraperFactory @@ -42,7 +42,7 @@ def _make_sure_filter_work( file_type=None, limit=None, store_id=None, - only_latest=False, + when_date=None, ): """make sure the file type filter works""" if file_type: @@ -55,7 +55,7 @@ def _make_sure_filter_work( for file in files_found: store_mark.append(int(file.split("-")[1])) assert len(set(store_mark)) == 1 and len(store_mark) == len(files_found) - if only_latest: + if when_date: files_sources = [] for file in files_found: source = file.split("-")[:2] @@ -63,9 +63,8 @@ def _make_sure_filter_work( store_mark.append(source) assert ( - not limit or len(files_found) == limit - ), f""" Found {files_found} - f"files but should be {limit}""" + limit is None or len(files_found) == limit + ), f""" Found {files_found} f"files but should be {limit}""" def _make_sure_file_contain_chain_ids(self, chain_ids, file): """make sure the scraper download only the chain id""" @@ -107,7 +106,7 @@ def _clean_scarpe_delete( store_id=None, limit=None, file_type=None, - only_latest=False, + when_date=None, ): with tempfile.TemporaryDirectory() as tmpdirname: self.__clean_scarpe_delete( @@ -116,7 +115,7 @@ def _clean_scarpe_delete( store_id=store_id, limit=limit, file_type=file_type, - only_latest=only_latest, + when_date=when_date, ) def __clean_scarpe_delete( @@ -126,7 +125,7 @@ def __clean_scarpe_delete( store_id=None, limit=None, file_type=None, - only_latest=False, + when_date=None, ): self._delete_download_folder(dump_path) os.makedirs(dump_path) @@ -142,7 +141,7 @@ def __clean_scarpe_delete( "limit": limit, "files_types": file_type, "store_id": store_id, - "only_latest": only_latest, + "when_date": when_date, "filter_null": True, "filter_zero": True, } @@ -164,14 +163,14 @@ def __clean_scarpe_delete( limit=limit, files_types=file_type, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) and not hasattr(scraper, "_is_flaky"): self._make_sure_filter_work( files_found, file_type=file_type, limit=limit, store_id=store_id, - only_latest=only_latest, + when_date=when_date, ) for file in files_found: @@ -201,7 +200,7 @@ def test_scrape_one(self): def test_scrape_ten(self): """scrape ten file and make sure they exists""" - self._clean_scarpe_delete(scraper_enum, limit=None) + self._clean_scarpe_delete(scraper_enum, limit=10) def test_scrape_promo(self): """scrape one promo file and make sure it exists""" @@ -253,9 +252,7 @@ def test_scrape_file_from_single_store(self): def test_scrape_file_from_single_store_last(self): """test fetching latest file only""" self._clean_scarpe_delete( - scraper_enum, - store_id=store_id, - only_latest=True, + scraper_enum, store_id=store_id, when_date=_now(), limit=1 ) return TestScapers diff --git a/il_supermarket_scarper/scrappers/tivtaam.py b/il_supermarket_scarper/scrappers/tivtaam.py index 3fbf6ba..f6b174f 100644 --- a/il_supermarket_scarper/scrappers/tivtaam.py +++ b/il_supermarket_scarper/scrappers/tivtaam.py @@ -19,7 +19,7 @@ def __init__(self, folder_name=None): ) def is_validate_scraper_found_no_files( - self, limit=None, files_types=None, store_id=None, only_latest=False + self, limit=None, files_types=None, store_id=None, when_date=None ): return ( _is_saturday_in_israel() diff --git a/il_supermarket_scarper/utils/connection.py b/il_supermarket_scarper/utils/connection.py index 5e17bcd..785e842 100644 --- a/il_supermarket_scarper/utils/connection.py +++ b/il_supermarket_scarper/utils/connection.py @@ -292,12 +292,12 @@ def collect_from_ftp(ftp_host, ftp_username, ftp_password, ftp_path, timeout=60 @download_connection_retry() def fetch_temporary_gz_file_from_ftp( - ftp_host, ftp_username, ftp_password, ftp_path, temporary_gz_file_path + ftp_host, ftp_username, ftp_password, ftp_path, temporary_gz_file_path, timeout=15 ): """download a file from a cerberus base site.""" with open(temporary_gz_file_path, "wb") as file_ftp: file_name = ntpath.basename(temporary_gz_file_path) - ftp = FTP_TLS(ftp_host, ftp_username, ftp_password) + ftp = FTP_TLS(ftp_host, ftp_username, ftp_password, timeout=timeout) ftp.trust_server_pasv_ipv4_address = True ftp.cwd(ftp_path) ftp.retrbinary("RETR " + file_name, file_ftp.write) diff --git a/il_supermarket_scarper/utils/gzip_utils.py b/il_supermarket_scarper/utils/gzip_utils.py index 2f8ad52..20aeffe 100644 --- a/il_supermarket_scarper/utils/gzip_utils.py +++ b/il_supermarket_scarper/utils/gzip_utils.py @@ -22,7 +22,7 @@ def extract_xml_file_from_gz_file(file_save_path): with open(target_file_name, "wb") as f_out: f_out.write(the_file.read()) - except ( # pylint: disable=broad-except,redefined-outer-name + except ( # pylint: disable=broad-except,redefined-outer-name Exception ) as exception: report_failed_zip(exception, file_save_path, target_file_name) diff --git a/setup.py b/setup.py index 6de0864..081b398 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ tests_require=dev_required, extras_require={"test": ["pytest", "pytest-xdist"]}, # *strongly* suggested for sharing - version="0.4.8", + version="0.4.9", # The license can be anything you like license="MIT", description="python package that implement a scraping for israeli supermarket data", diff --git a/stress_test.py b/stress_test.py index c4017e3..fdc9b52 100644 --- a/stress_test.py +++ b/stress_test.py @@ -4,10 +4,40 @@ import tempfile import pstats import cProfile -from io import StringIO +import io from il_supermarket_scarper.scrappers_factory import ScraperFactory +def format_stats_as_json(profile, project_name): + """get the stats from the profiler and format them as json""" + stream = io.StringIO() + ps = pstats.Stats(profile, stream=stream) + ps.sort_stats(pstats.SortKey.CUMULATIVE) # Sort by cumulative time + ps.print_stats() + + # Convert the printed stats to a list of lines + stats_output = stream.getvalue().splitlines() + + # Filter the lines to include only functions within the project + project_stats = [] + for line in stats_output: + if project_name in line: # Filter for project-specific lines + + parts = line.split() + if len(parts) >= 5: # Basic sanity check for the parts + function_data = { + "function": parts[-1], # Function path + "ncalls": parts[0], # Number of calls + "tottime": parts[1], + "tottime_per_call": parts[2], # Time spent in function + "cumtime": parts[3], # Cumulative time including subcalls + "cumtime_per_call": parts[4], # + } + project_stats.append(function_data) + + return project_stats + + if __name__ == "__main__": result = {} @@ -28,14 +58,9 @@ def full_execution(scraper): pr.disable() - stream = StringIO() - ps = pstats.Stats(pr, stream=stream) - ps.print_stats() - stream.seek(0) - end_time = time.time() result[scraper_name] = { - "status": stream.read(), + "status": format_stats_as_json(pr, "israeli-supermarket-scarpers"), "execution_time": execution_time, "start_time": start_time, "end_time": end_time, diff --git a/stress_test_results.json b/stress_test_results.json index a92a735..5a6cb0f 100644 --- a/stress_test_results.json +++ b/stress_test_results.json @@ -1,177 +1,12754 @@ { "BAREKET": { - "execution_time": "2024-09-28 16:29:08", - "start_time": 1727540948.674289, - "end_time": 1727541208.1898768, - "time": 259.51558780670166, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "140.608", + "cumtime_per_call": "140.608" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "140.552", + "cumtime_per_call": "140.552" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "140.552", + "cumtime_per_call": "140.552" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "140.552", + "cumtime_per_call": "140.552" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "139.871", + "cumtime_per_call": "139.871" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "139.870", + "cumtime_per_call": "139.870" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.004", + "tottime_per_call": "0.004", + "cumtime": "139.850", + "cumtime_per_call": "139.850" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.401", + "cumtime_per_call": "0.401" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.389", + "cumtime_per_call": "0.389" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.389", + "cumtime_per_call": "0.389" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.389", + "cumtime_per_call": "0.389" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.389", + "cumtime_per_call": "0.389" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.389", + "cumtime_per_call": "0.389" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.278", + "cumtime_per_call": "0.069" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.197", + "cumtime_per_call": "0.197" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.163", + "cumtime_per_call": "0.163" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.154", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.111", + "cumtime_per_call": "0.022" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.075", + "cumtime_per_call": "0.075" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.075", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.075", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.040", + "cumtime_per_call": "0.040" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.040", + "cumtime_per_call": "0.040" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.019", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/marking.py:26(__call__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/bareket.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/marking.py:40(wrap_method)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:33:04", + "start_time": 1728502384.2548087, + "end_time": 1728502524.8614137, + "time": 140.606605052948, "files": 1000 }, "YAYNO_BITAN": { - "execution_time": "2024-09-28 16:33:31", - "start_time": 1727541211.9969866, - "end_time": 1727541356.5669396, - "time": 144.56995296478271, - "files": 753 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "114.057", + "cumtime_per_call": "114.057" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "114.017", + "cumtime_per_call": "114.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "114.017", + "cumtime_per_call": "114.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "114.017", + "cumtime_per_call": "114.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "113.179", + "cumtime_per_call": "113.179" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "113.175", + "cumtime_per_call": "113.175" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.004", + "tottime_per_call": "0.004", + "cumtime": "113.162", + "cumtime_per_call": "113.162" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.568", + "cumtime_per_call": "0.568" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:38(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.388", + "cumtime_per_call": "0.388" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.270", + "cumtime_per_call": "0.067" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:61()", + "ncalls": "959", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.266", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.229", + "cumtime_per_call": "0.115" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.229", + "cumtime_per_call": "0.115" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.229", + "cumtime_per_call": "0.115" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.229", + "cumtime_per_call": "0.115" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.229", + "cumtime_per_call": "0.115" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.189", + "cumtime_per_call": "0.189" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.156", + "cumtime_per_call": "0.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "977", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.146", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.110", + "cumtime_per_call": "0.022" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.078", + "cumtime_per_call": "0.078" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.073", + "cumtime_per_call": "0.018" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.073", + "cumtime_per_call": "0.018" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:63(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.046", + "cumtime_per_call": "0.046" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:65(get_herf_element)", + "ncalls": "2877", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.043", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.040", + "cumtime_per_call": "0.040" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.040", + "cumtime_per_call": "0.040" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:71(get_path_from_herf)", + "ncalls": "1918", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.030", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:68(get_herf)", + "ncalls": "1918", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.029", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:85()", + "ncalls": "959", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.016", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:74(get_name_from_herf)", + "ncalls": "959", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.016", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:79()", + "ncalls": "959", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.015", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.012", + "cumtime_per_call": "0.012" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/bitan.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "959", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "1918", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:35:24", + "start_time": 1728502524.868611, + "end_time": 1728502638.9248798, + "time": 114.0562686920166, + "files": 959 }, "COFIX": { - "execution_time": "2024-09-28 16:35:56", - "start_time": 1727541356.5677948, - "end_time": 1727541427.2736921, - "time": 70.7058973312378, - "files": 573 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "61.366", + "cumtime_per_call": "61.366" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "61.350", + "cumtime_per_call": "61.350" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "61.350", + "cumtime_per_call": "61.350" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "61.350", + "cumtime_per_call": "61.350" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "60.937", + "cumtime_per_call": "60.937" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "60.935", + "cumtime_per_call": "60.935" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "60.897", + "cumtime_per_call": "60.897" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.244", + "cumtime_per_call": "0.061" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.170", + "cumtime_per_call": "0.170" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.169", + "cumtime_per_call": "0.169" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.165", + "cumtime_per_call": "0.165" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.165", + "cumtime_per_call": "0.165" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.165", + "cumtime_per_call": "0.165" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.165", + "cumtime_per_call": "0.165" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.142", + "cumtime_per_call": "0.142" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "886", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.132", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.099", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.071", + "cumtime_per_call": "0.071" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.062", + "cumtime_per_call": "0.015" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.061", + "cumtime_per_call": "0.015" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.040", + "cumtime_per_call": "0.040" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.040", + "cumtime_per_call": "0.040" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.038", + "cumtime_per_call": "0.038" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "870", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/cofix.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "870", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "1740", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:37:18", + "start_time": 1728502638.9315279, + "end_time": 1728502700.2968335, + "time": 61.36530566215515, + "files": 870 + }, + "CITY_MARKET_GIVATAYIM": { + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.375", + "cumtime_per_call": "135.375" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.360", + "cumtime_per_call": "135.360" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.360", + "cumtime_per_call": "135.360" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.360", + "cumtime_per_call": "135.360" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "134.784", + "cumtime_per_call": "134.784" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "134.782", + "cumtime_per_call": "134.782" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.004", + "tottime_per_call": "0.004", + "cumtime": "134.764", + "cumtime_per_call": "134.764" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.289", + "cumtime_per_call": "0.289" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.286", + "cumtime_per_call": "0.071" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.278", + "cumtime_per_call": "0.278" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.278", + "cumtime_per_call": "0.278" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.278", + "cumtime_per_call": "0.278" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.278", + "cumtime_per_call": "0.278" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.278", + "cumtime_per_call": "0.278" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.200", + "cumtime_per_call": "0.200" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.165", + "cumtime_per_call": "0.165" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.155", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.117", + "cumtime_per_call": "0.023" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.080", + "cumtime_per_call": "0.080" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.078", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.078", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.042", + "cumtime_per_call": "0.042" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.042", + "cumtime_per_call": "0.042" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.017", + "cumtime_per_call": "0.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/city_market.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:38:20", + "start_time": 1728502700.3017585, + "end_time": 1728502835.675384, + "time": 135.37362551689148, + "files": 1000 + }, + "CITY_MARKET_KIRYATONO": { + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.277", + "cumtime_per_call": "135.277" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.239", + "cumtime_per_call": "135.239" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "135.239", + "cumtime_per_call": "135.239" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "135.239", + "cumtime_per_call": "135.239" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "133.900", + "cumtime_per_call": "133.900" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "133.897", + "cumtime_per_call": "133.897" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.004", + "tottime_per_call": "0.004", + "cumtime": "133.880", + "cumtime_per_call": "133.880" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.038", + "cumtime_per_call": "1.038" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.026", + "cumtime_per_call": "1.026" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.026", + "cumtime_per_call": "1.026" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.026", + "cumtime_per_call": "1.026" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.026", + "cumtime_per_call": "1.026" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.026", + "cumtime_per_call": "1.026" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.300", + "cumtime_per_call": "0.075" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.206", + "cumtime_per_call": "0.206" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.169", + "cumtime_per_call": "0.169" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.125", + "cumtime_per_call": "0.025" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.089", + "cumtime_per_call": "0.089" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.083", + "cumtime_per_call": "0.021" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.082", + "cumtime_per_call": "0.021" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.047", + "cumtime_per_call": "0.047" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.047", + "cumtime_per_call": "0.047" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.016", + "cumtime_per_call": "0.016" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/city_market.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:40:35", + "start_time": 1728502835.6832678, + "end_time": 1728502970.9594235, + "time": 135.27615571022034, + "files": 1000 + }, + "CITY_MARKET_KIRYATGAT": { + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "153.183", + "cumtime_per_call": "153.183" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "153.156", + "cumtime_per_call": "153.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "153.156", + "cumtime_per_call": "153.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "153.156", + "cumtime_per_call": "153.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "148.797", + "cumtime_per_call": "148.797" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "148.794", + "cumtime_per_call": "148.794" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "148.776", + "cumtime_per_call": "148.776" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.056", + "cumtime_per_call": "4.056" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.048", + "cumtime_per_call": "4.048" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.048", + "cumtime_per_call": "4.048" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.048", + "cumtime_per_call": "4.048" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.048", + "cumtime_per_call": "4.048" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.048", + "cumtime_per_call": "4.048" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.301", + "cumtime_per_call": "0.075" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.200", + "cumtime_per_call": "0.200" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.165", + "cumtime_per_call": "0.165" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.155", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.118", + "cumtime_per_call": "0.024" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.096", + "cumtime_per_call": "0.096" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.080", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.080", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.055", + "cumtime_per_call": "0.055" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.055", + "cumtime_per_call": "0.055" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.017", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.017", + "cumtime_per_call": "0.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/city_market.py:32(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:42:50", + "start_time": 1728502970.969553, + "end_time": 1728503409.6091256, + "time": 438.63957262039185, + "files": 1000 + }, + "CITY_MARKET_SHOPS": { + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.137", + "cumtime_per_call": "5.137" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.134", + "cumtime_per_call": "5.134" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.134", + "cumtime_per_call": "5.134" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.134", + "cumtime_per_call": "5.134" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/multipage_web.py:74(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "4.609", + "cumtime_per_call": "4.609" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.806", + "cumtime_per_call": "1.903" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.805", + "cumtime_per_call": "1.903" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.790", + "cumtime_per_call": "1.895" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.247", + "cumtime_per_call": "1.247" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.247", + "cumtime_per_call": "1.247" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.247", + "cumtime_per_call": "1.247" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/multipage_web.py:44(get_number_of_pages)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.246", + "cumtime_per_call": "1.246" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.077", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.056", + "cumtime_per_call": "0.056" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "304", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.046", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.045", + "cumtime_per_call": "0.045" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.030", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.020", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.020", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.019", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.013", + "cumtime_per_call": "0.007" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.011", + "cumtime_per_call": "0.011" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.011", + "cumtime_per_call": "0.011" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "3", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/multipage_web.py:60(get_total_pages)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/city_market.py:44(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/multipage_web.py:25(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "288", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "576", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:16(multiple_page_aggregtion)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/multipage_web.py:93()", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:50:09", + "start_time": 1728503409.6176064, + "end_time": 1728503414.7544646, + "time": 5.136858224868774, + "files": 288 }, "DOR_ALON": { - "execution_time": "2024-09-28 16:37:07", - "start_time": 1727541427.2744784, - "end_time": 1727546029.4199014, - "time": 4602.145422935486, - "files": 5515 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1170.382", + "cumtime_per_call": "1170.382" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1170.256", + "cumtime_per_call": "1170.256" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1170.256", + "cumtime_per_call": "1170.256" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1170.255", + "cumtime_per_call": "585.127" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "2", + "tottime": "0.007", + "tottime_per_call": "0.004", + "cumtime": "1165.923", + "cumtime_per_call": "582.961" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1165.912", + "cumtime_per_call": "582.956" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "2", + "tottime": "0.020", + "tottime_per_call": "0.010", + "cumtime": "1165.834", + "cumtime_per_call": "582.917" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.841", + "cumtime_per_call": "0.480" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.526", + "cumtime_per_call": "1.263" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "2", + "tottime": "0.035", + "tottime_per_call": "0.018", + "cumtime": "2.093", + "cumtime_per_call": "1.046" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "13293", + "tottime": "0.014", + "tottime_per_call": "0.000", + "cumtime": "1.918", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "10", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.690", + "cumtime_per_call": "0.169" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "8", + "tottime": "0.003", + "tottime_per_call": "0.000", + "cumtime": "1.287", + "cumtime_per_call": "0.161" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.284", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.858", + "cumtime_per_call": "0.429" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "2", + "tottime": "0.003", + "tottime_per_call": "0.001", + "cumtime": "0.491", + "cumtime_per_call": "0.245" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "2", + "tottime": "0.002", + "tottime_per_call": "0.001", + "cumtime": "0.461", + "cumtime_per_call": "0.230" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.459", + "cumtime_per_call": "0.229" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.459", + "cumtime_per_call": "0.229" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.459", + "cumtime_per_call": "0.229" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.459", + "cumtime_per_call": "0.229" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.459", + "cumtime_per_call": "0.229" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.246", + "cumtime_per_call": "0.123" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.212", + "cumtime_per_call": "0.106" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.211", + "cumtime_per_call": "0.105" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "2", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.077", + "cumtime_per_call": "0.038" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "10", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.052", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.020", + "cumtime_per_call": "0.010" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "2", + "tottime": "0.004", + "tottime_per_call": "0.002", + "cumtime": "0.012", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "14000", + "tottime": "0.005", + "tottime_per_call": "0.000", + "cumtime": "0.007", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "2", + "tottime": "0.003", + "tottime_per_call": "0.002", + "cumtime": "0.005", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "2", + "tottime": "0.002", + "tottime_per_call": "0.001", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:95()", + "ncalls": "7368", + "tottime": "0.002", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "6633", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "20634", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "2", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/doralon.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "21", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 19:50:14", + "start_time": 1728503414.7633696, + "end_time": 1728509250.2443595, + "time": 5835.480989933014, + "files": 6631 }, "GOOD_PHARM": { - "execution_time": "2024-09-28 17:53:49", - "start_time": 1727546029.4208574, - "end_time": 1727546258.3901274, - "time": 228.96926999092102, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "131.006", + "cumtime_per_call": "131.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "130.974", + "cumtime_per_call": "130.974" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "130.974", + "cumtime_per_call": "130.974" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "130.973", + "cumtime_per_call": "130.973" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "128.259", + "cumtime_per_call": "128.259" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "128.257", + "cumtime_per_call": "128.257" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "128.239", + "cumtime_per_call": "128.239" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.429", + "cumtime_per_call": "2.429" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.416", + "cumtime_per_call": "2.416" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.416", + "cumtime_per_call": "2.416" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.416", + "cumtime_per_call": "2.416" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.416", + "cumtime_per_call": "2.416" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "2.416", + "cumtime_per_call": "2.416" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.284", + "cumtime_per_call": "0.071" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.197", + "cumtime_per_call": "0.197" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "0.162", + "cumtime_per_call": "0.162" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.153", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.118", + "cumtime_per_call": "0.024" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.081", + "cumtime_per_call": "0.081" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.079", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.079", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.042", + "cumtime_per_call": "0.042" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.042", + "cumtime_per_call": "0.042" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.018", + "cumtime_per_call": "0.018" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/good_pharm.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 21:27:30", + "start_time": 1728509250.2533228, + "end_time": 1728509381.2590764, + "time": 131.00575351715088, "files": 1000 }, "HAZI_HINAM": { - "execution_time": "2024-09-28 17:57:38", - "start_time": 1727546258.3911567, - "end_time": 1727546335.5806189, - "time": 77.189462184906, - "files": 180 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "112.157", + "cumtime_per_call": "112.157" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "112.125", + "cumtime_per_call": "112.125" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "112.125", + "cumtime_per_call": "112.125" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "112.125", + "cumtime_per_call": "112.125" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "111.895", + "cumtime_per_call": "111.895" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "111.894", + "cumtime_per_call": "111.894" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "111.888", + "cumtime_per_call": "111.888" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.141", + "cumtime_per_call": "0.141" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.139", + "cumtime_per_call": "0.139" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.139", + "cumtime_per_call": "0.139" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.139", + "cumtime_per_call": "0.139" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.139", + "cumtime_per_call": "0.139" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.088", + "cumtime_per_call": "0.022" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.055", + "cumtime_per_call": "0.055" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.044", + "cumtime_per_call": "0.044" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.043", + "cumtime_per_call": "0.009" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "318", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.042", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.032", + "cumtime_per_call": "0.032" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.027", + "cumtime_per_call": "0.007" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.027", + "cumtime_per_call": "0.007" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.017", + "cumtime_per_call": "0.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.017", + "cumtime_per_call": "0.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/hazihinam.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "302", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "302", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "604", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 21:29:41", + "start_time": 1728509381.2696736, + "end_time": 1728510401.2333632, + "time": 1019.9636895656586, + "files": 302 }, "HET_COHEN": { - "execution_time": "2024-09-28 17:58:55", - "start_time": 1727546335.5816627, - "end_time": 1727546342.982391, - "time": 7.400728464126587, - "files": 23 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.178", + "cumtime_per_call": "0.178" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.174", + "cumtime_per_call": "0.174" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/matrix.py:31(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.014", + "cumtime_per_call": "0.014" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/het_cohen.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/matrix.py:11(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 21:46:41", + "start_time": 1728510401.2411578, + "end_time": 1728510401.418835, + "time": 0.17767715454101562, + "files": 0 }, "KESHET": { - "execution_time": "2024-09-28 17:59:02", - "start_time": 1727546342.9833107, - "end_time": 1727546496.7949462, - "time": 153.81163549423218, - "files": 94 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "95.014", + "cumtime_per_call": "95.014" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "94.982", + "cumtime_per_call": "94.982" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "94.982", + "cumtime_per_call": "94.982" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "94.982", + "cumtime_per_call": "94.982" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "94.778", + "cumtime_per_call": "94.778" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "94.778", + "cumtime_per_call": "94.778" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "94.769", + "cumtime_per_call": "94.769" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.145", + "cumtime_per_call": "0.145" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.142", + "cumtime_per_call": "0.142" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.142", + "cumtime_per_call": "0.142" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.142", + "cumtime_per_call": "0.142" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.142", + "cumtime_per_call": "0.142" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.058", + "cumtime_per_call": "0.015" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.037", + "cumtime_per_call": "0.037" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.029", + "cumtime_per_call": "0.029" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "240", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.029", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.028", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.020", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.018", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.018", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.012", + "cumtime_per_call": "0.012" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.011", + "cumtime_per_call": "0.011" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/keshet.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "224", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "224", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "448", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 21:46:41", + "start_time": 1728510401.4400787, + "end_time": 1728510496.4573336, + "time": 95.01725482940674, + "files": 224 }, "KING_STORE": { - "execution_time": "2024-09-28 18:01:36", - "start_time": 1727546496.7960172, - "end_time": 1727547119.1490183, - "time": 622.3530011177063, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "616.578", + "cumtime_per_call": "616.578" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "616.532", + "cumtime_per_call": "616.532" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "616.532", + "cumtime_per_call": "616.532" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "616.532", + "cumtime_per_call": "616.532" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "560.856", + "cumtime_per_call": "560.856" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "560.855", + "cumtime_per_call": "560.855" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.004", + "tottime_per_call": "0.004", + "cumtime": "560.835", + "cumtime_per_call": "560.835" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "55.060", + "cumtime_per_call": "13.765" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "54.989", + "cumtime_per_call": "54.989" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "54.955", + "cumtime_per_call": "54.955" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "54.945", + "cumtime_per_call": "0.054" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.613", + "cumtime_per_call": "0.613" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.600", + "cumtime_per_call": "0.600" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.600", + "cumtime_per_call": "0.600" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.600", + "cumtime_per_call": "0.600" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.600", + "cumtime_per_call": "0.600" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.600", + "cumtime_per_call": "0.600" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.101", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.068", + "cumtime_per_call": "0.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.067", + "cumtime_per_call": "0.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.066", + "cumtime_per_call": "0.066" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.038", + "cumtime_per_call": "0.038" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.037", + "cumtime_per_call": "0.037" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.019", + "cumtime_per_call": "0.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/king_store.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 21:48:16", + "start_time": 1728510496.4653792, + "end_time": 1728513843.447128, + "time": 3346.981748819351, "files": 1000 }, "MAAYAN_2000": { - "execution_time": "2024-09-28 18:11:59", - "start_time": 1727547119.1500554, - "end_time": 1727547380.9200287, - "time": 261.76997327804565, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "719.871", + "cumtime_per_call": "719.871" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "719.828", + "cumtime_per_call": "719.828" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "719.828", + "cumtime_per_call": "719.828" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "719.828", + "cumtime_per_call": "719.828" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "607.091", + "cumtime_per_call": "607.091" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "607.089", + "cumtime_per_call": "607.089" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "607.075", + "cumtime_per_call": "607.075" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "112.076", + "cumtime_per_call": "28.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "111.987", + "cumtime_per_call": "111.987" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "111.950", + "cumtime_per_call": "111.950" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "111.939", + "cumtime_per_call": "0.110" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.659", + "cumtime_per_call": "0.659" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.647", + "cumtime_per_call": "0.647" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.647", + "cumtime_per_call": "0.647" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.647", + "cumtime_per_call": "0.647" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.647", + "cumtime_per_call": "0.647" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.647", + "cumtime_per_call": "0.647" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.121", + "cumtime_per_call": "0.024" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.085", + "cumtime_per_call": "0.085" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.079", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.079", + "cumtime_per_call": "0.020" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.046", + "cumtime_per_call": "0.046" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.046", + "cumtime_per_call": "0.046" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.014", + "cumtime_per_call": "0.014" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/maayan2000.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-09 22:44:03", + "start_time": 1728513843.459227, + "end_time": 1728518590.409105, + "time": 4746.949877977371, "files": 1000 }, "MAHSANI_ASHUK": { - "execution_time": "2024-09-28 18:16:20", - "start_time": 1727547380.9210534, - "end_time": 1727548209.458096, - "time": 828.5370426177979, - "files": 2381 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "179.952", + "cumtime_per_call": "179.952" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "179.925", + "cumtime_per_call": "179.925" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "179.925", + "cumtime_per_call": "179.925" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "179.925", + "cumtime_per_call": "179.925" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "129.951", + "cumtime_per_call": "129.951" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "129.949", + "cumtime_per_call": "129.949" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "129.944", + "cumtime_per_call": "129.944" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "48.078", + "cumtime_per_call": "12.019" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "48.032", + "cumtime_per_call": "48.032" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "48.017", + "cumtime_per_call": "48.017" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "404", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "48.014", + "cumtime_per_call": "0.119" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.895", + "cumtime_per_call": "1.895" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.481", + "cumtime_per_call": "0.740" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.481", + "cumtime_per_call": "0.740" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.481", + "cumtime_per_call": "0.740" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.481", + "cumtime_per_call": "0.740" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.481", + "cumtime_per_call": "0.740" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/matrix.py:31(get_data_from_page)", + "ncalls": "2", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.377", + "cumtime_per_call": "0.189" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.058", + "cumtime_per_call": "0.012" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.044", + "cumtime_per_call": "0.044" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.038", + "cumtime_per_call": "0.010" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.038", + "cumtime_per_call": "0.010" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.034", + "cumtime_per_call": "0.034" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "764", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.033", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/matrix.py:23(get_href_from_entry)", + "ncalls": "764", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.033", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.022", + "cumtime_per_call": "0.022" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.022", + "cumtime_per_call": "0.022" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:46(debug)", + "ncalls": "15", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/matrix.py:27(get_file_name_no_ext_from_entry)", + "ncalls": "764", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/machsani_ashuk.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/matrix.py:11(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "764", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "1528", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 00:03:10", + "start_time": 1728518590.420451, + "end_time": 1728520102.2098181, + "time": 1511.789367198944, + "files": 382 }, "MEGA": { - "execution_time": "2024-09-28 18:30:09", - "start_time": 1727548209.459298, - "end_time": 1727554914.047921, - "time": 6704.588623046875, - "files": 3940 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.392", + "cumtime_per_call": "5.392" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.380", + "cumtime_per_call": "5.380" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.380", + "cumtime_per_call": "5.380" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "5.380", + "cumtime_per_call": "5.380" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.566", + "cumtime_per_call": "3.566" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.546", + "cumtime_per_call": "1.773" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.546", + "cumtime_per_call": "1.773" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "4/2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.546", + "cumtime_per_call": "1.773" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.546", + "cumtime_per_call": "1.773" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "3.546", + "cumtime_per_call": "1.773" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.804", + "cumtime_per_call": "1.804" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.803", + "cumtime_per_call": "1.803" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.799", + "cumtime_per_call": "1.799" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:38(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.191", + "cumtime_per_call": "0.191" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.009", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "29", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:61()", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:63(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:65(get_herf_element)", + "ncalls": "33", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:71(get_path_from_herf)", + "ncalls": "22", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/mega.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:68(get_herf)", + "ncalls": "22", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:79()", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:85()", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/publishprice.py:74(get_name_from_herf)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "22", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 00:28:22", + "start_time": 1728520102.2380702, + "end_time": 1728520107.6301908, + "time": 5.392120599746704, + "files": 11 }, "NETIV_HASED": { - "execution_time": "2024-09-28 20:21:54", - "start_time": 1727554914.049241, - "end_time": 1727554914.2598617, - "time": 0.21062064170837402, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.190", + "cumtime_per_call": "0.190" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.189", + "cumtime_per_call": "0.189" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.189", + "cumtime_per_call": "0.189" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.189", + "cumtime_per_call": "0.189" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.186", + "cumtime_per_call": "0.186" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.177", + "cumtime_per_call": "0.177" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:20(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.009", + "cumtime_per_call": "0.009" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/nativ_hashed.py:13(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:31(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 00:28:27", + "start_time": 1728520107.6467896, + "end_time": 1728520107.836813, + "time": 0.19002342224121094, "files": 0 }, "MESHMAT_YOSEF_1": { - "execution_time": "2024-09-28 20:21:54", - "start_time": 1727554914.2611983, - "end_time": 1727555002.4540706, - "time": 88.1928722858429, - "files": 396 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "174.419", + "cumtime_per_call": "174.419" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "174.411", + "cumtime_per_call": "174.411" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "174.411", + "cumtime_per_call": "174.411" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "174.411", + "cumtime_per_call": "174.411" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "147.327", + "cumtime_per_call": "147.327" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "147.325", + "cumtime_per_call": "147.325" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "147.318", + "cumtime_per_call": "147.318" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "26.252", + "cumtime_per_call": "6.563" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "26.204", + "cumtime_per_call": "26.204" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "26.187", + "cumtime_per_call": "26.187" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "438", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "26.181", + "cumtime_per_call": "0.060" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.831", + "cumtime_per_call": "0.831" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.829", + "cumtime_per_call": "0.829" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.829", + "cumtime_per_call": "0.829" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.829", + "cumtime_per_call": "0.829" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.829", + "cumtime_per_call": "0.829" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.829", + "cumtime_per_call": "0.829" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.062", + "cumtime_per_call": "0.012" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.047", + "cumtime_per_call": "0.047" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.042", + "cumtime_per_call": "0.010" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.041", + "cumtime_per_call": "0.010" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.024", + "cumtime_per_call": "0.024" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.024", + "cumtime_per_call": "0.024" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/meshnat_yosef.py:16(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/meshnat_yosef.py:24(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/meshnat_yosef.py:29(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "422", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "844", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/meshnat_yosef.py:32()", + "ncalls": "422", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/meshnat_yosef.py:31()", + "ncalls": "422", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 00:28:27", + "start_time": 1728520107.8533406, + "end_time": 1728521200.6203043, + "time": 1092.7669637203217, + "files": 422 }, "MESHMAT_YOSEF_2": { - "execution_time": "2024-09-28 20:23:22", - "start_time": 1727555002.4567657, - "end_time": 1727558704.8806741, - "time": 3702.423908472061, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "659.980", + "cumtime_per_call": "659.980" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "659.955", + "cumtime_per_call": "659.955" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "659.955", + "cumtime_per_call": "659.955" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "659.954", + "cumtime_per_call": "659.954" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "551.078", + "cumtime_per_call": "551.078" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "551.074", + "cumtime_per_call": "551.074" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "551.056", + "cumtime_per_call": "551.056" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "107.863", + "cumtime_per_call": "26.966" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "107.773", + "cumtime_per_call": "107.773" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "107.733", + "cumtime_per_call": "107.733" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.002", + "tottime_per_call": "0.000", + "cumtime": "107.721", + "cumtime_per_call": "0.106" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "1.011", + "cumtime_per_call": "1.011" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.998", + "cumtime_per_call": "0.998" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.998", + "cumtime_per_call": "0.998" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.998", + "cumtime_per_call": "0.998" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.998", + "cumtime_per_call": "0.998" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.998", + "cumtime_per_call": "0.998" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.125", + "cumtime_per_call": "0.025" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.086", + "cumtime_per_call": "0.021" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.086", + "cumtime_per_call": "0.021" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.085", + "cumtime_per_call": "0.085" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.044", + "cumtime_per_call": "0.044" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.043", + "cumtime_per_call": "0.043" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.018", + "cumtime_per_call": "0.018" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/meshnat_yosef.py:46(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 00:46:40", + "start_time": 1728521200.6340995, + "end_time": 1728524972.0405202, + "time": 3771.4064207077026, "files": 1000 }, "OSHER_AD": { - "execution_time": "2024-09-28 21:25:04", - "start_time": 1727558704.8824348, - "end_time": 1727559824.0203772, - "time": 1119.137942314148, - "files": 396 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.326", + "cumtime_per_call": "0.326" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.325", + "cumtime_per_call": "0.325" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.325", + "cumtime_per_call": "0.325" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.325", + "cumtime_per_call": "0.325" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.160", + "cumtime_per_call": "0.160" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.158", + "cumtime_per_call": "0.158" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.158", + "cumtime_per_call": "0.158" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.156", + "cumtime_per_call": "0.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.156", + "cumtime_per_call": "0.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.156", + "cumtime_per_call": "0.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.156", + "cumtime_per_call": "0.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.007", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "18", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/osherad.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 01:49:32", + "start_time": 1728524972.0546088, + "end_time": 1728524972.381255, + "time": 0.326646089553833, + "files": 2 }, "POLIZER": { - "execution_time": "2024-09-28 21:43:44", - "start_time": 1727559824.0217428, - "end_time": 1727559845.9296584, - "time": 21.907915592193604, - "files": 51 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.364", + "cumtime_per_call": "16.364" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.359", + "cumtime_per_call": "16.359" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.359", + "cumtime_per_call": "16.359" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.359", + "cumtime_per_call": "16.359" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.214", + "cumtime_per_call": "16.214" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.214", + "cumtime_per_call": "16.214" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "16.211", + "cumtime_per_call": "16.211" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.135", + "cumtime_per_call": "0.135" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.133", + "cumtime_per_call": "0.133" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.133", + "cumtime_per_call": "0.133" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.133", + "cumtime_per_call": "0.133" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.133", + "cumtime_per_call": "0.133" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.009", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "31", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/polizer.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "15", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "15", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "30", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 01:49:32", + "start_time": 1728524972.4066072, + "end_time": 1728524988.7702718, + "time": 16.363664627075195, + "files": 15 }, "RAMI_LEVY": { - "execution_time": "2024-09-28 21:44:05", - "start_time": 1727559845.930972, - "end_time": 1727567691.9167693, - "time": 7845.985797166824, - "files": 1549 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "128.112", + "cumtime_per_call": "128.112" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "128.075", + "cumtime_per_call": "128.075" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "128.075", + "cumtime_per_call": "128.075" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "128.075", + "cumtime_per_call": "64.038" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "127.673", + "cumtime_per_call": "63.836" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "127.672", + "cumtime_per_call": "63.836" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "2", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "127.663", + "cumtime_per_call": "63.832" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.296", + "cumtime_per_call": "0.148" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.291", + "cumtime_per_call": "0.146" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.291", + "cumtime_per_call": "0.146" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.291", + "cumtime_per_call": "0.146" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.291", + "cumtime_per_call": "0.146" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.106", + "cumtime_per_call": "0.013" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.058", + "cumtime_per_call": "0.029" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "10", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.057", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "307", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.046", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "2", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.045", + "cumtime_per_call": "0.023" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.045", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.044", + "cumtime_per_call": "0.006" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.032", + "cumtime_per_call": "0.016" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.016", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.015", + "cumtime_per_call": "0.008" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.010", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.007", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.007", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.006", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "10", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/ramilevy.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "279", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "21", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:95()", + "ncalls": "140", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "141", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "422", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "8", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 01:49:48", + "start_time": 1728524988.7853355, + "end_time": 1728526943.5464442, + "time": 1954.761108636856, + "files": 139 }, "SALACH_DABACH": { - "execution_time": "2024-09-28 23:54:51", - "start_time": 1727567691.9187238, - "end_time": 1727567707.2292793, - "time": 15.310555458068848, - "files": 22 + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.480", + "cumtime_per_call": "7.480" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.473", + "cumtime_per_call": "7.473" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.473", + "cumtime_per_call": "7.473" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:38(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.473", + "cumtime_per_call": "7.473" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.309", + "cumtime_per_call": "7.309" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.309", + "cumtime_per_call": "7.309" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "7.306", + "cumtime_per_call": "7.306" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:78(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.156", + "cumtime_per_call": "0.156" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.154", + "cumtime_per_call": "0.154" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.154", + "cumtime_per_call": "0.154" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.154", + "cumtime_per_call": "0.154" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:277(collect_from_ftp)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.154", + "cumtime_per_call": "0.154" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.008", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "26", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.003" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/salachdabach.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:20(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/cerberus.py:108()", + "ncalls": "10", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "10", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:84()", + "ncalls": "20", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 02:22:23", + "start_time": 1728526943.5607216, + "end_time": 1728526951.0410368, + "time": 7.480315208435059, + "files": 10 }, "SHEFA_BARCART_ASHEM": { - "execution_time": "2024-09-28 23:55:07", - "start_time": 1727567707.2307417, - "end_time": 1727573810.4829617, - "time": 6103.25221991539, - "files": 1000 - }, - "SHUFERSAL": { - "execution_time": "2024-09-29 01:36:50", - "start_time": 1727573810.4847457, - "end_time": 1727573897.0302274, - "time": 86.54548168182373, - "files": 0 - }, - "SHUK_AHIR": { - "execution_time": "2024-09-29 01:38:17", - "start_time": 1727573897.0332646, - "end_time": 1727578495.0227447, - "time": 4597.989480018616, - "files": 1000 - }, - "STOP_MARKET": { - "execution_time": "2024-09-29 02:54:55", - "start_time": 1727578495.0245478, - "end_time": 1727580362.1401498, - "time": 1867.115602016449, - "files": 35 - }, - "SUPER_PHARM": { - "execution_time": "2024-09-29 03:26:02", - "start_time": 1727580362.1417964, - "end_time": 1727581430.9625795, - "time": 1068.8207831382751, - "files": 0 - }, - "SUPER_YUDA": { - "execution_time": "2024-09-29 03:43:50", - "start_time": 1727581430.9655378, - "end_time": 1727586100.6588426, - "time": 4669.693304777145, + "status": [ + { + "function": "/workspaces/israeli-supermarket-scarpers/stress_test.py:45(full_execution)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "687.725", + "cumtime_per_call": "687.725" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:200(retry_files_decorator)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "687.679", + "cumtime_per_call": "687.679" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:209(__retry_files)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "687.679", + "cumtime_per_call": "687.679" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:106(scrape)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "687.679", + "cumtime_per_call": "687.679" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:30(execute_in_parallel)", + "ncalls": "1", + "tottime": "0.002", + "tottime_per_call": "0.002", + "cumtime": "578.287", + "cumtime_per_call": "578.287" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:50(run_tasks)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "578.282", + "cumtime_per_call": "578.282" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:62()", + "ncalls": "1", + "tottime": "0.003", + "tottime_per_call": "0.003", + "cumtime": "578.253", + "cumtime_per_call": "578.253" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:28(wrapper)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "108.904", + "cumtime_per_call": "27.226" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:113(on_scrape_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "108.813", + "cumtime_per_call": "108.813" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:109(log_folder_details)", + "ncalls": "1", + "tottime": "0.005", + "tottime_per_call": "0.005", + "cumtime": "108.778", + "cumtime_per_call": "108.778" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/logger.py:40(info)", + "ncalls": "1017", + "tottime": "0.002", + "tottime_per_call": "0.000", + "cumtime": "108.761", + "cumtime_per_call": "0.107" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:66(collect_files_details_from_site)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.485", + "cumtime_per_call": "0.485" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:133(retry_decorator)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.471", + "cumtime_per_call": "0.471" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:36(__retry_internal)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.471", + "cumtime_per_call": "0.471" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:76(inner)", + "ncalls": "2/1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.471", + "cumtime_per_call": "0.471" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:229(session_and_check_status)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.471", + "cumtime_per_call": "0.471" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/connection.py:179(session_with_cookies)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.471", + "cumtime_per_call": "0.471" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:48(_write_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.122", + "cumtime_per_call": "0.024" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:61(on_download_completed)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.085", + "cumtime_per_call": "0.085" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:130(_insert_an_update)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.083", + "cumtime_per_call": "0.021" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:70(insert_document)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.082", + "cumtime_per_call": "0.021" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:100(_add_downloaded_files_to_list)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.043", + "cumtime_per_call": "0.043" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:55(insert_documents)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.043", + "cumtime_per_call": "0.043" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:61()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.029", + "cumtime_per_call": "0.029" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:40(apply_limit_zip)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:46(on_collected_details)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.005", + "cumtime_per_call": "0.005" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:79(apply_limit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:36(get_data_from_page)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.004", + "cumtime_per_call": "0.004" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:33(_read_database)", + "ncalls": "5", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.003", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:32(extract_task_from_entry)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/loop.py:5(defualt_aggregtion_function)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.002", + "cumtime_per_call": "0.002" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:43(get_file_name_no_ext_from_entry)", + "ncalls": "1000", + "tottime": "0.001", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:185(unique)", + "ncalls": "1", + "tottime": "0.001", + "tottime_per_call": "0.001", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:208(scrape)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:34()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:67(filter_already_downloaded)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.001", + "cumtime_per_call": "0.001" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:28(on_scraping_start)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:164(_now)", + "ncalls": "6", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers/shefa_barcart_ashem.py:8(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:18(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/retry.py:244(compute_retry)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:15(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:24(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:98()", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/bina.py:39(get_href_from_entry)", + "ncalls": "1000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:202(post_scraping)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:46()", + "ncalls": "2000", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/scraper_status.py:21(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:73(get)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:10(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:68(all_scrapers_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:71()", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:29(_get_database_file_path)", + "ncalls": "11", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:84(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:56()", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:90(is_scraper_enabled)", + "ncalls": "38", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:22(_ensure_db_file_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/json_file.py:17(_ensure_db_directory_exists)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:232(make_storage_path_dir)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/lock_utils.py:11(get_lock)", + "ncalls": "4", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:98(convert_unit)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:7(__init__)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:41(_build_query_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/status.py:78(get_output_folder)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/web.py:25(get_request_url)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/databases/base.py:23(is_collection_enabled)", + "ncalls": "2", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/scrappers_factory.py:53(all_active)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/apsx.py:47(_get_all_possible_query_string_params)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/engines/engine.py:36(get_storage_path)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + }, + { + "function": "/workspaces/israeli-supermarket-scarpers/il_supermarket_scarper/utils/folders_name.py:45(is_valid_folder_name)", + "ncalls": "1", + "tottime": "0.000", + "tottime_per_call": "0.000", + "cumtime": "0.000", + "cumtime_per_call": "0.000" + } + ], + "execution_time": "2024-10-10 02:22:31", + "start_time": 1728526951.0568316, + "end_time": 1728530899.7414224, + "time": 3948.684590816498, "files": 1000 } } \ No newline at end of file