update

src/lightning/data/streaming/cache.py

@@ -51,7 +51,7 @@ def __init__(
         chunk_size: Optional[int] = None,
         chunk_bytes: Optional[Union[int, str]] = None,
         item_loader: Optional[BaseItemLoader] = None,
-        max_cache_size: Union[int, str] = "10GB",
+        max_cache_size: Union[int, str] = "200GB",
         serializers: Optional[Dict[str, Serializer]] = None,
     ):
         """The Cache enables to optimise dataset format for cloud training. This is done by grouping several elements

src/lightning/data/streaming/reader.py

@@ -46,7 +46,6 @@ def __init__(self, config: ChunksConfig, item_loader, max_cache_size: Optional[i
         self._to_download_queue: multiprocessing.Queue = multiprocessing.Queue()
         self._to_delete_queue: multiprocessing.Queue = multiprocessing.Queue()
         self._to_stop_queue: multiprocessing.Queue = multiprocessing.Queue()
-        self._pre_downloaded = 0
 
     def download(self, chunk_indexes: List[int]) -> None:
         """Receive the list of the chunk indices to download for the current epoch."""
@@ -77,7 +76,6 @@ def _delete_chunks(self):
                     while (self._max_cache_size and self._chunks_index_to_be_deleted and total >= self._max_cache_size):
                         self._delete(self._chunks_index_to_be_deleted.pop(0))
                         total = _get_folder_size(self._parent_cache_dir)
-                        self._pre_downloaded -= 1
                 else:
                     self._chunks_index_to_be_deleted.append(chunk_index)
         except Empty:
@@ -94,8 +92,6 @@ def run(self) -> None:
             try:
                 chunk_index = self._to_download_queue.get(timeout=0.01)
                 self._config.download_chunk_from_index(chunk_index)
-
-                self._pre_downloaded += 1
             except Empty:
                 pass
             except OSError as e: