From e140af84e6b90c4e7bdd49ef1a6ab37367e14b97 Mon Sep 17 00:00:00 2001 From: Sanger2000 Date: Fri, 21 Jan 2022 18:33:16 -0500 Subject: [PATCH 1/2] Gets length of dataloader in number of batches --- deepspeed/runtime/dataloader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deepspeed/runtime/dataloader.py b/deepspeed/runtime/dataloader.py index acd21d9557fe..2e446df3f50d 100644 --- a/deepspeed/runtime/dataloader.py +++ b/deepspeed/runtime/dataloader.py @@ -68,10 +68,15 @@ def __init__(self, self.device_count = device_count self.batch_size = batch_size self.pin_memory = pin_memory - self.len = len(self.data_sampler) self.data = None self.dataloader_drop_last = dataloader_drop_last + if self.dataloader_drop_last: + self.len = len(self.data_sampler // self.batch_size) + else: + from math import ceil + self.len = ceil(len(self.data_sampler) / self.batch_size) + def __iter__(self): self._create_dataloader() return self From f0657740164db35a06c0efe11c9a9cf0b827cc10 Mon Sep 17 00:00:00 2001 From: Sanger2000 Date: Sun, 13 Feb 2022 19:24:53 -0500 Subject: [PATCH 2/2] Fixed parenthesis mismatch --- deepspeed/runtime/dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/dataloader.py b/deepspeed/runtime/dataloader.py index 2e446df3f50d..84b206e1eecd 100644 --- a/deepspeed/runtime/dataloader.py +++ b/deepspeed/runtime/dataloader.py @@ -72,7 +72,7 @@ def __init__(self, self.dataloader_drop_last = dataloader_drop_last if self.dataloader_drop_last: - self.len = len(self.data_sampler // self.batch_size) + self.len = len(self.data_sampler) // self.batch_size else: from math import ceil self.len = ceil(len(self.data_sampler) / self.batch_size)