[blocked by #789] Fix/dataset check [wip]

CHANGELOG.md

@@ -26,6 +26,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed a bug where the model checkpointer didn't write to the same directory as the logger ([#771](https://github.com/PyTorchLightning/pytorch-lightning/pull/771))
 - Fixed a bug where the `TensorBoardLogger` class would create an additional empty log file during fitting ([#777](https://github.com/PyTorchLightning/pytorch-lightning/pull/777))
 - Fixed a bug where `global_step` was advanced incorrectly when using `accumulate_grad_batches > 1` ([#832](https://github.com/PyTorchLightning/pytorch-lightning/pull/832))
+- Fixed a bug when using a `DataLoader` without a `dataset` attribute ([#883](https://github.com/PyTorchLightning/pytorch-lightning/pull/883))
+- Fixed a bug when using a `IterableDataset` inside the train `DataLoader` ([#883](https://github.com/PyTorchLightning/pytorch-lightning/pull/883))
 
 ## [0.6.0] - 2020-01-21
 ### Added

pytorch_lightning/trainer/data_loading.py

@@ -55,13 +55,24 @@ def init_train_dataloader(self, model):
         self.get_train_dataloader = model.train_dataloader
 
         # determine number of training batches
-        if EXIST_ITER_DATASET and isinstance(self.get_train_dataloader().dataset, IterableDataset):
+        self.is_iterable_train_dataloader = (
+            EXIST_ITER_DATASET and hasattr(self.get_train_dataloader(), 'dataset') and
+            isinstance(self.get_train_dataloader().dataset, IterableDataset))
+        if self.is_iterable_train_dataloader:
             self.num_training_batches = float('inf')
         else:
             self._percent_range_check('train_percent_check')
 
-            self.num_training_batches = len(self.get_train_dataloader())
-            self.num_training_batches = int(self.num_training_batches * self.train_percent_check)
+            try:
+                self.num_training_batches = len(self.get_train_dataloader())
+                self.num_training_batches = int(self.num_training_batches *
+                                                self.train_percent_check)
+            except TypeError as e:
+                # Assume infinite data loading if the dataloader doesn't implement __len__
+                if 'has no len' in str(e):
+                    self.num_training_batches = float('inf')
+                else:
+                    raise e
 
         # determine when to check validation
         # if int passed in, val checks that often
@@ -74,6 +85,14 @@ def init_train_dataloader(self, model):
                     f"to the number of the training batches ({self.num_training_batches}). "
                     f"If you want to disable validation set `val_percent_check` to 0.0 instead.")
         else:
+            if self.num_training_batches == float('inf'):
+                # support IterableDataset for train data
+                msg = """
+                When using an infinite DataLoader (e.g. with an IterableDataset or when DataLoader
+                does not implement `__len__`) for `train_dataloader`, `Trainer(val_check_interval)`
+                must be an int. An int k specifies checking validation every k training batches.
+                """
+                raise MisconfigurationException(msg)
             self._percent_range_check('val_check_interval')
 
             self.val_check_batch = int(self.num_training_batches * self.val_check_interval)
@@ -210,17 +229,6 @@ def get_dataloaders(self, model):
             self.get_test_dataloaders()
             self.get_val_dataloaders()
 
-        # support IterableDataset for train data
-        self.is_iterable_train_dataloader = (
-            EXIST_ITER_DATASET and isinstance(self.get_train_dataloader().dataset, IterableDataset))
-        if self.is_iterable_train_dataloader and not isinstance(self.val_check_interval, int):
-            m = '''
-            When using an iterableDataset for `train_dataloader`,
-            `Trainer(val_check_interval)` must be an int.
-            An int k specifies checking validation every k training batches
-            '''
-            raise MisconfigurationException(m)
-
     def determine_data_use_amount(self, train_percent_check, val_percent_check,
                                   test_percent_check, overfit_pct):
         """

pytorch_lightning/trainer/training_loop.py

@@ -322,7 +322,7 @@ def train(self):
 
             # reset progress bar
             # .reset() doesn't work on disabled progress bar so we should check
-            if not self.main_progress_bar.disable:
+            if not self.main_progress_bar.disable and num_iterations != float('inf'):
                 self.main_progress_bar.reset(num_iterations)
             desc = f'Epoch {epoch + 1}' if not self.is_iterable_train_dataloader else ''
             self.main_progress_bar.set_description(desc)

tests/test_trainer.py

@@ -16,7 +16,9 @@
     LightningTestMultipleDataloadersMixin,
 )
 from pytorch_lightning.core.lightning import load_hparams_from_tags_csv
+from pytorch_lightning.core.decorators import data_loader
 from pytorch_lightning.trainer.logging import TrainerLoggingMixin
+from pytorch_lightning.utilities.debugging import MisconfigurationException
 
 
 def test_no_val_module(tmpdir):
@@ -373,6 +375,64 @@ def test_model_freeze_unfreeze():
     model.unfreeze()
 
 
+def test_inf_train_dataloader(tmpdir):
+    """Test inf train data loader (e.g. IterableDataset)"""
+    tutils.reset_seed()
+
+    class CurrentTestModel(
+        LightningTestModelBase
+    ):
+
+        @data_loader
+        def train_dataloader(self):
+            dataloader = self._dataloader(train=True)
+
+            class CustomInfDataLoader:
+                def __init__(self, dataloader):
+                    self.dataloader = dataloader
+                    self.iter = iter(dataloader)
+                    self.count = 0
+
+                def __iter__(self):
+                    self.count = 0
+                    return self
+
+                def __next__(self):
+                    if self.count >= 500:
+                        raise StopIteration
+                    self.count = self.count + 1
+                    try:
+                        return next(self.iter)
+                    except StopIteration:
+                        self.iter = iter(self.dataloader)
+                        return next(self.iter)
+
+            return CustomInfDataLoader(dataloader)
+
+    hparams = tutils.get_hparams()
+    model = CurrentTestModel(hparams)
+
+    # fit model
+    with pytest.raises(MisconfigurationException):
+        trainer = Trainer(
+            default_save_path=tmpdir,
+            max_epochs=1,
+            val_check_interval=0.5
+        )
+        trainer.fit(model)
+
+    # logger file to get meta
+    trainer = Trainer(
+        default_save_path=tmpdir,
+        max_epochs=1,
+        val_check_interval=50,
+    )
+    result = trainer.fit(model)
+
+    # verify training completed
+    assert result == 1
+
+
 def test_multiple_val_dataloader(tmpdir):
     """Verify multiple val_dataloader."""
     tutils.reset_seed()