diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 400527a225cb..296317d783dc 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -374,7 +374,9 @@ def _remove_unused_columns(self, dataset: "datasets.Dataset", description: Optio
         dataset.set_format(type=dataset.format["type"], columns=columns)
 
     def _get_train_sampler(self) -> Optional[torch.utils.data.sampler.Sampler]:
-        if not isinstance(self.train_dataset, collections.abc.Sized):
+        if isinstance(self.train_dataset, torch.utils.data.IterableDataset) or not isinstance(
+            self.train_dataset, collections.abc.Sized
+        ):
             return None
         elif is_torch_tpu_available():
             return get_tpu_sampler(self.train_dataset)