diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index a546b731fcb..942ddde04d1 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -384,7 +384,9 @@ class Trainer:
         dataset.set_format(type=dataset.format["type"], columns=columns)
 
     def _get_train_sampler(self) -> Optional[torch.utils.data.sampler.Sampler]:
-        if not isinstance(self.train_dataset, collections.abc.Sized):
+        if isinstance(self.train_dataset, torch.utils.data.IterableDataset) or not isinstance(
+            self.train_dataset, collections.abc.Sized
+        ):
             return None
         elif is_torch_tpu_available():
             return get_tpu_sampler(self.train_dataset)