diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 2ff4f93fdfc..6beb0a6b68f 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -3250,7 +3250,7 @@ class Trainer:
         elif is_sagemaker_mp_enabled():
             tensors = smp_gather(tensors)
         elif (self.args.distributed_state is not None and self.args.distributed_state.distributed_type != "NO") or (
-            self.args.distributed_state is None and self.local_rank != -1
+            self.args.distributed_state is None and self.args.local_rank != -1
         ):
             tensors = distributed_concat(tensors)
         return tensors