diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 254f7d8e6e3..9635dc40a3f 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -1017,7 +1017,7 @@ class Trainer:
                         "yield to errors or unwanted behaviors."
                     )
 
-            if self.deepspeed:
+            if args.deepspeed:
                 # will be resumed in deepspeed_init
                 pass
             else: