diff --git a/examples/run_glue.py b/examples/run_glue.py
index 62d655ecc98..6387ed448af 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -382,7 +382,8 @@ def main():
 
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`
-        model.save_pretrained(args.output_dir)
+        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
         tokenizer.save_pretrained(args.output_dir)
 
         # Good practice: save your training arguments together with the trained model