diff --git a/examples/run_glue.py b/examples/run_glue.py
index ea5ac5bbb77..8749593a1f4 100644
--- a/examples/run_glue.py
+++ b/examples/run_glue.py
@@ -171,22 +171,22 @@ def train(args, train_dataset, model, tokenizer):
                 global_step += 1
 
                 if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
-                    # Log metrics
-                    logs = {'step': global_step}
+                    logs = {}
                     if args.local_rank == -1 and args.evaluate_during_training:  # Only evaluate when single GPU otherwise metrics may not average well
                         results = evaluate(args, model, tokenizer)
                         for key, value in results.items():
                             eval_key = 'eval_{}'.format(key)
-                            tb_writer.add_scalar(eval_key, value, global_step)
-                            logs[eval_key] = str(value)
-                    logging_loss = tr_loss
+                            logs[eval_key] = value
+
                     loss_scalar = (tr_loss - logging_loss) / args.logging_steps
                     learning_rate_scalar = scheduler.get_lr()[0]
-                    tb_writer.add_scalar('lr', learning_rate_scalar, global_step)
-                    tb_writer.add_scalar('loss', loss_scalar, global_step)
                     logs['learning_rate'] = learning_rate_scalar
                     logs['loss'] = loss_scalar
-                    print(json.dumps(logs))
+                    logging_loss = tr_loss
+
+                    for key, value in logs.items():
+                        tb_writer.add_scalar(key, value, global_step)
+                    print(json.dumps({**logs, **{'step': global_step}}))
 
                 if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
                     # Save model checkpoint