From 19cc2c084e265d1cf2ae2218147cd52bdd71bf1a Mon Sep 17 00:00:00 2001 From: lukovnikov Date: Mon, 18 Mar 2019 15:13:35 +0100 Subject: [PATCH] same --- pytorch_pretrained_bert/optimization_openai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/optimization_openai.py b/pytorch_pretrained_bert/optimization_openai.py index 7f56a1284c5..99ac15e1089 100644 --- a/pytorch_pretrained_bert/optimization_openai.py +++ b/pytorch_pretrained_bert/optimization_openai.py @@ -26,7 +26,8 @@ logger = logging.getLogger(__name__) def warmup_cosine(x, warmup=0.002): if x < warmup: return x/warmup - return 0.5 * (1.0 + torch.cos(math.pi * x)) + x_ = (x - warmup) / (1 - warmup) # progress after warmup + return 0.5 * (1. + math.cos(math.pi * x_)) def warmup_constant(x, warmup=0.002): """ Linearly increases learning rate over `warmup`*`t_total` (as provided to OpenAIAdam) training steps.