riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #389 from lukovnikov/master

Browse Source 
Fix cosine schedule
This commit is contained in:
Thomas Wolf 2019-04-03 11:21:43 +02:00 committed by GitHub
commit db4dccd1b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pytorch_pretrained_bert/optimization.py
View File

@ -26,7 +26,8 @@ logger = logging.getLogger(__name__)
def warmup_cosine(x, warmup=0.002): def warmup_cosine(x, warmup=0.002):
if x < warmup: if x < warmup:
return x/warmup return x/warmup
return 0.5 * (1.0 + torch.cos(math.pi * x)) x_ = (x - warmup) / (1 - warmup) # progress after warmup -
return 0.5 * (1. + math.cos(math.pi * x_))
def warmup_constant(x, warmup=0.002): def warmup_constant(x, warmup=0.002):
""" Linearly increases learning rate over `warmup`*`t_total` (as provided to BertAdam) training steps. """ Linearly increases learning rate over `warmup`*`t_total` (as provided to BertAdam) training steps.

3
pytorch_pretrained_bert/optimization_openai.py
View File

@ -26,7 +26,8 @@ logger = logging.getLogger(__name__)
def warmup_cosine(x, warmup=0.002): def warmup_cosine(x, warmup=0.002):
if x < warmup: if x < warmup:
return x/warmup return x/warmup
return 0.5 * (1.0 + torch.cos(math.pi * x)) x_ = (x - warmup) / (1 - warmup) # progress after warmup
return 0.5 * (1. + math.cos(math.pi * x_))
def warmup_constant(x, warmup=0.002): def warmup_constant(x, warmup=0.002):
""" Linearly increases learning rate over `warmup`*`t_total` (as provided to OpenAIAdam) training steps. """ Linearly increases learning rate over `warmup`*`t_total` (as provided to OpenAIAdam) training steps.