riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-03 03:31:05 +06:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

restart cosine lr schedule

Browse Source
This commit is contained in:
lukovnikov 2019-03-12 13:22:23 +01:00
parent 51efde54a9
commit baf66d1419
1 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pytorch_pretrained_bert/optimization.py
View File

@ -69,7 +69,23 @@ class WarmupCosineSchedule(LRSchedule):
return progress / self.warmup
else:
progress = (progress - self.warmup) / (1 - self.warmup) # progress after warmup
return 0.5 * (1. + math.cos(math.pi * self.cycles * 2 * progress))
return 0.5 * (1. + math.cos(math.pi * ((self.cycles * 2 * progress) % 1))
class WarmupCosineWithRestartsSchedule(WarmupCosineSchedule):
warn_t_total = True
def __init__(self, warmup=0.002, t_total=-1, cycles=1., **kw):
super(WarmupCosineWithRestartsSchedule, self).__init__(warmup=warmup, t_total=t_total, cycles=cycles, **kw)
def get_lr_(self, progress):
if self.t_total <= 0:
return 1.
if progress < self.warmup:
return progress / self.warmup
else:
progress = (progress - self.warmup) / (1 - self.warmup) # progress after warmup
ret = 0.5 * (1. + math.cos(math.pi * self.cycles * 2 * progress))
return ret
class WarmupConstantSchedule(LRSchedule):