riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Don't modify labels inplace in LabelSmoother (#13464)

Browse Source
This commit is contained in:
Sylvain Gugger 2021-09-08 07:45:36 -04:00 committed by GitHub
parent c164c651dc
commit cd66539662
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/trainer_pt_utils.py
View File

@ -458,7 +458,7 @@ class LabelSmoother:
padding_mask = labels.eq(self.ignore_index) padding_mask = labels.eq(self.ignore_index)
# In case the ignore_index is -100, the gather will fail, so we replace labels by 0. The padding_mask # In case the ignore_index is -100, the gather will fail, so we replace labels by 0. The padding_mask
# will ignore them in any case. # will ignore them in any case.
labels.clamp_min_(0) labels = torch.clamp(labels, min=0)
nll_loss = log_probs.gather(dim=-1, index=labels) nll_loss = log_probs.gather(dim=-1, index=labels)
# works for fp16 input tensor too, by internally upcasting it to fp32 # works for fp16 input tensor too, by internally upcasting it to fp32
smoothed_loss = log_probs.sum(dim=-1, keepdim=True, dtype=torch.float32) smoothed_loss = log_probs.sum(dim=-1, keepdim=True, dtype=torch.float32)