riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 10:12:23 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

Scale loss before backward (#35207)

Browse Source
This commit is contained in:
Quentin Gallouédec 2024-12-23 16:16:38 +01:00 committed by GitHub
parent f5264a86ee
commit 3cd3cd50ac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/trainer.py
View File

@ -3698,10 +3698,12 @@ class Trainer:
with amp.scale_loss(loss, self.optimizer) as scaled_loss:
scaled_loss.backward()
else:
self.accelerator.backward(loss, **kwargs)
# Finally we need to normalize the loss for reporting
if num_items_in_batch is None:
return loss.detach() / self.args.gradient_accumulation_steps
loss /= self.args.gradient_accumulation_steps
self.accelerator.backward(loss, **kwargs)
return loss.detach()
def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):