riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

[lm examples] fix overflow in perplexity calc (#11855)

Browse Source 
* fix overflow in perplexity calc

* use inf

* fix
This commit is contained in:
Stas Bekman 2021-05-25 08:11:26 -07:00 committed by GitHub
parent 7630c11f32
commit 6287c929c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 20 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/pytorch/language-modeling/run_clm.py
View File

@ -440,7 +440,10 @@ def main():
max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
try:
perplexity = math.exp(metrics["eval_loss"]) perplexity = math.exp(metrics["eval_loss"])
except OverflowError:
perplexity = float("inf")
metrics["perplexity"] = perplexity metrics["perplexity"] = perplexity
trainer.log_metrics("eval", metrics) trainer.log_metrics("eval", metrics)

3
examples/pytorch/language-modeling/run_clm_no_trainer.py
View File

@ -442,7 +442,10 @@ def main():
losses = torch.cat(losses) losses = torch.cat(losses)
losses = losses[: len(eval_dataset)] losses = losses[: len(eval_dataset)]
try:
perplexity = math.exp(torch.mean(losses)) perplexity = math.exp(torch.mean(losses))
except OverflowError:
perplexity = float("inf")
logger.info(f"epoch {epoch}: perplexity: {perplexity}") logger.info(f"epoch {epoch}: perplexity: {perplexity}")

3
examples/pytorch/language-modeling/run_mlm.py
View File

@ -469,7 +469,10 @@ def main():
max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
try:
perplexity = math.exp(metrics["eval_loss"]) perplexity = math.exp(metrics["eval_loss"])
except OverflowError:
perplexity = float("inf")
metrics["perplexity"] = perplexity metrics["perplexity"] = perplexity
trainer.log_metrics("eval", metrics) trainer.log_metrics("eval", metrics)

3
examples/pytorch/language-modeling/run_mlm_no_trainer.py
View File

@ -486,7 +486,10 @@ def main():
losses = torch.cat(losses) losses = torch.cat(losses)
losses = losses[: len(eval_dataset)] losses = losses[: len(eval_dataset)]
try:
perplexity = math.exp(torch.mean(losses)) perplexity = math.exp(torch.mean(losses))
except OverflowError:
perplexity = float("inf")
logger.info(f"epoch {epoch}: perplexity: {perplexity}") logger.info(f"epoch {epoch}: perplexity: {perplexity}")

3
examples/pytorch/language-modeling/run_plm.py
View File

@ -445,7 +445,10 @@ def main():
max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
try:
perplexity = math.exp(metrics["eval_loss"]) perplexity = math.exp(metrics["eval_loss"])
except OverflowError:
perplexity = float("inf")
metrics["perplexity"] = perplexity metrics["perplexity"] = perplexity
trainer.log_metrics("eval", metrics) trainer.log_metrics("eval", metrics)