riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

Improve naming and only do regex when necessary

Browse Source
This commit is contained in:
jinoobaek-qz 2019-10-07 15:48:39 -07:00 committed by Lysandre Debut
parent bf34a252b8
commit 69629c4f0f
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
examples/run_lm_finetuning.py
View File

@ -117,16 +117,16 @@ def _rotate_checkpoints(args, checkpoint_prefix, use_mtime=False):
if len(glob_checkpoints) <= args.save_total_limit:
return
checkpoints_sorted = []
ordering_and_checkpoint_path = []
for path in glob_checkpoints:
regex_match = re.match('.*{}-([0-9]+)'.format(checkpoint_prefix), path)
if regex_match and regex_match.groups():
if use_mtime:
checkpoints_sorted.append((os.path.getmtime(path), path))
else:
checkpoints_sorted.append((int(regex_match.groups()[0]), path))
if use_mtime:
ordering_and_checkpoint_path.append((os.path.getmtime(path), path))
else:
regex_match = re.match('.*{}-([0-9]+)'.format(checkpoint_prefix), path)
if regex_match and regex_match.groups():
ordering_and_checkpoint_path.append((int(regex_match.groups()[0]), path))
checkpoints_sorted = sorted(checkpoints_sorted)
checkpoints_sorted = sorted(ordering_and_checkpoint_path)
checkpoints_sorted = [checkpoint[1] for checkpoint in checkpoints_sorted]
number_of_checkpoints_to_delete = max(0, len(checkpoints_sorted) - args.save_total_limit)
checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete]