riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Improve naming and only do regex when necessary

Browse Source
This commit is contained in:
jinoobaek-qz 2019-10-07 15:48:39 -07:00 committed by Lysandre Debut
parent bf34a252b8
commit 69629c4f0f
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
examples/run_lm_finetuning.py
View File

@ -117,16 +117,16 @@ def _rotate_checkpoints(args, checkpoint_prefix, use_mtime=False):
if len(glob_checkpoints) <= args.save_total_limit: if len(glob_checkpoints) <= args.save_total_limit:
return return
checkpoints_sorted = [] ordering_and_checkpoint_path = []
for path in glob_checkpoints: for path in glob_checkpoints:
regex_match = re.match('.*{}-([0-9]+)'.format(checkpoint_prefix), path) if use_mtime:
if regex_match and regex_match.groups(): ordering_and_checkpoint_path.append((os.path.getmtime(path), path))
if use_mtime: else:
checkpoints_sorted.append((os.path.getmtime(path), path)) regex_match = re.match('.*{}-([0-9]+)'.format(checkpoint_prefix), path)
else: if regex_match and regex_match.groups():
checkpoints_sorted.append((int(regex_match.groups()[0]), path)) ordering_and_checkpoint_path.append((int(regex_match.groups()[0]), path))
checkpoints_sorted = sorted(checkpoints_sorted) checkpoints_sorted = sorted(ordering_and_checkpoint_path)
checkpoints_sorted = [checkpoint[1] for checkpoint in checkpoints_sorted] checkpoints_sorted = [checkpoint[1] for checkpoint in checkpoints_sorted]
number_of_checkpoints_to_delete = max(0, len(checkpoints_sorted) - args.save_total_limit) number_of_checkpoints_to_delete = max(0, len(checkpoints_sorted) - args.save_total_limit)
checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete] checkpoints_to_be_deleted = checkpoints_sorted[:number_of_checkpoints_to_delete]