riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 33 Packages Projects Releases Wiki Activity

Don't push checkpoints to hub in no_trainer scripts (#16703)

Browse Source 
Adds checkpoint prefixes to the gitignore if `push_to_hub` is used along with `checkpointint_steps`
This commit is contained in:
Zachary Mueller 2022-04-11 12:42:45 -04:00 committed by GitHub
parent c04619ecf3
commit d4b3e359aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 63 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
examples/pytorch/language-modeling/run_clm_no_trainer.py
View File

@ -39,6 +39,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator, DistributedType
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
CONFIG_MAPPING,
@ -50,7 +51,6 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
@ -258,6 +258,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()
@ -542,7 +548,6 @@ def main():
if args.output_dir is not None:
output_dir = os.path.join(args.output_dir, output_dir)
accelerator.save_state(output_dir)
if completed_steps >= args.max_train_steps:
break

8
examples/pytorch/language-modeling/run_mlm_no_trainer.py
View File

@ -39,6 +39,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator, DistributedType
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
CONFIG_MAPPING,
@ -50,7 +51,6 @@ from transformers import (
DataCollatorForLanguageModeling,
SchedulerType,
get_scheduler,
set_seed,
)
from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
@ -269,6 +269,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/multiple-choice/run_swag_no_trainer.py
View File

@ -37,6 +37,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
CONFIG_MAPPING,
@ -49,7 +50,6 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import PaddingStrategy, get_full_repo_name
@ -296,6 +296,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py
View File

@ -34,6 +34,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
AdamW,
@ -45,7 +46,6 @@ from transformers import (
XLNetTokenizerFast,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import check_min_version, get_full_repo_name
from transformers.utils.versions import require_version
@ -290,6 +290,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/question-answering/run_qa_no_trainer.py
View File

@ -35,6 +35,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
CONFIG_MAPPING,
@ -48,7 +49,6 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import check_min_version, get_full_repo_name
from transformers.utils.versions import require_version
@ -320,6 +320,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/summarization/run_summarization_no_trainer.py
View File

@ -36,6 +36,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from filelock import FileLock
from huggingface_hub import Repository
from transformers import (
@ -48,7 +49,6 @@ from transformers import (
DataCollatorForSeq2Seq,
SchedulerType,
get_scheduler,
set_seed,
)
from transformers.utils import get_full_repo_name, is_offline_mode
from transformers.utils.versions import require_version
@ -346,6 +346,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/text-classification/run_glue_no_trainer.py
View File

@ -28,6 +28,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
AdamW,
@ -39,7 +40,6 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
@ -223,6 +223,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/token-classification/run_ner_no_trainer.py
View File

@ -34,6 +34,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
CONFIG_MAPPING,
@ -47,7 +48,6 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
@ -277,6 +277,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()

8
examples/pytorch/translation/run_translation_no_trainer.py
View File

@ -35,6 +35,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
CONFIG_MAPPING,
@ -49,7 +50,6 @@ from transformers import (
SchedulerType,
default_data_collator,
get_scheduler,
set_seed,
)
from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
@ -319,6 +319,12 @@ def main():
else:
repo_name = args.hub_model_id
repo = Repository(args.output_dir, clone_from=repo_name)
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
if "step_*" not in gitignore:
gitignore.write("step_*\n")
if "epoch_*" not in gitignore:
gitignore.write("epoch_*\n")
elif args.output_dir is not None:
os.makedirs(args.output_dir, exist_ok=True)
accelerator.wait_for_everyone()