riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

[Speech Recognition] - Distributed training: Make sure vocab file removal and creation don't interfer (#14161)

Browse Source 
* up

* better
This commit is contained in:
Patrick von Platen 2021-10-26 15:59:33 +02:00 committed by GitHub
parent 840fc8dbca
commit f5ed19f57d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
examples/pytorch/speech-recognition/run_speech_recognition_ctc.py
View File

@ -395,20 +395,20 @@ def main():
# the training and evaluation datasets # the training and evaluation datasets
# We need to make sure that only first rank saves vocabulary # We need to make sure that only first rank saves vocabulary
# make sure all processes wait until vocab is created # make sure all processes wait until vocab is created
vocab_file = os.path.join(training_args.output_dir, "vocab.json")
with training_args.main_process_first(desc="dataset map vocabulary creation"): with training_args.main_process_first():
vocab_dict = create_vocabulary_from_data(raw_datasets)
vocab_file = os.path.join(training_args.output_dir, "vocab.json")
# save vocab dict to be loaded into tokenizer
os.makedirs(training_args.output_dir, exist_ok=True)
if training_args.overwrite_output_dir and os.path.isfile(vocab_file): if training_args.overwrite_output_dir and os.path.isfile(vocab_file):
os.remove(vocab_file) os.remove(vocab_file)
with training_args.main_process_first(desc="dataset map vocabulary creation"):
if not os.path.isfile(vocab_file): if not os.path.isfile(vocab_file):
with open(vocab_file, "w") as vocab_file: os.makedirs(training_args.output_dir, exist_ok=True)
json.dump(vocab_dict, vocab_file) vocab_dict = create_vocabulary_from_data(raw_datasets)
# save vocab dict to be loaded into tokenizer
with open(vocab_file, "w") as file:
json.dump(vocab_dict, file)
# 4. Now we can instantiate the configuration, feature extractor, tokenizer and model # 4. Now we can instantiate the configuration, feature extractor, tokenizer and model
# Note for distributed training, the .from_pretrained methods guarantee that only # Note for distributed training, the .from_pretrained methods guarantee that only