riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-24 06:48:58 +06:00
Code Issues Actions 22 Packages Projects Releases Wiki Activity

Sync preprocesses before loading the processor at run_speech_recognition_ctc.py (#21926)

Browse Source 
* Update run_speech_recognition_ctc.py

Make sure all processes wait until data is saved before loading the processor from the output_dit

* Make sure all processes wait until data is saved before loading the processor from the output_dit

* Update run_speech_recognition_ctc.py

* Update run_speech_recognition_seq2seq.py
This commit is contained in:
Mikel Penagarikano 2023-04-05 15:36:04 +02:00 committed by GitHub
parent f49b0762a1
commit d5239bab5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/pytorch/speech-recognition/run_speech_recognition_ctc.py
View File

@ -673,6 +673,9 @@ def main():
return metrics return metrics
# Now save everything to be able to create a single processor later # Now save everything to be able to create a single processor later
# make sure all processes wait until data is saved
with training_args.main_process_first():
# only the main process saves them
if is_main_process(training_args.local_rank): if is_main_process(training_args.local_rank):
# save feature extractor, tokenizer and config # save feature extractor, tokenizer and config
feature_extractor.save_pretrained(training_args.output_dir) feature_extractor.save_pretrained(training_args.output_dir)

3
examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py
View File

@ -506,6 +506,9 @@ def main():
return {"wer": wer} return {"wer": wer}
# 9. Create a single speech processor # 9. Create a single speech processor
# make sure all processes wait until data is saved
with training_args.main_process_first():
# only the main process saves them
if is_main_process(training_args.local_rank): if is_main_process(training_args.local_rank):
# save feature extractor, tokenizer and config # save feature extractor, tokenizer and config
feature_extractor.save_pretrained(training_args.output_dir) feature_extractor.save_pretrained(training_args.output_dir)