riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

push (#11667)

Browse Source
This commit is contained in:
Patrick von Platen 2021-05-10 17:38:17 +01:00 committed by GitHub
parent 05a930671f
commit dcb0e61430
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/auto/tokenization_auto.py
View File

@ -63,6 +63,7 @@ from .configuration_auto import (
BertConfig,
BertGenerationConfig,
BigBirdConfig,
BigBirdPegasusConfig,
BlenderbotConfig,
BlenderbotSmallConfig,
CamembertConfig,
@ -275,6 +276,7 @@ TOKENIZER_MAPPING = OrderedDict(
(Wav2Vec2Config, (Wav2Vec2CTCTokenizer, None)),
(GPTNeoConfig, (GPT2Tokenizer, GPT2TokenizerFast)),
(LukeConfig, (LukeTokenizer, None)),
(BigBirdPegasusConfig, (PegasusTokenizer, PegasusTokenizerFast)),
]
)