riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 19:21:31 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Ignore not initialized NO_CONFIG_TOKENIZERs (#10936)

Browse Source
This commit is contained in:
Masatoshi Suzuki 2021-03-29 23:26:15 +09:00 committed by GitHub
parent ddea8771c6
commit cc2366bbb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/auto/tokenization_auto.py
View File

@ -288,7 +288,7 @@ def tokenizer_class_from_name(class_name: str):
all_tokenizer_classes = (
[v[0] for v in TOKENIZER_MAPPING.values() if v[0] is not None]
+ [v[1] for v in TOKENIZER_MAPPING.values() if v[1] is not None]
+ NO_CONFIG_TOKENIZER
+ [v for v in NO_CONFIG_TOKENIZER if v is not None]
)
for c in all_tokenizer_classes:
if c.__name__ == class_name: