riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

Add-warning-tokenizer (#20826)

Browse Source 
* add fast not use warning

* update
This commit is contained in:
Arthur 2022-12-21 18:18:34 +01:00 committed by GitHub
parent 76d02feadb
commit 4a433e321f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/transformers/models/auto/tokenization_auto.py
View File

@ -576,9 +576,14 @@ class AutoTokenizer:
tokenizer_class_name, tokenizer_fast_class_name = tokenizer_class_tuple
if use_fast and tokenizer_fast_class_name is not None:
tokenizer_class = tokenizer_class_from_name(tokenizer_fast_class_name)
if use_fast:
if tokenizer_fast_class_name is not None:
tokenizer_class = tokenizer_class_from_name(tokenizer_fast_class_name)
else:
logger.warning(
"`use_fast` is set to `True` but the tokenizer class does not have a fast version. "
" Falling back to the slow version."
)
if tokenizer_class is None:
tokenizer_class = tokenizer_class_from_name(tokenizer_class_name)