riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 19:21:31 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix tokenizer for XLMRobertaXL (#19004)

Browse Source 
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar 2022-09-13 14:04:14 +02:00 committed by GitHub
parent 2848c9ce42
commit 2886f7f08a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/transformers/models/auto/tokenization_auto.py
View File

@ -271,7 +271,13 @@ else:
"XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
),
),
("xlm-roberta-xl", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
(
"xlm-roberta-xl",
(
"XLMRobertaTokenizer" if is_sentencepiece_available() else None,
"XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
),
),
(
"xlnet",
(

2
src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py
View File

@ -54,7 +54,7 @@ logger = logging.get_logger(__name__)
_CHECKPOINT_FOR_DOC = "xlm-roberta-xlarge"
_CONFIG_FOR_DOC = "XLMRobertaXLConfig"
_TOKENIZER_FOR_DOC = "RobertaTokenizer"
_TOKENIZER_FOR_DOC = "XLMRobertaTokenizer"
XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST = [
"facebook/xlm-roberta-xl",