riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 19:21:31 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

remove tokenizer_file from herbert slow tokenizer init

Browse Source
This commit is contained in:
SaulLu 2022-01-25 19:13:16 +01:00
parent da0895330b
commit f5e10007b7
1 changed files with 0 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/herbert/tokenization_herbert.py
View File

@ -62,7 +62,6 @@ class HerbertTokenizer(XLMTokenizer):
self,
vocab_file,
merges_file,
tokenizer_file=None,
cls_token="<s>",
unk_token="<unk>",
pad_token="<pad>",
@ -75,7 +74,6 @@ class HerbertTokenizer(XLMTokenizer):
super().__init__(
vocab_file,
merges_file,
tokenizer_file=None,
cls_token=cls_token,
unk_token=unk_token,
pad_token=pad_token,