riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add missing comment quotes (#17379)

Browse Source
This commit is contained in:
Leon Derczynski 2022-06-29 12:16:36 +02:00 committed by GitHub
parent e113c5cb64
commit b8142753f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/tokenization_{{cookiecutter.lowercase_modelname}}.py
View File

@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer):
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
"Initialisation"
""" Initialisation """
@property
def vocab_size(self):
"Returns vocab size"
""" Returns vocab size """
def get_vocab(self):
"Returns vocab as a dict"
""" Returns vocab as a dict """
def _tokenize(self, text):
""" Returns a tokenized string. """