riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 19:21:31 +06:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Typo in "end_of_word_suffix" (#13477)

Browse Source 
But does it really work?
This commit is contained in:
Koichi Yasuoka 2021-09-09 00:26:07 +09:00 committed by GitHub
parent 2a15e8ccfb
commit 330d83fdbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_utils_fast.py
View File

@ -666,7 +666,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
kwargs["continuing_subword_prefix"] = tokenizer_json["model"]["continuing_subword_prefix"]
if (
tokenizer_json["model"]["type"] == "BPE"
and "end_of_work_suffix" not in kwargs
and "end_of_word_suffix" not in kwargs
and tokenizer_json["model"]["end_of_word_suffix"] is not None
):
kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"]