riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

Merge pull request #1831 from iedmrc/gpt2-tokenization-sum-func-replacement

Browse Source 
sum() is replaced by itertools.chain.from_iterable()
This commit is contained in:
Thomas Wolf 2019-11-14 22:11:54 +01:00 committed by GitHub
commit 1a237d7f42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
transformers/tokenization_utils.py
View File

@ -21,6 +21,7 @@ import os
import json
import six
import copy
import itertools
from io import open
from .file_utils import cached_path, is_tf_available, is_torch_available
@ -641,9 +642,9 @@ class PreTrainedTokenizer(object):
tokenized_text += [sub_text]
text_list = tokenized_text
return sum((self._tokenize(token, **kwargs) if token not \
return list(itertools.chain.from_iterable((self._tokenize(token, **kwargs) if token not \
in self.added_tokens_encoder and token not in self.all_special_tokens \
else [token] for token in tokenized_text), [])
else [token] for token in tokenized_text)))
added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens
tokenized_text = split_on_tokens(added_tokens, text)