riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

fix #1260 - remove special logic for decoding pairs of sequence

Browse Source
This commit is contained in:
thomwolf 2019-10-01 19:09:13 -04:00
parent 963529e29b
commit 391db836ab
1 changed files with 4 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
transformers/tokenization_utils.py
View File

@ -933,20 +933,11 @@ class PreTrainedTokenizer(object):
sub_texts.append(self.convert_tokens_to_string(current_sub_text)) sub_texts.append(self.convert_tokens_to_string(current_sub_text))
text = ''.join(sub_texts) text = ''.join(sub_texts)
if self._sep_token is not None and self._sep_token in text: if clean_up_tokenization_spaces:
text = text.replace(self._cls_token, self._sep_token) clean_text = self.clean_up_tokenization(text)
split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self._sep_token))) return clean_text
if clean_up_tokenization_spaces:
clean_text = [self.clean_up_tokenization(text) for text in split_text]
return clean_text
else:
return split_text
else: else:
if clean_up_tokenization_spaces: return text
clean_text = self.clean_up_tokenization(text)
return clean_text
else:
return text
@property @property
def special_tokens_map(self): def special_tokens_map(self):