riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

more options on special tokens

Browse Source
This commit is contained in:
thomwolf 2019-02-04 17:26:25 +01:00
parent 05f961840b
commit 01a3966bc6
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pytorch_pretrained_bert/tokenization_openai.py
View File

@ -131,6 +131,10 @@ class OpenAIGPTTokenizer(object):
return len(self.encoder) + len(self.special_tokens) return len(self.encoder) + len(self.special_tokens)
def set_special_tokens(self, special_tokens): def set_special_tokens(self, special_tokens):
""" Add a list of additional tokens to the encoder.
The additional tokens are indexed starting from the last index of the
current vocabulary in the order of the `special_tokens` list.
"""
if not special_tokens: if not special_tokens:
self.special_tokens = {} self.special_tokens = {}
self.special_tokens_decoder = {} self.special_tokens_decoder = {}
@ -210,18 +214,19 @@ class OpenAIGPTTokenizer(object):
) )
return ids return ids
def convert_ids_to_tokens(self, ids): def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
"""Converts a sequence of ids in BPE tokens using the vocab.""" """Converts a sequence of ids in BPE tokens using the vocab."""
tokens = [] tokens = []
for i in ids: for i in ids:
if i in self.special_tokens_decoder: if i in self.special_tokens_decoder:
tokens.append(self.special_tokens_decoder[i]) if not skip_special_tokens:
tokens.append(self.special_tokens_decoder[i])
else: else:
tokens.append(self.decoder[i]) tokens.append(self.decoder[i])
return tokens return tokens
def decode(self, ids): def decode(self, ids, skip_special_tokens=False):
"""Converts a sequence of ids in a string.""" """Converts a sequence of ids in a string."""
tokens = self.convert_ids_to_tokens(ids) tokens = self.convert_ids_to_tokens(ids, skip_special_tokens=skip_special_tokens)
out_string = ''.join(tokens).replace('</w>', ' ') out_string = ''.join(tokens).replace('</w>', ' ')
return out_string return out_string