From 7ead04ce1453940914abfdd24c826814c117f49e Mon Sep 17 00:00:00 2001 From: Anthony MOI Date: Thu, 26 Dec 2019 14:39:39 -0500 Subject: [PATCH] FastPreTrainedTokenizer => PreTrainedTokenizerFast --- src/transformers/tokenization_bert.py | 4 ++-- src/transformers/tokenization_gpt2.py | 4 ++-- src/transformers/tokenization_utils.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/tokenization_bert.py b/src/transformers/tokenization_bert.py index 5362806bdc0..4d416540378 100644 --- a/src/transformers/tokenization_bert.py +++ b/src/transformers/tokenization_bert.py @@ -22,7 +22,7 @@ import unicodedata import tokenizers as tk -from .tokenization_utils import FastPreTrainedTokenizer, PreTrainedTokenizer +from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer logger = logging.getLogger(__name__) @@ -529,7 +529,7 @@ def _is_punctuation(char): return False -class BertTokenizerFast(FastPreTrainedTokenizer): +class BertTokenizerFast(PreTrainedTokenizerFast): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION diff --git a/src/transformers/tokenization_gpt2.py b/src/transformers/tokenization_gpt2.py index 4cc1d4708e1..1cdc263e25e 100644 --- a/src/transformers/tokenization_gpt2.py +++ b/src/transformers/tokenization_gpt2.py @@ -23,7 +23,7 @@ from functools import lru_cache import regex as re import tokenizers as tk -from .tokenization_utils import FastPreTrainedTokenizer, PreTrainedTokenizer +from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer logger = logging.getLogger(__name__) @@ -249,7 +249,7 @@ class GPT2Tokenizer(PreTrainedTokenizer): return vocab_file, merge_file -class GPT2TokenizerFast(FastPreTrainedTokenizer): +class GPT2TokenizerFast(PreTrainedTokenizerFast): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py index c52ba6ff791..535322be10a 100644 --- a/src/transformers/tokenization_utils.py +++ b/src/transformers/tokenization_utils.py @@ -1412,9 +1412,9 @@ class PreTrainedTokenizer(object): return out_string -class FastPreTrainedTokenizer(PreTrainedTokenizer): +class PreTrainedTokenizerFast(PreTrainedTokenizer): def __init__(self, **kwargs): - super(FastPreTrainedTokenizer, self).__init__(**kwargs) + super(PreTrainedTokenizerFast, self).__init__(**kwargs) @property def tokenizer(self):