From 95792a948e68b8dc89a68bb9cc5bb7fc0a8a3e9c Mon Sep 17 00:00:00 2001
From: rmroczkowski <64909124+rmroczkowski@users.noreply.github.com>
Date: Thu, 22 Oct 2020 11:48:29 +0200
Subject: [PATCH] Herbert tokenizer auto load (#7968)

---
 src/transformers/tokenization_auto.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/transformers/tokenization_auto.py b/src/transformers/tokenization_auto.py
index 24d391b596d..45699c195c5 100644
--- a/src/transformers/tokenization_auto.py
+++ b/src/transformers/tokenization_auto.py
@@ -73,6 +73,7 @@ from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_fsmt import FSMTTokenizer
 from .tokenization_funnel import FunnelTokenizer
 from .tokenization_gpt2 import GPT2Tokenizer
+from .tokenization_herbert import HerbertTokenizer
 from .tokenization_layoutlm import LayoutLMTokenizer
 from .tokenization_longformer import LongformerTokenizer
 from .tokenization_lxmert import LxmertTokenizer
@@ -123,6 +124,7 @@ if is_tokenizers_available():
     from .tokenization_electra_fast import ElectraTokenizerFast
     from .tokenization_funnel_fast import FunnelTokenizerFast
     from .tokenization_gpt2_fast import GPT2TokenizerFast
+    from .tokenization_herbert_fast import HerbertTokenizerFast
     from .tokenization_layoutlm_fast import LayoutLMTokenizerFast
     from .tokenization_longformer_fast import LongformerTokenizerFast
     from .tokenization_lxmert_fast import LxmertTokenizerFast
@@ -147,6 +149,7 @@ else:
     ElectraTokenizerFast = None
     FunnelTokenizerFast = None
     GPT2TokenizerFast = None
+    HerbertTokenizerFast = None
     LayoutLMTokenizerFast = None
     LongformerTokenizerFast = None
     LxmertTokenizerFast = None
@@ -191,6 +194,7 @@ TOKENIZER_MAPPING = OrderedDict(
         (LayoutLMConfig, (LayoutLMTokenizer, LayoutLMTokenizerFast)),
         (DPRConfig, (DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast)),
         (SqueezeBertConfig, (SqueezeBertTokenizer, SqueezeBertTokenizerFast)),
+        (BertConfig, (HerbertTokenizer, HerbertTokenizerFast)),
         (BertConfig, (BertTokenizer, BertTokenizerFast)),
         (OpenAIGPTConfig, (OpenAIGPTTokenizer, OpenAIGPTTokenizerFast)),
         (GPT2Config, (GPT2Tokenizer, GPT2TokenizerFast)),