From 934e2799da633c1cfa46b4a6be0f812ea65fa336 Mon Sep 17 00:00:00 2001 From: "Li-Huai (Allan) Lin" Date: Thu, 2 Dec 2021 03:16:52 +0800 Subject: [PATCH] Fix mask token handling (#14364) * Fix mask token handling * Revert "Fix mask token handling" This reverts commit daaa3f5291b1f71e5bc3604ca281c000000c4648. * Fix FNet mask token tokenization --- src/transformers/models/fnet/tokenization_fnet.py | 9 +++++++-- src/transformers/models/fnet/tokenization_fnet_fast.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/fnet/tokenization_fnet.py b/src/transformers/models/fnet/tokenization_fnet.py index 1e1bfc050a8..7d9f248d868 100644 --- a/src/transformers/models/fnet/tokenization_fnet.py +++ b/src/transformers/models/fnet/tokenization_fnet.py @@ -113,8 +113,13 @@ class FNetTokenizer(PreTrainedTokenizer): sp_model_kwargs: Optional[Dict[str, Any]] = None, **kwargs ) -> None: - # Mask token behave like a normal word, i.e. include the space before it - mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token + # Mask token behave like a normal word, i.e. include the space before it and + # is included in the raw text, there should be a match in a non-normalized sentence. + mask_token = ( + AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False) + if isinstance(mask_token, str) + else mask_token + ) self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs diff --git a/src/transformers/models/fnet/tokenization_fnet_fast.py b/src/transformers/models/fnet/tokenization_fnet_fast.py index 78640b0f26e..099fd9c0c5d 100644 --- a/src/transformers/models/fnet/tokenization_fnet_fast.py +++ b/src/transformers/models/fnet/tokenization_fnet_fast.py @@ -107,8 +107,13 @@ class FNetTokenizerFast(PreTrainedTokenizerFast): mask_token="[MASK]", **kwargs ): - # Mask token behave like a normal word, i.e. include the space before it - mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token + # Mask token behave like a normal word, i.e. include the space before it and + # is included in the raw text, there should be a match in a non-normalized sentence. + mask_token = ( + AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False) + if isinstance(mask_token, str) + else mask_token + ) super().__init__( vocab_file,