diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 40dc51b80d2..3a5d3cec788 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -78,7 +78,7 @@ if is_tokenizers_available(): from tokenizers import Encoding as EncodingFast else: - @dataclass(frozen=False, eq=True) + @dataclass(repr=True, frozen=False, eq=True) class AddedToken: """ AddedToken represents a token to be added to a Tokenizer An AddedToken can have special options defining the @@ -104,6 +104,9 @@ else: def __str__(self): return self.content + def __repr__(self): + return self.content + @dataclass class EncodingFast: """This is dummy class because without the `tokenizers` library we don't have these objects anyway"""