riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Adding __repr__ function and repr=True in dataclass

Browse Source
This commit is contained in:
Uri Alon 2023-09-19 01:21:31 +00:00
parent 1a1439225a
commit 18839505d1
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/transformers/tokenization_utils_base.py
View File

@ -78,7 +78,7 @@ if is_tokenizers_available():
from tokenizers import Encoding as EncodingFast from tokenizers import Encoding as EncodingFast
else: else:
@dataclass(frozen=False, eq=True) @dataclass(repr=True, frozen=False, eq=True)
class AddedToken: class AddedToken:
""" """
AddedToken represents a token to be added to a Tokenizer An AddedToken can have special options defining the AddedToken represents a token to be added to a Tokenizer An AddedToken can have special options defining the
@ -104,6 +104,9 @@ else:
def __str__(self): def __str__(self):
return self.content return self.content
def __repr__(self):
return self.content
@dataclass @dataclass
class EncodingFast: class EncodingFast:
"""This is dummy class because without the `tokenizers` library we don't have these objects anyway""" """This is dummy class because without the `tokenizers` library we don't have these objects anyway"""