mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-31 02:02:21 +06:00
fixing tokenization of extra_id symbols in T5Tokenizer. Related to issue 4021 (#4353)
This commit is contained in:
parent
5139733623
commit
3dea40b858
@ -503,6 +503,7 @@ class SpecialTokensMixin:
|
||||
if key in self.SPECIAL_TOKENS_ATTRIBUTES:
|
||||
if key == "additional_special_tokens":
|
||||
assert isinstance(value, (list, tuple)) and all(isinstance(t, str) for t in value)
|
||||
setattr(self, key, value)
|
||||
elif isinstance(value, AddedTokenFast):
|
||||
setattr(self, key, str(value))
|
||||
elif isinstance(value, str):
|
||||
|
Loading…
Reference in New Issue
Block a user