riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 10:12:23 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

Make the order of additional special tokens deterministic (#5704)

Browse Source 
* Make the order of additional special tokens deterministic regardless of hash seeds

* Fix
This commit is contained in:
Gong Linyuan 2020-08-04 14:38:30 +08:00 committed by GitHub
parent d740351f7d
commit b390a5672a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/tokenization_utils_base.py
View File

@ -23,7 +23,7 @@ import json
import logging
import os
import warnings
from collections import UserDict
from collections import OrderedDict, UserDict
from enum import Enum
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
@ -1071,7 +1071,7 @@ class SpecialTokensMixin:
set_attr = self.special_tokens_map_extended
for attr_value in set_attr.values():
all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value])
all_toks = list(set(all_toks))
all_toks = list(OrderedDict.fromkeys(all_toks))
return all_toks
@property