riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

remove two tokens that should not be suppressed (#20302)

Browse Source
This commit is contained in:
Arthur 2022-11-18 08:57:42 +01:00 committed by GitHub
parent 532e60bedf
commit 95754b47a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/models/whisper/configuration_whisper.py
View File

@ -35,7 +35,7 @@ WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
# fmt: off
NON_SPEECH_TOKENS = [
1, 2, 6, 7, 8, 9, 10, 12, 14, 25,
1, 2, 7, 8, 9, 10, 14, 25,
26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
@ -46,7 +46,7 @@ NON_SPEECH_TOKENS = [
34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
NON_SPEECH_TOKENS_MULTI = [
1, 2, 6, 7, 8, 9, 10, 12, 14, 25,
1, 2, 7, 8, 9, 10, 14, 25,
26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,