riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-04 13:20:12 +06:00
Code Issues Actions 21 Packages Projects Releases Wiki Activity

fix: use correct var names for check_tokenizers script (#33702)

Browse Source
This commit is contained in:
Nicola De Angeli 2024-09-26 17:24:46 +02:00 committed by GitHub
parent 46841d3eb2
commit c716fc0e48
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/check_tokenizers.py
View File

@ -88,7 +88,7 @@ def check_details(line, spm_ids, tok_ids, slow, fast):
if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width] if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
] ]
for j in possible_matches: for j in possible_matches:
if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], sp, tok) and check_details( if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], slow, fast) and check_details(
line, line,
spm_ids[first + i : last], spm_ids[first + i : last],
tok_ids[first + j : last], tok_ids[first + j : last],