From c716fc0e4845ae26f9f080e6cf92856af92cd628 Mon Sep 17 00:00:00 2001 From: Nicola De Angeli <112023843+niqodea@users.noreply.github.com> Date: Thu, 26 Sep 2024 17:24:46 +0200 Subject: [PATCH] fix: use correct var names for check_tokenizers script (#33702) --- scripts/check_tokenizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_tokenizers.py b/scripts/check_tokenizers.py index 6d6773b00e8..ba73460ce4d 100644 --- a/scripts/check_tokenizers.py +++ b/scripts/check_tokenizers.py @@ -88,7 +88,7 @@ def check_details(line, spm_ids, tok_ids, slow, fast): if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width] ] for j in possible_matches: - if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], sp, tok) and check_details( + if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], slow, fast) and check_details( line, spm_ids[first + i : last], tok_ids[first + j : last],