riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 18:22:34 +06:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

ruff

Browse Source
This commit is contained in:
Ita Zaporozhets 2024-09-09 11:09:39 +02:00
parent aad810db47
commit 0d3085db94
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/models/llama/test_tokenization_llama.py
View File

@ -857,6 +857,7 @@ class CommonSpmIntegrationTests(unittest.TestCase):
tokens = tokenizer_pretrained_fast.tokenize("hello", add_special_tokens=True) tokens = tokenizer_pretrained_fast.tokenize("hello", add_special_tokens=True)
self.assertEqual(tokens[-1], new_eos_token) self.assertEqual(tokens[-1], new_eos_token)
@require_tiktoken @require_tiktoken
@require_read_token @require_read_token
class TikTokenIntegrationTests(unittest.TestCase): class TikTokenIntegrationTests(unittest.TestCase):