riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

rebase

Browse Source
This commit is contained in:
Ita Zaporozhets 2024-07-29 15:26:08 +02:00
parent ef0bd27e54
commit e8bfe9051f
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/models/llama/test_tokenization_llama.py
View File

@ -827,6 +827,7 @@ class CommonSpmIntegrationTests(unittest.TestCase):
@unittest.skip("@require_read_token does not work? getting gated repo error")
@require_read_token
@unittest.skip
def test_bos_eos_tokens(self):
tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b", add_bos_token=False, add_eos_token=True)
assert tokenizer("hello")["input_ids"][0] != tokenizer.bos_token_id # no bos token