riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

add prefix space ignored in llama #29625 (#30964)

Browse Source 
* add prefix space ignored in llama #29625

* adding test with add_prefix_space=False

* ruff

---------

Co-authored-by: Ita Zaporozhets <itazaporozhets@Itas-MBP.localdomain>
This commit is contained in:
Ita Zaporozhets 2024-05-24 10:03:00 +02:00 committed by GitHub
parent 6657fb5fed
commit 7f6e87413f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/models/llama/tokenization_llama_fast.py
View File

@ -163,6 +163,7 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
use_default_system_prompt=use_default_system_prompt,
add_prefix_space=add_prefix_space,
legacy=legacy,
**kwargs,
)

4
tests/models/llama/test_tokenization_llama.py
View File

@ -602,6 +602,10 @@ class LlamaIntegrationTest(unittest.TestCase):
self.assertEqual(decoded_tokens, "hello")
def test_no_prefix_space(self):
tokenizer_no_prefix_space = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", add_prefix_space=False)
no_prefix_space_tokens = tokenizer_no_prefix_space.tokenize("Hey")
self.assertEqual(no_prefix_space_tokens, ["H", "ey"])
tokenizer = LlamaTokenizerFast.from_pretrained(
"huggyllama/llama-7b", legacy=False, from_slow=True, add_prefix_space=False
)