riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 03:01:07 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix tokenizer export for LLamaTokenizerFast (#27222)

Browse Source 
* fix tokenizer

* fix tokenizer
This commit is contained in:
Mayank Mishra 2023-11-06 14:56:18 +05:30 committed by GitHub
parent cc3e478185
commit b026b5ca6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/code_llama/tokenization_code_llama_fast.py
View File

@ -149,6 +149,8 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token,
bos_token=bos_token,
eos_token=eos_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
prefix_token=prefix_token,
middle_token=middle_token,
suffix_token=suffix_token,

2
src/transformers/models/llama/tokenization_llama_fast.py
View File

@ -128,6 +128,8 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token,
bos_token=bos_token,
eos_token=eos_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
use_default_system_prompt=use_default_system_prompt,
**kwargs,
)