riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-23 14:29:01 +06:00
Code Issues Actions 25 Packages Projects Releases Wiki Activity

Fix Llama 3 TikToken conversion (#33538)

Browse Source 
* Fix Llama 3 TikToken conversion

* No need to add tokens again
This commit is contained in:
Pedro Cuenca 2024-09-20 01:28:33 +02:00 committed by GitHub
parent 4d8908df27
commit 0c718f16d1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/transformers/models/llama/convert_llama_weights_to_hf.py
View File

@ -332,7 +332,7 @@ def write_model(
class Llama3Converter(TikTokenConverter): class Llama3Converter(TikTokenConverter):
def __init__(self, vocab_file, special_tokens=None, instruct=False, model_max_length=None, **kwargs): def __init__(self, vocab_file, special_tokens=None, instruct=False, model_max_length=None, **kwargs):
super().__init__(vocab_file, **kwargs) super().__init__(vocab_file, additional_special_tokens=special_tokens, **kwargs)
tokenizer = self.converted() tokenizer = self.converted()
chat_template = ( chat_template = (
"{% set loop_messages = messages %}" "{% set loop_messages = messages %}"
@ -345,7 +345,6 @@ class Llama3Converter(TikTokenConverter):
"{% endfor %}" "{% endfor %}"
"{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}" "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"
) )
tokenizer.add_special_tokens(special_tokens)
self.tokenizer = PreTrainedTokenizerFast( self.tokenizer = PreTrainedTokenizerFast(
tokenizer_object=tokenizer, tokenizer_object=tokenizer,