Fix broken example generation script for Llama3 (#38062)

Fix broken example generation script for llama3
This commit is contained in:
Yong Hoon Shin 2025-05-20 01:53:43 -07:00 committed by GitHub
parent 7a611f0afd
commit 555715f418
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -34,7 +34,6 @@ from transformers import (
GPT2Tokenizer,
GPTJForCausalLM,
LlamaForCausalLM,
LlamaTokenizer,
OpenAIGPTLMHeadModel,
OpenAIGPTTokenizer,
OPTForCausalLM,
@ -63,7 +62,7 @@ MODEL_CLASSES = {
"xlm": (XLMWithLMHeadModel, XLMTokenizer),
"gptj": (GPTJForCausalLM, AutoTokenizer),
"bloom": (BloomForCausalLM, BloomTokenizerFast),
"llama": (LlamaForCausalLM, LlamaTokenizer),
"llama": (LlamaForCausalLM, AutoTokenizer),
"opt": (OPTForCausalLM, GPT2Tokenizer),
}