From db8c7caeb6b3969a2153b36ba3e5fdef6534c1d6 Mon Sep 17 00:00:00 2001 From: Viktor Scherbakov Date: Thu, 1 Aug 2024 14:30:10 +0200 Subject: [PATCH] Empty list in defaults for LLaMA special tokens during weights conversion (#32342) empty list in defaults --- src/transformers/models/llama/convert_llama_weights_to_hf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/llama/convert_llama_weights_to_hf.py b/src/transformers/models/llama/convert_llama_weights_to_hf.py index 384daab6b6d..a75ce5245ee 100644 --- a/src/transformers/models/llama/convert_llama_weights_to_hf.py +++ b/src/transformers/models/llama/convert_llama_weights_to_hf.py @@ -449,7 +449,8 @@ def main(): if args.model_size is None and args.num_shards is None: raise ValueError("You have to set at least `num_shards` if you are not giving the `model_size`") if args.special_tokens is None: - args.special_tokens = DEFAULT_LLAMA_SPECIAL_TOKENS[str(args.llama_version)] + # no special tokens by default + args.special_tokens = DEFAULT_LLAMA_SPECIAL_TOKENS.get(str(args.llama_version), []) spm_path = os.path.join(args.input_dir, "tokenizer.model") vocab_size = len(