riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-30 09:42:22 +06:00
Code Issues Actions 29 Packages Projects Releases Wiki Activity

Empty list in defaults for LLaMA special tokens during weights conversion (#32342)

Browse Source 
empty list in defaults
This commit is contained in:
Viktor Scherbakov 2024-08-01 14:30:10 +02:00 committed by GitHub
parent 2229ebe722
commit db8c7caeb6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/transformers/models/llama/convert_llama_weights_to_hf.py
View File

@ -449,7 +449,8 @@ def main():
if args.model_size is None and args.num_shards is None:
raise ValueError("You have to set at least `num_shards` if you are not giving the `model_size`")
if args.special_tokens is None:
args.special_tokens = DEFAULT_LLAMA_SPECIAL_TOKENS[str(args.llama_version)]
# no special tokens by default
args.special_tokens = DEFAULT_LLAMA_SPECIAL_TOKENS.get(str(args.llama_version), [])
spm_path = os.path.join(args.input_dir, "tokenizer.model")
vocab_size = len(