riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

save space when converting hf model to megatron model. (#25950)

Browse Source 
* fix convert megatron model too large

* fix convert megatron model too large
This commit is contained in:
tju_skywalker 2023-09-06 04:47:48 +08:00 committed by GitHub
parent b8def68934
commit 172f42c512
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/models/megatron_gpt2/checkpoint_reshaping_and_interoperability.py
View File

@ -737,7 +737,7 @@ def convert_checkpoint_from_transformers_to_megatron(args):
word_emb_dict = get_element_from_dict_by_path(
output_state_dict[i], "model.language_model.embedding.word_embeddings"
)
word_emb_dict["weight"] = out_word_embed[i]
word_emb_dict["weight"] = out_word_embed[i].clone()
# Transformer layers
print("converting transformer layers")
@ -845,7 +845,7 @@ def convert_checkpoint_from_transformers_to_megatron(args):
for i in range(args.target_tensor_model_parallel_size):
params_dict = get_element_from_dict_by_path(output_state_dict[i], "model.language_model.encoder")
params_dict[layer_name] = (
params[i] if (op_name + "." + weight_or_bias in tensor_parallel_params) else params
params[i].clone() if (op_name + "." + weight_or_bias in tensor_parallel_params) else params
)
if pp_rank == args.target_pipeline_model_parallel_size - 1:
@ -860,7 +860,7 @@ def convert_checkpoint_from_transformers_to_megatron(args):
# add the LM head
for i in range(args.target_tensor_model_parallel_size):
params_dict = get_element_from_dict_by_path(output_state_dict[i], "model.word_embeddings_for_head")
params_dict["weight"] = out_word_embed[i]
params_dict["weight"] = out_word_embed[i].clone()
# saving the state dict as per the tp_rank and pp_rank
for tp_rank in range(args.target_tensor_model_parallel_size):