riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

[FSMT] Fix non-shared weights (#26187)

Browse Source 
* Fix non-shared weights

* Add tests

* Edit tied weights keys
This commit is contained in:
Lysandre Debut 2023-09-18 16:58:38 +02:00 committed by GitHub
parent f0a6057fbc
commit 77ed9fa1a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/transformers/models/fsmt/modeling_fsmt.py
View File

@ -1034,7 +1034,7 @@ def _get_shape(t):
FSMT_START_DOCSTRING, FSMT_START_DOCSTRING,
) )
class FSMTModel(PretrainedFSMTModel): class FSMTModel(PretrainedFSMTModel):
_tied_weights_keys = ["decoder.embed_tokens.weight"] _tied_weights_keys = ["decoder.embed_tokens.weight", "decoder.output_projection.weight"]
def __init__(self, config: FSMTConfig): def __init__(self, config: FSMTConfig):
super().__init__(config) super().__init__(config)
@ -1055,6 +1055,10 @@ class FSMTModel(PretrainedFSMTModel):
def get_decoder(self): def get_decoder(self):
return self.decoder return self.decoder
def _tie_weights(self):
self._tie_or_clone_weights(self.decoder.embed_tokens, self.get_input_embeddings())
self._tie_or_clone_weights(self.decoder.output_projection, self.get_input_embeddings())
@add_start_docstrings_to_model_forward(FSMT_INPUTS_DOCSTRING) @add_start_docstrings_to_model_forward(FSMT_INPUTS_DOCSTRING)
@add_code_sample_docstrings( @add_code_sample_docstrings(
checkpoint=_CHECKPOINT_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC,
@ -1171,7 +1175,7 @@ class FSMTModel(PretrainedFSMTModel):
) )
class FSMTForConditionalGeneration(PretrainedFSMTModel): class FSMTForConditionalGeneration(PretrainedFSMTModel):
base_model_prefix = "model" base_model_prefix = "model"
_tied_weights_keys = ["model.decoder.embed_tokens.weight"] _tied_weights_keys = ["decoder.embed_tokens.weight", "decoder.output_projection.weight"]
def __init__(self, config: FSMTConfig): def __init__(self, config: FSMTConfig):
super().__init__(config) super().__init__(config)

17
tests/models/fsmt/test_modeling_fsmt.py
View File

@ -271,6 +271,23 @@ class FSMTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin
input_names=["input_ids", "attention_mask"], input_names=["input_ids", "attention_mask"],
) )
def test_ensure_weights_are_shared(self):
config, inputs_dict = self.model_tester.prepare_config_and_inputs()
model = FSMTForConditionalGeneration(config)
# FSMT shares three weights.
# Not an issue to not have these correctly tied for torch.load, but it is an issue for safetensors.
self.assertEqual(
len(
{
model.get_output_embeddings().weight.data_ptr(),
model.get_input_embeddings().weight.data_ptr(),
model.base_model.decoder.output_projection.weight.data_ptr(),
}
),
1,
)
@unittest.skip("can't be implemented for FSMT due to dual vocab.") @unittest.skip("can't be implemented for FSMT due to dual vocab.")
def test_resize_tokens_embeddings(self): def test_resize_tokens_embeddings(self):
pass pass