riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 02:31:11 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix typo in a parameter name for open llama model (#23637)

Browse Source 
* Update modeling_open_llama.py

Fix typo in `use_memorry_efficient_attention` parameter name

* Update configuration_open_llama.py

Fix typo in `use_memorry_efficient_attention` parameter name

* Update configuration_open_llama.py

Take care of backwards compatibility ensuring that the previous parameter name is taken into account if used

* Update configuration_open_llama.py

format to adjust the line length

* Update configuration_open_llama.py

proper code formatting using `make fixup`

* Update configuration_open_llama.py

pop the argument not to let it be set later down the line
This commit is contained in:
Alex 2023-05-23 13:57:58 +02:00 committed by GitHub
parent 527ab894e5
commit b687af0b36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/transformers/models/open_llama/configuration_open_llama.py
View File

@ -99,7 +99,7 @@ class OpenLlamaConfig(PretrainedConfig):
bos_token_id=1, bos_token_id=1,
eos_token_id=2, eos_token_id=2,
tie_word_embeddings=False, tie_word_embeddings=False,
use_memorry_efficient_attention=True, use_memory_efficient_attention=True,
hidden_dropout_prob=0.1, hidden_dropout_prob=0.1,
attention_dropout_prob=0.1, attention_dropout_prob=0.1,
use_stable_embedding=True, use_stable_embedding=True,
@ -116,7 +116,9 @@ class OpenLlamaConfig(PretrainedConfig):
self.initializer_range = initializer_range self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache self.use_cache = use_cache
self.use_memorry_efficient_attention = use_memorry_efficient_attention self.use_memory_efficient_attention = kwargs.pop(
"use_memorry_efficient_attention", use_memory_efficient_attention
)
self.hidden_dropout_prob = hidden_dropout_prob self.hidden_dropout_prob = hidden_dropout_prob
self.attention_dropout_prob = attention_dropout_prob self.attention_dropout_prob = attention_dropout_prob
self.use_stable_embedding = use_stable_embedding self.use_stable_embedding = use_stable_embedding

6
src/transformers/models/open_llama/modeling_open_llama.py
View File

@ -40,7 +40,7 @@ try:
except ImportError: except ImportError:
xops = None xops = None
logger.warn( logger.warn(
"Xformers is not installed correctly. If you want to use memorry_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers." "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers."
) )
@ -223,7 +223,7 @@ class OpenLlamaAttention(nn.Module):
past_key_value = (key_states, value_states) if use_cache else None past_key_value = (key_states, value_states) if use_cache else None
if self.config.use_memorry_efficient_attention and xops is not None and self.training: if self.config.use_memory_efficient_attention and xops is not None and self.training:
attn_weights = None attn_weights = None
query_states = query_states.transpose(1, 2) query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2) key_states = key_states.transpose(1, 2)
@ -563,7 +563,7 @@ class OpenLlamaModel(OpenLlamaPreTrainedModel):
if self.embed_layer_norm: if self.embed_layer_norm:
inputs_embeds = self.embed_layer_norm(inputs_embeds) inputs_embeds = self.embed_layer_norm(inputs_embeds)
# embed positions # embed positions
if self.config.use_memorry_efficient_attention and self.training: if self.config.use_memory_efficient_attention and self.training:
attention_mask = None attention_mask = None
elif attention_mask is None: elif attention_mask is None:
attention_mask = torch.ones( attention_mask = torch.ones(