riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 03:01:07 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix typo of Block. (#28727)

Browse Source
This commit is contained in:
xkszltl 2024-01-29 07:25:00 -08:00 committed by GitHub
parent 9e8f35fa28
commit e694e985d7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/transformers/models/mixtral/modeling_mixtral.py
View File

@ -787,7 +787,7 @@ MIXTRAL_ATTENTION_CLASSES = {
} }
class MixtralBLockSparseTop2MLP(nn.Module): class MixtralBlockSparseTop2MLP(nn.Module):
def __init__(self, config: MixtralConfig): def __init__(self, config: MixtralConfig):
super().__init__() super().__init__()
self.ffn_dim = config.intermediate_size self.ffn_dim = config.intermediate_size
@ -805,6 +805,14 @@ class MixtralBLockSparseTop2MLP(nn.Module):
return current_hidden_states return current_hidden_states
class MixtralBLockSparseTop2MLP(MixtralBlockSparseTop2MLP):
def __init__(self, *args, **kwargs):
logger.warning_once(
"MixtralBLockSparseTop2MLP is deprecated by MixtralBlockSparseTop2MLP and will be removed in v4.40."
)
super().__init__(*args, **kwargs)
class MixtralSparseMoeBlock(nn.Module): class MixtralSparseMoeBlock(nn.Module):
""" """
This implementation is This implementation is
@ -827,7 +835,7 @@ class MixtralSparseMoeBlock(nn.Module):
# gating # gating
self.gate = nn.Linear(self.hidden_dim, self.num_experts, bias=False) self.gate = nn.Linear(self.hidden_dim, self.num_experts, bias=False)
self.experts = nn.ModuleList([MixtralBLockSparseTop2MLP(config) for _ in range(self.num_experts)]) self.experts = nn.ModuleList([MixtralBlockSparseTop2MLP(config) for _ in range(self.num_experts)])
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
""" """ """ """