riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Update: ignore padding support for TransfoXL training when n_clusters==0 (#22457)

Browse Source 
* Update: ignore padding support for TransfoXL training when n_clusters==0

* Update: transformer XL always pad

* Update: drop doc
This commit is contained in:
Stefan Heng 2023-03-29 14:36:39 -04:00 committed by GitHub
parent 2194943a34
commit cd73b9a8c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py
View File

@ -86,7 +86,7 @@ class ProjectedAdaptiveLogSoftmax(nn.Module):
""" """
Params: Params:
hidden :: [len*bsz x d_proj] hidden :: [len*bsz x d_proj]
labels :: [len*bsz labels :: [len*bsz]
Return: Return:
if labels is None: out :: [len*bsz x n_tokens] log probabilities of tokens over the vocabulary else: out :: if labels is None: out :: [len*bsz x n_tokens] log probabilities of tokens over the vocabulary else: out ::
@ -109,7 +109,11 @@ class ProjectedAdaptiveLogSoftmax(nn.Module):
if self.n_clusters == 0: if self.n_clusters == 0:
logit = self._compute_logit(hidden, self.out_layers[0].weight, self.out_layers[0].bias, self.out_projs[0]) logit = self._compute_logit(hidden, self.out_layers[0].weight, self.out_layers[0].bias, self.out_projs[0])
if labels is not None: if labels is not None:
out = -nn.functional.log_softmax(logit, dim=-1).gather(1, labels.unsqueeze(1)).squeeze(1) mask = labels != -100
out = torch.zeros_like(labels, dtype=hidden.dtype, device=hidden.device)
out[mask] = (
-nn.functional.log_softmax(logit, dim=-1)[mask].gather(1, labels[mask].unsqueeze(1)).squeeze(1)
)
else: else:
out = nn.functional.log_softmax(logit, dim=-1) out = nn.functional.log_softmax(logit, dim=-1)
else: else: