riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 03:01:07 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix #3954 - GPT2 is not traceable (#3955)

Browse Source 
* Update sqrt computation so it can survive a torch.jit.trace

* Update modeling_gpt2.py

Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>
This commit is contained in:
jazzcook15 2020-04-28 12:18:56 -07:00 committed by GitHub
parent 9a0a8c1c6f
commit c7d06b79ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/transformers/modeling_gpt2.py
View File

@ -17,7 +17,6 @@
import logging import logging
import math
import os import os
import torch import torch
@ -143,7 +142,7 @@ class Attention(nn.Module):
def _attn(self, q, k, v, attention_mask=None, head_mask=None): def _attn(self, q, k, v, attention_mask=None, head_mask=None):
w = torch.matmul(q, k) w = torch.matmul(q, k)
if self.scale: if self.scale:
w = w / math.sqrt(v.size(-1)) w = w / (v.size(-1) ** 0.5)
nd, ns = w.size(-2), w.size(-1) nd, ns = w.size(-2), w.size(-1)
mask = self.bias[:, :, ns - nd : ns, :ns] mask = self.bias[:, :, ns - nd : ns, :ns]
w = torch.where(mask, w, self.masked_bias) w = torch.where(mask, w, self.masked_bias)