riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

add everything (#11651)

Browse Source
This commit is contained in:
Vasudev Gupta 2021-05-13 16:21:30 +05:30 committed by GitHub
parent 57b6a80de8
commit 6ee1a4fd3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/transformers/models/big_bird/modeling_big_bird.py
View File

@ -647,13 +647,13 @@ class BigBirdBlockSparseAttention(nn.Module):
[ [
to_mask[:, :, :, : 3 * to_block_size], to_mask[:, :, :, : 3 * to_block_size],
to_mask[:, :, :, -to_block_size:], to_mask[:, :, :, -to_block_size:],
first_context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]), to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
], ],
dim=3, dim=3,
) )
second_rand_pad = torch.cat( second_rand_pad = torch.cat(
[ [
first_context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]), rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, 0], rand_mask[:, :, 0],
], ],
dim=3, dim=3,
@ -781,13 +781,13 @@ class BigBirdBlockSparseAttention(nn.Module):
[ [
to_mask[:, :, :, :to_block_size], to_mask[:, :, :, :to_block_size],
to_mask[:, :, :, -3 * to_block_size :], to_mask[:, :, :, -3 * to_block_size :],
context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]), to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
], ],
dim=3, dim=3,
) )
second_last_rand_pad = torch.cat( second_last_rand_pad = torch.cat(
[ [
context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]), rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, -1], rand_mask[:, :, -1],
], ],
dim=3, dim=3,

8
src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py
View File

@ -475,13 +475,13 @@ class BigBirdPegasusBlockSparseAttention(nn.Module):
[ [
to_mask[:, :, :, : 3 * to_block_size], to_mask[:, :, :, : 3 * to_block_size],
to_mask[:, :, :, -to_block_size:], to_mask[:, :, :, -to_block_size:],
first_context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]), to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
], ],
dim=3, dim=3,
) )
second_rand_pad = torch.cat( second_rand_pad = torch.cat(
[ [
first_context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]), rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, 0], rand_mask[:, :, 0],
], ],
dim=3, dim=3,
@ -609,13 +609,13 @@ class BigBirdPegasusBlockSparseAttention(nn.Module):
[ [
to_mask[:, :, :, :to_block_size], to_mask[:, :, :, :to_block_size],
to_mask[:, :, :, -3 * to_block_size :], to_mask[:, :, :, -3 * to_block_size :],
context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]), to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
], ],
dim=3, dim=3,
) )
second_last_rand_pad = torch.cat( second_last_rand_pad = torch.cat(
[ [
context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]), rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, -1], rand_mask[:, :, -1],
], ],
dim=3, dim=3,