riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 33 Packages Projects Releases Wiki Activity

add everything (#11651)

Browse Source
This commit is contained in:
Vasudev Gupta 2021-05-13 16:21:30 +05:30 committed by GitHub
parent 57b6a80de8
commit 6ee1a4fd3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/transformers/models/big_bird/modeling_big_bird.py
View File

@ -647,13 +647,13 @@ class BigBirdBlockSparseAttention(nn.Module):
[
to_mask[:, :, :, : 3 * to_block_size],
to_mask[:, :, :, -to_block_size:],
first_context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
],
dim=3,
)
second_rand_pad = torch.cat(
[
first_context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, 0],
],
dim=3,
@ -781,13 +781,13 @@ class BigBirdBlockSparseAttention(nn.Module):
[
to_mask[:, :, :, :to_block_size],
to_mask[:, :, :, -3 * to_block_size :],
context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
],
dim=3,
)
second_last_rand_pad = torch.cat(
[
context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, -1],
],
dim=3,

8
src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py
View File

@ -475,13 +475,13 @@ class BigBirdPegasusBlockSparseAttention(nn.Module):
[
to_mask[:, :, :, : 3 * to_block_size],
to_mask[:, :, :, -to_block_size:],
first_context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
],
dim=3,
)
second_rand_pad = torch.cat(
[
first_context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, 0],
],
dim=3,
@ -609,13 +609,13 @@ class BigBirdPegasusBlockSparseAttention(nn.Module):
[
to_mask[:, :, :, :to_block_size],
to_mask[:, :, :, -3 * to_block_size :],
context_layer.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
to_mask.new_ones([bsz, 1, 1, n_rand_blocks * to_block_size]),
],
dim=3,
)
second_last_rand_pad = torch.cat(
[
context_layer.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask.new_ones([bsz, n_heads, from_block_size, 4 * to_block_size]),
rand_mask[:, :, -1],
],
dim=3,