From 31e67dd19f1b3fe2bc9a13f86d814f3f7bba48e4 Mon Sep 17 00:00:00 2001 From: Jangwon Park Date: Tue, 12 May 2020 21:32:44 +0900 Subject: [PATCH] Remove hard-coded pad token id in distilbert and albert (#3965) --- src/transformers/modeling_albert.py | 2 +- src/transformers/modeling_distilbert.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_albert.py b/src/transformers/modeling_albert.py index f6cd770cdef..1dd1bcf5537 100644 --- a/src/transformers/modeling_albert.py +++ b/src/transformers/modeling_albert.py @@ -175,7 +175,7 @@ class AlbertEmbeddings(BertEmbeddings): def __init__(self, config): super().__init__(config) - self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=config.pad_token_id) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) diff --git a/src/transformers/modeling_distilbert.py b/src/transformers/modeling_distilbert.py index 5ab6f3c2d0a..86470fcd6bf 100644 --- a/src/transformers/modeling_distilbert.py +++ b/src/transformers/modeling_distilbert.py @@ -61,7 +61,7 @@ def create_sinusoidal_embeddings(n_pos, dim, out): class Embeddings(nn.Module): def __init__(self, config): super().__init__() - self.word_embeddings = nn.Embedding(config.vocab_size, config.dim, padding_idx=0) + self.word_embeddings = nn.Embedding(config.vocab_size, config.dim, padding_idx=config.pad_token_id) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.dim) if config.sinusoidal_pos_embds: create_sinusoidal_embeddings(