From 83446a88d902661fab12bf8c37a1aa2845cdca5f Mon Sep 17 00:00:00 2001 From: Bram Vanroy Date: Wed, 29 Jan 2020 09:34:37 +0100 Subject: [PATCH] Use _pad_token of pad_token_id Requesting pad_token_id would cause an error message when it is None. Use private _pad_token instead. --- src/transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py index cbb4a821030..469181325aa 100644 --- a/src/transformers/tokenization_utils.py +++ b/src/transformers/tokenization_utils.py @@ -999,7 +999,7 @@ class PreTrainedTokenizer(object): padded_value = value # verify that the tokenizer has a pad_token_id - if key != "input_len" and self.pad_token_id is not None: + if key != "input_len" and self._pad_token is not None: # Padding handle padded_value = [ v + [self.pad_token_id if key == "input_ids" else 1] * (max_seq_len - len(v))