From c4d4f3ec8cdea133112fa986f222f5d8faedb59d Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Sep 2019 11:22:04 +0200 Subject: [PATCH] Updated DistilBERT test to reflect the sequence encoding --- ...zation_dilbert_test.py => tokenization_distilbert_test.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename pytorch_transformers/tests/{tokenization_dilbert_test.py => tokenization_distilbert_test.py} (92%) diff --git a/pytorch_transformers/tests/tokenization_dilbert_test.py b/pytorch_transformers/tests/tokenization_distilbert_test.py similarity index 92% rename from pytorch_transformers/tests/tokenization_dilbert_test.py rename to pytorch_transformers/tests/tokenization_distilbert_test.py index 42f80609981..30160fc98e9 100644 --- a/pytorch_transformers/tests/tokenization_dilbert_test.py +++ b/pytorch_transformers/tests/tokenization_distilbert_test.py @@ -39,8 +39,8 @@ class DistilBertTokenizationTest(BertTokenizationTest): encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) - assert encoded_sentence == [101] + text + [102] - assert encoded_pair == [101] + text + [102] + text_2 + [102] + assert encoded_sentence == text + assert encoded_pair == text + [102] + text_2 if __name__ == '__main__': unittest.main()