From 07f4cd73f6d13e43b69a6e34a2a756a80fc7f70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 31 Oct 2019 09:48:27 +0100 Subject: [PATCH] update function to add special tokens Since I started my PR the `add_special_token_single_sequence` function has been deprecated for another; I replaced it with the new function. --- examples/utils_summarization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py index 327ca8cc3eb..087c88bd4e2 100644 --- a/examples/utils_summarization.py +++ b/examples/utils_summarization.py @@ -139,11 +139,11 @@ def encode_for_summarization(story_lines, summary_lines, tokenizer): sentences. """ story_lines_token_ids = [ - tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) + tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line)) for line in story_lines ] summary_lines_token_ids = [ - tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) + tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line)) for line in summary_lines ]