From 079bfb32fba4f2b39d344ca7af88d79a3ff27c7c Mon Sep 17 00:00:00 2001 From: altsoph Date: Fri, 25 Oct 2019 13:26:37 +0300 Subject: [PATCH] Evaluation fixed. --- examples/run_lm_finetuning.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 4d32385e40b..982d8aa1b79 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -86,7 +86,6 @@ class TextDataset(Dataset): # Note that we are loosing the last truncated example here for the sake of simplicity (no padding) # If your dataset is small, first you should loook for a bigger one :-) and second you # can change this behavior by adding (model specific) padding. - self.examples.append(tokenizer.build_inputs_with_special_tokens(tokenized_text[-block_size:])) # DIRTY! logger.info("Saving features into cached file %s", cached_features_file) with open(cached_features_file, 'wb') as handle: @@ -543,4 +542,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main()