riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-30 17:52:35 +06:00
Code Issues Actions 30 Packages Projects Releases Wiki Activity

Evaluation fixed.

Browse Source
This commit is contained in:
altsoph 2019-10-25 13:26:37 +03:00 committed by Lysandre Debut
parent 438f2730a0
commit 079bfb32fb
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/run_lm_finetuning.py
View File

@ -86,7 +86,6 @@ class TextDataset(Dataset):
# Note that we are loosing the last truncated example here for the sake of simplicity (no padding)
# If your dataset is small, first you should loook for a bigger one :-) and second you
# can change this behavior by adding (model specific) padding.
self.examples.append(tokenizer.build_inputs_with_special_tokens(tokenized_text[-block_size:])) # DIRTY!
logger.info("Saving features into cached file %s", cached_features_file)
with open(cached_features_file, 'wb') as handle:
@ -543,4 +542,4 @@ def main():
if __name__ == "__main__":
main()
main()