riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-03 03:31:05 +06:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Remove 50k limits bug

Browse Source
This commit is contained in:
peterandluc 2020-04-23 17:10:57 +02:00 committed by Julien Chaumond
parent 6af5a54c28
commit 8e093e5981
1 changed files with 0 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/data/datasets/language_modeling.py
View File

@ -90,7 +90,6 @@ class LineByLineTextDataset(Dataset):
with open(file_path, encoding="utf-8") as f:
lines = [line for line in f.read().splitlines() if (len(line) > 0 and not line.isspace())]
lines = lines[:50_000]
batch_encoding = tokenizer.batch_encode_plus(lines, add_special_tokens=True, max_length=block_size)
self.examples = batch_encoding["input_ids"]