riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

allowing to load small number of examples

Browse Source
This commit is contained in:
thomwolf 2018-11-05 13:21:24 +01:00
parent a725db4f6c
commit 5622d8320f
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tensorflow_code/run_squad.py
View File

@ -207,7 +207,7 @@ class InputFeatures(object):
self.end_position = end_position
def read_squad_examples(input_file, is_training):
def read_squad_examples(input_file, is_training, max_num=-1):
"""Read a SQuAD json file into a list of SquadExample."""
with tf.gfile.Open(input_file, "r") as reader:
input_data = json.load(reader)["data"]
@ -219,6 +219,8 @@ def read_squad_examples(input_file, is_training):
examples = []
for entry in input_data:
if max_num != -1 and len(examples) > max_num:
break
for paragraph in entry["paragraphs"]:
paragraph_text = paragraph["context"]
doc_tokens = []