riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-03 03:31:05 +06:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Interface with TFDS

Browse Source
This commit is contained in:
Lysandre 2019-11-28 17:17:20 -05:00
parent bd41e8292a
commit f671997ef7
1 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
transformers/data/processors/squad.py
View File

@ -246,16 +246,24 @@ class SquadProcessor(DataProcessor):
dev_file = None
def get_example_from_tensor_dict(self, tensor_dict):
"""See base class."""
return NewSquadExample(
tensor_dict['id'].numpy(),
tensor_dict['id'].numpy().decode("utf-8"),
tensor_dict['question'].numpy().decode('utf-8'),
tensor_dict['context'].numpy().decode('utf-8'),
tensor_dict['answers']['text'].numpy().decode('utf-8'),
tensor_dict['answers']['answers_start'].numpy().decode('utf-8'),
tensor_dict['answers']['text'][0].numpy().decode('utf-8'),
tensor_dict['answers']['answer_start'][0].numpy(),
tensor_dict['title'].numpy().decode('utf-8')
)
def get_examples_from_dataset(self, dataset):
"""See base class."""
examples = []
for tensor_dict in tqdm(dataset):
examples.append(self.get_example_from_tensor_dict(tensor_dict))
return examples
def get_train_examples(self, data_dir, only_first=None):
"""See base class."""
if self.train_file is None: