riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 18:22:34 +06:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Handle the case when title is None (#7941)

Browse Source
This commit is contained in:
Lalit Pagaria 2020-10-23 15:54:45 +02:00 committed by GitHub
parent 023f0f3708
commit 88b3a91e61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
examples/rag/use_own_knowledge_dataset.py
View File

@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict:
"""Split documents into passages""" """Split documents into passages"""
titles, texts = [], [] titles, texts = [], []
for title, text in zip(documents["title"], documents["text"]): for title, text in zip(documents["title"], documents["text"]):
for passage in split_text(text): if text is not None:
titles.append(title) for passage in split_text(text):
texts.append(passage) titles.append(title if title is not None else "")
texts.append(passage)
return {"title": titles, "text": texts} return {"title": titles, "text": texts}