Handle the case when title is None (#7941)

This commit is contained in:
Lalit Pagaria 2020-10-23 15:54:45 +02:00 committed by GitHub
parent 023f0f3708
commit 88b3a91e61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict:
"""Split documents into passages"""
titles, texts = [], []
for title, text in zip(documents["title"], documents["text"]):
for passage in split_text(text):
titles.append(title)
texts.append(passage)
if text is not None:
for passage in split_text(text):
titles.append(title if title is not None else "")
texts.append(passage)
return {"title": titles, "text": texts}