From 28e64ad5a4b01a1b7de092694e3a321edf7021bd Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 14:12:54 +0100 Subject: [PATCH] Raise an exception if the pipeline allocator can't determine the tokenizer from the model. --- transformers/pipelines.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index b0b5848c019..853735a2567 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -370,11 +370,12 @@ def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokeni Utility factory method to build pipeline. """ # Try to infer tokenizer from model name (if provided as str) - if tokenizer is None and isinstance(model, str): - tokenizer = model - else: - # Impossible to guest what is the right tokenizer here - raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') + if not isinstance(tokenizer, PreTrainedTokenizer): + if not isinstance(model, str): + # Impossible to guest what is the right tokenizer here + raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') + else: + tokenizer = model tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer)