riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

Fix reference of import when called for the second time

Browse Source
This commit is contained in:
Shijie Wu 2019-09-03 18:27:29 -07:00 committed by GitHub
parent 0287d264e9
commit a15562e170
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pytorch_transformers/tokenization_xlm.py
View File

@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try:
if 'pythainlp' not in sys.modules:
from pythainlp.tokenize import word_tokenize as th_word_tokenize
else:
th_word_tokenize = sys.modules['pythainlp'].word_tokenize
except (AttributeError, ImportError) as e:
logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
logger.error("1. pip install pythainlp")
@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try:
if 'jieba' not in sys.modules:
import jieba
else:
jieba = sys.modules['jieba']
except (AttributeError, ImportError) as e:
logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
logger.error("1. pip install jieba")