riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 11:11:05 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Support Python 2

Browse Source
This commit is contained in:
Masatoshi Suzuki 2019-12-05 11:19:02 +09:00 committed by Julien Chaumond
parent a09da4eeb0
commit 6a43dc9d7d
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
transformers/tokenization_bert_japanese.py
View File

@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
import collections
import logging
import os
import six
import unicodedata
from io import open
@ -186,8 +187,13 @@ class MecabTokenizer(object):
never_split = self.never_split + (never_split if never_split is not None else [])
tokens = []
if six.PY2:
mecab_output = self.mecab.parse(text.encode('utf-8')).decode('utf-8')
else:
mecab_output = self.mecab.parse(text)
cursor = 0
for line in self.mecab.parse(text).split('\n'):
for line in mecab_output.split('\n'):
if line == 'EOS':
break