riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-15 10:38:23 +06:00
Code Issues Actions 16 Packages Projects Releases Wiki Activity

fix unicode in tokenization tests

Browse Source
This commit is contained in:
thomwolf 2019-02-06 00:28:00 +01:00
parent 34bdb7f9cb
commit ba9e4eb354
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tests/tokenization_test.py
View File

@ -12,16 +12,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from __future__ import absolute_import from __future__ import absolute_import, division, print_function, unicode_literals
from __future__ import division
from __future__ import print_function
import os import os
import unittest import unittest
from io import open from io import open
from pytorch_pretrained_bert.tokenization import (BertTokenizer, BasicTokenizer, WordpieceTokenizer, from pytorch_pretrained_bert.tokenization import (BasicTokenizer,
_is_whitespace, _is_control, _is_punctuation) BertTokenizer,
WordpieceTokenizer,
_is_control, _is_punctuation,
_is_whitespace)
class TokenizationTest(unittest.TestCase): class TokenizationTest(unittest.TestCase):