From ba9e4eb3541837bf32442cc0deb5a8a7c7961f3b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 6 Feb 2019 00:28:00 +0100 Subject: [PATCH] fix unicode in tokenization tests --- tests/tokenization_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/tokenization_test.py b/tests/tokenization_test.py index d4699adadb1..6a14e05ae81 100644 --- a/tests/tokenization_test.py +++ b/tests/tokenization_test.py @@ -12,16 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function, unicode_literals import os import unittest from io import open -from pytorch_pretrained_bert.tokenization import (BertTokenizer, BasicTokenizer, WordpieceTokenizer, - _is_whitespace, _is_control, _is_punctuation) +from pytorch_pretrained_bert.tokenization import (BasicTokenizer, + BertTokenizer, + WordpieceTokenizer, + _is_control, _is_punctuation, + _is_whitespace) class TokenizationTest(unittest.TestCase):