riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 18:22:34 +06:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

tests pass on python 2 and 3

Browse Source
This commit is contained in:
thomwolf 2019-02-11 10:47:52 +01:00
parent 2071a9b86e
commit 0a9860daa7
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tests/tokenization_openai_test.py
View File

@ -17,7 +17,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera
import os import os
import unittest import unittest
import json import json
from io import open
from pytorch_pretrained_bert.tokenization_openai import OpenAIGPTTokenizer from pytorch_pretrained_bert.tokenization_openai import OpenAIGPTTokenizer
@ -32,10 +31,10 @@ class OpenAIGPTTokenizationTest(unittest.TestCase):
"low</w>", "lowest</w>", "newer</w>", "wider</w>"] "low</w>", "lowest</w>", "newer</w>", "wider</w>"]
vocab_tokens = dict(zip(vocab, range(len(vocab)))) vocab_tokens = dict(zip(vocab, range(len(vocab))))
merges = ["#version: 0.2", "l o", "lo w", "e r</w>", ""] merges = ["#version: 0.2", "l o", "lo w", "e r</w>", ""]
with open("/tmp/openai_tokenizer_vocab_test.json", "wb") as fp: with open("/tmp/openai_tokenizer_vocab_test.json", "w") as fp:
json.dump(vocab_tokens, fp) json.dump(vocab_tokens, fp)
vocab_file = fp.name vocab_file = fp.name
with open("/tmp/openai_tokenizer_merges_test.txt", "w", encoding='utf-8') as fp: with open("/tmp/openai_tokenizer_merges_test.txt", "w") as fp:
fp.write("\n".join(merges)) fp.write("\n".join(merges))
merges_file = fp.name merges_file = fp.name