riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-03 12:50:06 +06:00
Code Issues Actions 28 Packages Projects Releases Wiki Activity

Fix: Fixed directory path for utils folder in test_tokenization_utils.py (#32601)

Browse Source 
* Removed un-necessary expressions.

* Fixed directory path for utils folder in test_tokenization_utils.py
This commit is contained in:
Sai-Suraj-27 2024-08-13 21:18:15 +05:30 committed by GitHub
parent cc25757a44
commit c3cd9d807e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
examples/research_projects/jax-projects/big_bird/evaluate.py
View File

@ -94,7 +94,6 @@ def main():
short_validation_dataset = dataset.filter(lambda x: (len(x["question"]) + len(x["context"])) < 4 * 4096)
short_validation_dataset = short_validation_dataset.filter(lambda x: x["category"] != "null")
short_validation_dataset
model_id = "vasudevgupta/flax-bigbird-natural-questions"
model = FlaxBigBirdForNaturalQuestions.from_pretrained(model_id)

3
tests/utils/test_tokenization_utils.py
View File

@ -36,7 +36,7 @@ from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tok
from transformers.tokenization_utils import ExtensionsTrie, Trie
sys.path.append(str(Path(__file__).parent.parent / "utils"))
sys.path.append(str(Path(__file__).parent.parent.parent / "utils"))
from test_module.custom_tokenization import CustomTokenizer # noqa E402
@ -253,7 +253,6 @@ class TrieTest(unittest.TestCase):
trie.add("Hello 友達")
self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}})
trie.add("Hello")
trie.data
self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}})
def test_trie_split(self):