riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 33 Packages Projects Releases Wiki Activity

fix: Fixed CodeGenTokenizationTest::test_truncation failing test (#32850)

Browse Source 
* Fixed failing CodeGenTokenizationTest::test_truncation.

* [run_slow] Codegen

* [run_slow] codegen
This commit is contained in:
Sai-Suraj-27 2024-08-27 12:50:59 +05:30 committed by GitHub
parent 9578c2597e
commit 3bf6dd8aa1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/models/codegen/test_tokenization_codegen.py
View File

@ -254,12 +254,12 @@ class CodeGenTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
tokenizer = CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono")
text = "\nif len_a > len_b:\n result = a\nelse:\n result = b\n\n\n\n#"
expected_trucated_text = "\nif len_a > len_b: result = a\nelse: result = b"
expected_truncated_text = "\nif len_a > len_b:\n result = a\nelse:\n result = b"
input_ids = tokenizer.encode(text)
truncation_pattern = ["^#", re.escape("<|endoftext|>"), "^'''", '^"""', "\n\n\n"]
decoded_text = tokenizer.decode(input_ids, truncate_before_pattern=truncation_pattern)
self.assertEqual(decoded_text, expected_trucated_text)
self.assertEqual(decoded_text, expected_truncated_text)
# TODO @ArthurZ outputs of the fast tokenizer are different in this case, un-related to the PR
# tokenizer has no padding token