From 839893c86bf372ee35b2c8dd750d3cdc21a995f5 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 27 Jun 2025 15:44:10 +0200 Subject: [PATCH] fix `mistral3` tests (#38989) * fix * fix * fix * fix * fix --------- Co-authored-by: ydshieh --- .../models/mistral3/test_modeling_mistral3.py | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/models/mistral3/test_modeling_mistral3.py b/tests/models/mistral3/test_modeling_mistral3.py index dd1c940938c..595044a6fd3 100644 --- a/tests/models/mistral3/test_modeling_mistral3.py +++ b/tests/models/mistral3/test_modeling_mistral3.py @@ -307,6 +307,7 @@ class Mistral3IntegrationTest(unittest.TestCase): @require_read_token def test_mistral3_integration_generate_text_only(self): processor = AutoProcessor.from_pretrained(self.model_checkpoint) + processor.chat_template = processor.chat_template.replace('strftime_now("%Y-%m-%d")', '"2025-06-20"') messages = [ { @@ -329,7 +330,6 @@ class Mistral3IntegrationTest(unittest.TestCase): expected_outputs = Expectations( { ("xpu", 3): "Sure, here is a haiku for you:\n\nWhispers of the breeze,\nCherry blossoms softly fall,\nSpring's gentle embrace.", - ("cuda", 7): "Sure, here is a haiku for you:\n\nWhispers of the breeze,\nCherry blossoms softly fall,\nSpring's gentle embrace.", ("cuda", 8): "Sure, here is a haiku for you:\n\nWhispers of the breeze,\nCherry blossoms softly fall,\nSpring's gentle embrace.", } ) # fmt: skip @@ -339,6 +339,7 @@ class Mistral3IntegrationTest(unittest.TestCase): @require_read_token def test_mistral3_integration_generate(self): processor = AutoProcessor.from_pretrained(self.model_checkpoint) + processor.chat_template = processor.chat_template.replace('strftime_now("%Y-%m-%d")', '"2025-06-20"') messages = [ { "role": "user", @@ -361,18 +362,17 @@ class Mistral3IntegrationTest(unittest.TestCase): expected_outputs = Expectations( { ("xpu", 3): "The image features two cats resting on a pink blanket. The cat on the left is a kitten", - ("cuda", 7): "The image features two cats resting on a pink blanket. The cat on the left is a kitten", - ("cuda", 8): "The image features two cats resting on a pink blanket. The cat on the left is a small kit", + ("cuda", 8): 'The image features two cats lying on a pink surface, which appears to be a couch or a bed', } ) # fmt: skip expected_output = expected_outputs.get_expectation() - self.assertEqual(decoded_output, expected_output) @require_read_token @require_deterministic_for_xpu def test_mistral3_integration_batched_generate(self): processor = AutoProcessor.from_pretrained(self.model_checkpoint) + processor.chat_template = processor.chat_template.replace('strftime_now("%Y-%m-%d")', '"2025-06-20"') messages = [ [ { @@ -408,8 +408,7 @@ class Mistral3IntegrationTest(unittest.TestCase): expected_outputs = Expectations( { ("xpu", 3): "Calm lake's mirror gleams,\nWhispering pines stand in silence,\nPath to peace begins.", - ("cuda", 7): "Calm waters reflect\nWhispering pines stand in silence\nPath to peace begins", - ("cuda", 8): "Calm waters reflect\nWhispering pines stand in silence\nPath to peace begins", + ("cuda", 8): "Wooden path to calm,\nReflections whisper secrets,\nNature's peace unfolds.", } ) # fmt: skip expected_output = expected_outputs.get_expectation() @@ -424,8 +423,7 @@ class Mistral3IntegrationTest(unittest.TestCase): expected_outputs = Expectations( { ("xpu", 3): "The image depicts a vibrant urban scene in what appears to be Chinatown. The focal point is a traditional Chinese archway", - ("cuda", 7): 'The image depicts a vibrant street scene in Chinatown, likely in a major city. The focal point is a traditional Chinese', - ("cuda", 8): 'The image depicts a vibrant street scene in what appears to be Chinatown in a major city. The focal point is a', + ("cuda", 8): 'The image depicts a street scene in what appears to be a Chinatown district. The focal point is a traditional Chinese arch', } ) # fmt: skip expected_output = expected_outputs.get_expectation() @@ -439,6 +437,7 @@ class Mistral3IntegrationTest(unittest.TestCase): @require_deterministic_for_xpu def test_mistral3_integration_batched_generate_multi_image(self): processor = AutoProcessor.from_pretrained(self.model_checkpoint) + processor.chat_template = processor.chat_template.replace('strftime_now("%Y-%m-%d")', '"2025-06-20"') # Prepare inputs messages = [ @@ -482,9 +481,7 @@ class Mistral3IntegrationTest(unittest.TestCase): decoded_output = processor.decode(gen_tokens[0], skip_special_tokens=True) expected_outputs = Expectations( { - ("xpu", 3): "Still lake reflects skies,\nWooden path to nature's heart,\nSilence speaks volumes.", - ("cuda", 7): "Calm waters reflect\nWhispering pines stand in silence\nPath to peace begins", - ("cuda", 8): "Calm waters reflect\nWhispering pines stand in silence\nPath to peace begins", + ("cuda", 8): 'Calm waters reflect\nWooden path to distant shore\nSilence in the scene', } ) # fmt: skip expected_output = expected_outputs.get_expectation() @@ -499,12 +496,10 @@ class Mistral3IntegrationTest(unittest.TestCase): expected_outputs = Expectations( { ("xpu", 3): "Certainly! The images depict two iconic landmarks:\n\n1. The first image shows the Statue of Liberty in New York City.", - ("cuda", 7): "Certainly! The images depict the following landmarks:\n\n1. The first image shows the Statue of Liberty and the New York City", - ("cuda", 8): "Certainly! The images depict the following landmarks:\n\n1. The first image shows the Statue of Liberty and the New York City", + ("cuda", 8): 'Certainly! The images depict two famous landmarks in the United States:\n\n1. The first image shows the Statue of Liberty,', } ) # fmt: skip expected_output = expected_outputs.get_expectation() - self.assertEqual( decoded_output, expected_output,