diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 7c3c6ccac8b..cedfad084cc 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -3334,9 +3334,18 @@ class Expectations(UserDict[PackedDeviceProperties, Any]): def find_expectation(self, properties: DeviceProperties = (None, None, None)) -> Any: """ - Find best matching expectation based on provided device properties. + Find best matching expectation based on provided device properties. We score each expectation, and to + distinguish between expectations with the same score, we use the major and minor version numbers, prioritizing + most recent versions. """ - (result_key, result) = max(self.unpacked(), key=lambda x: Expectations.score(properties, x[0])) + (result_key, result) = max( + self.unpacked(), + key=lambda x: ( + Expectations.score(properties, x[0]), # x[0] is a device properties tuple (device_type, major, minor) + x[0][1] if x[0][1] is not None else -1, # This key is the major version, -1 if major is None + x[0][2] if x[0][2] is not None else -1, # This key is the minor version, -1 if minor is None + ), + ) if Expectations.score(properties, result_key) == 0: raise ValueError(f"No matching expectation found for {properties}") diff --git a/tests/models/gemma3/test_modeling_gemma3.py b/tests/models/gemma3/test_modeling_gemma3.py index b0b25579b82..2c4643cf6c5 100644 --- a/tests/models/gemma3/test_modeling_gemma3.py +++ b/tests/models/gemma3/test_modeling_gemma3.py @@ -409,6 +409,7 @@ class Gemma3IntegrationTest(unittest.TestCase): ("xpu", 3): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water in the background. It looks like a lovely,'], ("cuda", 7): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water in the background. It looks like a lovely,'], ("cuda", 8): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and'], + ("rocm", (9, 5)): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant coastline in the background. It looks'], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -461,6 +462,11 @@ class Gemma3IntegrationTest(unittest.TestCase): 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. It looks like a very sunny and', 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes:\n\n* **Image 1** shows a cow standing on a beach.', ], + ("rocm", (9, 5)): + [ + 'user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nCertainly! \n\nThe image shows a brown and white cow standing on a sandy beach next to a turquoise ocean. There are some clouds in the blue', + 'user\nYou are a helpful assistant.\n\n\n\n\n\n\n\n\n\nAre these images identical?\nmodel\nNo, these images are not identical. They depict very different scenes. \n\n* **Image 1** shows a cow standing on a beach', + ], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -558,6 +564,10 @@ class Gemma3IntegrationTest(unittest.TestCase): 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown cow standing on a sandy beach next to a turquoise ocean. There are clouds in the blue sky above.', 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nAre these images identical?\nmodel\nNo, the images are not identical. \n\nThe first image shows a cow on a beach, while the second image shows a street scene with a', ], + ("rocm", (9, 5)) : [ + 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown cow standing on a sandy beach next to a turquoise ocean. There are clouds in the blue sky above.', + 'user\nYou are a helpful assistant.\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nHere is the original image \n\n\n\n and here are some crops to help you see better \n\n\n\n \n\n\n\nAre these images identical?\nmodel\nNo, the images are not identical. \n\nThe first image shows a cow on a beach, while the second image shows a street scene with a', + ], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -618,6 +628,7 @@ class Gemma3IntegrationTest(unittest.TestCase): ("xpu", 3): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a river deep,\nWith patterns hidden, secrets sleep.\nA neural net, a watchful eye,\nLearning'], ("cuda", 7): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a silent stream,\nInto the neural net, a waking dream.\nAlgorithms hum, a coded grace,\n'], ("cuda", 8): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a silent stream,\nInto the neural net, a waking dream.\nAlgorithms hum, a coded grace,\n'], + ("rocm", (9, 5)): ['Write a poem about Machine Learning.\n\n---\n\nThe data flows, a river deep,\nWith patterns hidden, secrets sleep.\nA neural net, a watchful eye,\nLearning'], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() @@ -650,6 +661,7 @@ class Gemma3IntegrationTest(unittest.TestCase): ("xpu", 3): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant island in the background. It looks like a sunny day'], ("cuda", 7): [], ("cuda", 8): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown and white cow standing on a sandy beach with turquoise water and a distant island in the background. It looks like a sunny day'], + ("rocm", (9, 5)): ['user\nYou are a helpful assistant.\n\n\n\n\n\nWhat is shown in this image?\nmodel\nThe image shows a brown and white cow standing on a sandy beach with a turquoise ocean and a distant island in the background. It looks like a sunny'], } ) # fmt: skip EXPECTED_TEXT = EXPECTED_TEXTS.get_expectation() diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 874323b1e9a..c482cd43d4d 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3801,9 +3801,9 @@ class ModelTesterMixin: self.skipTest( "PaliGemma-like models currently (transformers==4.41.0) requires an attention_mask input" ) - if config.model_type in ["modernbert"]: + if config.model_type in ["modernbert", "gemma3"]: self.skipTest( - reason="ModernBert currently (transformers==4.52.0) automatically adds an attention_mask input" + reason=f"{config.model_type} currently (transformers==4.52.0) automatically adds an attention_mask input" ) if config.model_type in ["idefics", "idefics2", "idefics3"]: self.skipTest(reason="Idefics currently (transformers==4.39.1) requires an image_attention_mask input")