riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-23 22:38:58 +06:00
Code Issues Actions 31 Packages Projects Releases Wiki Activity

Fix InternVL integration test (#38612)
Some checks failed
Self-hosted runner (benchmark) / Benchmark (aws-g5-4xlarge-cache) (push) Has been cancelled
Details
Build documentation / build (push) Has been cancelled
Details
Slow tests on important models (on Push - A10) / Get all modified files (push) Has been cancelled
Details
Self-hosted runner (push-caller) / Check if setup was changed (push) Has been cancelled
Details
Secret Leaks / trufflehog (push) Has been cancelled
Details
Update Transformers metadata / build_and_package (push) Has been cancelled
Details
Slow tests on important models (on Push - A10) / Slow & FA2 tests (push) Has been cancelled
Details
Self-hosted runner (push-caller) / build-docker-containers (push) Has been cancelled
Details
Self-hosted runner (push-caller) / Trigger Push CI (push) Has been cancelled
Details

Browse Source 
* fix

* fix

* fix OOM

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar 2025-06-07 08:30:47 +02:00 committed by GitHub
parent 3fb7e7bc01
commit ebeec13609
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 29 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
tests/models/internvl/test_modeling_internvl.py
View File

@ -284,6 +284,7 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
def setUp(self):
self.small_model_checkpoint = "OpenGVLab/InternVL3-1B-hf"
self.medium_model_checkpoint = "OpenGVLab/InternVL3-2B-hf"
cleanup(torch_device, gc_collect=True)
def tearDown(self):
cleanup(torch_device, gc_collect=True)
@ -329,7 +330,8 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
expected_logits_all = Expectations(
{
("xpu", 3): torch.tensor([11.7500, 14.7500, 14.1250, 10.5625, 6.7812], dtype=torch.bfloat16),
("cuda", 7): torch.tensor([11.9375, 14.8750, 14.0625, 10.7500, 6.9062], dtype=torch.bfloat16),
("cuda", 7): torch.tensor([11.9375, 14.7500, 14.4375, 10.8125, 7.0938], dtype=torch.bfloat16),
("cuda", 8): torch.tensor([11.8750, 14.8125, 14.3125, 10.8125, 6.9375], dtype=torch.bfloat16),
}
) # fmt: skip
expected_logits = expected_logits_all.get_expectation()
@ -358,7 +360,8 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light.",
("cuda", 7): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light begins.",
("cuda", 7): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light.",
("cuda", 8): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light begins.",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -425,7 +428,7 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): 'user\n\nDescribe this image\nassistant\nThe image shows a street scene with a traditional Chinese archway, known as a "Chinese Gate" or "Chinese Gate"',
("cuda", 7): 'user\n\nDescribe this image\nassistant\nThe image shows a street scene with a traditional Chinese archway, known as a "Chinese Gate" or "Chinese Gate of',
("cuda", 7): 'user\n\nDescribe this image\nassistant\nThe image shows a street scene with a traditional Chinese archway, known as a "Chinese Gate" or "Chinese Arch,"',
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -483,7 +486,7 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "user\n\nWhat are the differences between these two images?\nassistant\nThe images show the Statue of Liberty and the Golden Gate Bridge from different angles. Here are the differences:\n\n1. **Foreground",
("cuda", 7): "user\n\nWhat are the differences between these two images?\nassistant\nThe images show the Statue of Liberty and the Golden Gate Bridge from different angles. Here are the differences:\n\n1. **Angle",
("cuda", 7): "user\n\nWhat are the differences between these two images?\nassistant\nThe images show the Statue of Liberty and the Golden Gate Bridge from different angles. Here are the differences:\n\n1. **Foreground",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -606,7 +609,7 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "user\n\n\nWhat are the differences between these two images?\nassistant\nThe images depict two distinct scenes:\n\n1. **Left Image:**\n - The Statue of Liberty is prominently featured on an",
("cuda", 7): "user\n\n\nWhat are the differences between these two images?\nassistant\nThe images depict two distinct scenes:\n\n1. **Left Image**: This shows the Statue of Liberty on Liberty Island, with the",
("cuda", 7): "user\n\n\nWhat are the differences between these two images?\nassistant\nThe images depict two distinct scenes:\n\n1. **Left Image:**\n - The Statue of Liberty is prominently featured on an",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -620,7 +623,7 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot.",
("cuda", 7): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nA forehand shot",
("cuda", 7): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot.",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -646,6 +649,7 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
def setUp(self):
self.small_model_checkpoint = "OpenGVLab/InternVL2_5-2B-MPO-hf"
self.medium_model_checkpoint = "OpenGVLab/InternVL2_5-8B-MPO-hf"
cleanup(torch_device, gc_collect=True)
def tearDown(self):
cleanup(torch_device, gc_collect=True)
@ -688,13 +692,22 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
output = model(**inputs)
actual_logits = output.logits[0, -1, :5].cpu()
expected_logits = torch.tensor([-9.8750, -0.4258, 1.4844, -10.3125, -10.3125], dtype=torch.bfloat16)
expected_logits_all = Expectations(
{
("xpu", 3): torch.tensor([-9.8750, -0.5703, 1.4297, -10.3125, -10.3125], dtype=torch.bfloat16),
("cuda", 7): torch.tensor([-9.8750, -0.5703, 1.4297, -10.3125, -10.3125], dtype=torch.bfloat16),
("cuda", 8): torch.tensor([-9.8750, -0.5117, 1.4297, -10.3750, -10.3750], dtype=torch.bfloat16),
}
) # fmt: skip
expected_logits = torch.tensor(expected_logits_all.get_expectation(), dtype=torch.bfloat16)
# The original implementation and the transformers implementation do not match exactly, hence the higher tolerance.
# The difference is likely due to the different implementations of the attention mechanism (different order of operations)
# between the transformers Llama model and the original InternLM model.
# The difference has almost no effect on the output tokens, but it does affect the logits a lot more.
self.assertTrue(
torch.allclose(actual_logits, expected_logits, atol=1),
torch.allclose(actual_logits, expected_logits, atol=1e-3),
f"Actual logits: {actual_logits}"
f"\nExpected logits: {expected_logits}"
f"\nDifference: {torch.abs(actual_logits - expected_logits)}",
@ -765,7 +778,8 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden path leads to calm lake,\nNature's peaceful grace.",
("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden path leads to calm lake,\nNature's peaceful grace.",
("cuda", 8): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nA wooden path leads to the sea,\nPeaceful, still waters.",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -819,7 +833,7 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
# Check first output
decoded_output = processor.decode(output[0], skip_special_tokens=True)
# Batching seems to alter the output slightly, but it is also the case in the original implementation. This seems to be expected: https://github.com/huggingface/transformers/issues/23017#issuecomment-1649630232
expected_output = 'user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nA wooden path leads to the sea,\nPeaceful, still waters.' # fmt: skip
expected_output = "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden path leads to calm lake,\nNature's peaceful grace." # fmt: skip
self.assertEqual(
decoded_output,
expected_output,
@ -940,7 +954,8 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. After re-examining the images, I can see that they are actually",
("cuda", 7): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. Upon closer inspection, the differences between the two images are:\n\n1. **",
("cuda", 7): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. After re-examining the images, I can see that they are actually",
("cuda", 8): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. After closely examining the images again, I can see that there are several differences",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -956,6 +971,7 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
{
("xpu", 3): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot. This is a common shot in tennis where the player swings the racket across their",
("cuda", 7): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot. This is a common shot in tennis where the player swings the racket across their",
("cuda", 8): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot. This is a common shot in tennis where the player swings the racket across their",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()
@ -970,7 +986,8 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
expected_outputs = Expectations(
{
("xpu", 3): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nA wooden path leads to the sea,\nPeaceful, untouched dreams.",
("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
("cuda", 8): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
}
) # fmt: skip
expected_output = expected_outputs.get_expectation()