diff --git a/tests/models/qwen2_5_omni/test_processor_qwen2_5_omni.py b/tests/models/qwen2_5_omni/test_processor_qwen2_5_omni.py index bcdaeb45bd6..52ce99c86ff 100644 --- a/tests/models/qwen2_5_omni/test_processor_qwen2_5_omni.py +++ b/tests/models/qwen2_5_omni/test_processor_qwen2_5_omni.py @@ -204,8 +204,7 @@ class Qwen2_5OmniProcessorTest(ProcessorTesterMixin, unittest.TestCase): @classmethod def setUpClass(cls): cls.tmpdirname = tempfile.mkdtemp() - processor_kwargs = cls.prepare_processor_dict() - processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B", **processor_kwargs) + processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B") processor.save_pretrained(cls.tmpdirname) def get_tokenizer(self, **kwargs): @@ -220,11 +219,8 @@ class Qwen2_5OmniProcessorTest(ProcessorTesterMixin, unittest.TestCase): def get_feature_extractor(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).feature_extractor - @staticmethod - def prepare_processor_dict(): - return { - "chat_template": "{% set audio_count = namespace(value=0) %}{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_bos|><|IMAGE|><|vision_eos|>{% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}{% set audio_count.value = audio_count.value + 1 %}{% if add_audio_id %}Audio {{ audio_count.value }}: {% endif %}<|audio_bos|><|AUDIO|><|audio_eos|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_bos|><|VIDEO|><|vision_eos|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" - } + def get_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs) @classmethod def tearDownClass(cls): diff --git a/tests/models/qwen2_5_vl/test_processor_qwen2_5_vl.py b/tests/models/qwen2_5_vl/test_processor_qwen2_5_vl.py index 86c2707a8c6..2fc284c638b 100644 --- a/tests/models/qwen2_5_vl/test_processor_qwen2_5_vl.py +++ b/tests/models/qwen2_5_vl/test_processor_qwen2_5_vl.py @@ -58,11 +58,8 @@ class Qwen2_5_VLProcessorTest(ProcessorTesterMixin, unittest.TestCase): def get_video_processor(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).video_processor - @staticmethod - def prepare_processor_dict(): - return { - "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}", - } # fmt: skip + def get_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs) @classmethod def tearDownClass(cls): @@ -340,12 +337,7 @@ class Qwen2_5_VLProcessorTest(ProcessorTesterMixin, unittest.TestCase): self.assertEqual(len(out_dict_with_video[self.videos_input_name]), 160) def test_kwargs_overrides_custom_image_processor_kwargs(self): - processor_components = self.prepare_components() - processor_components["image_processor"] = self.get_component("image_processor") - processor_components["tokenizer"] = self.get_component("tokenizer") - processor_kwargs = self.prepare_processor_dict() - - processor = self.processor_class(**processor_components, **processor_kwargs, use_fast=True) + processor = self.get_processor() self.skip_processor_without_typed_kwargs(processor) input_str = self.prepare_text_inputs() diff --git a/tests/models/qwen2_audio/test_processor_qwen2_audio.py b/tests/models/qwen2_audio/test_processor_qwen2_audio.py index 67451144c4b..007b0d3c7ab 100644 --- a/tests/models/qwen2_audio/test_processor_qwen2_audio.py +++ b/tests/models/qwen2_audio/test_processor_qwen2_audio.py @@ -36,8 +36,7 @@ class Qwen2AudioProcessorTest(ProcessorTesterMixin, unittest.TestCase): cls.checkpoint = "Qwen/Qwen2-Audio-7B-Instruct" cls.tmpdirname = tempfile.mkdtemp() - processor_kwargs = cls.prepare_processor_dict() - processor = Qwen2AudioProcessor.from_pretrained(cls.checkpoint, **processor_kwargs) + processor = Qwen2AudioProcessor.from_pretrained(cls.checkpoint) processor.save_pretrained(cls.tmpdirname) cls.audio_token = processor.audio_token @@ -47,16 +46,13 @@ class Qwen2AudioProcessorTest(ProcessorTesterMixin, unittest.TestCase): def get_audio_processor(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).audio_processor + def get_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs) + @classmethod def tearDownClass(cls): shutil.rmtree(cls.tmpdirname, ignore_errors=True) - @staticmethod - def prepare_processor_dict(): - return { - "chat_template": "{% set audio_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if 'audio' in content or 'audio_url' in content or content['type'] == 'audio' %}{% set audio_count.value = audio_count.value + 1 %}Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\n{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}", - } - def test_can_load_various_tokenizers(self): processor = Qwen2AudioProcessor.from_pretrained(self.checkpoint) tokenizer = AutoTokenizer.from_pretrained(self.checkpoint) diff --git a/tests/models/qwen2_vl/test_processor_qwen2_vl.py b/tests/models/qwen2_vl/test_processor_qwen2_vl.py index d38c3d484f9..b109fbe078c 100644 --- a/tests/models/qwen2_vl/test_processor_qwen2_vl.py +++ b/tests/models/qwen2_vl/test_processor_qwen2_vl.py @@ -61,9 +61,8 @@ class Qwen2VLProcessorTest(ProcessorTesterMixin, unittest.TestCase): def get_video_processor(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).video_processor - @staticmethod - def prepare_processor_dict(): - return {"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"} # fmt: skip + def get_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs) @classmethod def tearDownClass(cls): @@ -406,12 +405,7 @@ class Qwen2VLProcessorTest(ProcessorTesterMixin, unittest.TestCase): self.assertEqual(len(out_dict_with_video[self.videos_input_name]), 21960) def test_kwargs_overrides_custom_image_processor_kwargs(self): - processor_components = self.prepare_components() - processor_components["image_processor"] = self.get_component("image_processor") - processor_components["tokenizer"] = self.get_component("tokenizer") - processor_kwargs = self.prepare_processor_dict() - - processor = self.processor_class(**processor_components, **processor_kwargs, use_fast=True) + processor = self.get_processor() self.skip_processor_without_typed_kwargs(processor) input_str = self.prepare_text_inputs()