From ffad4f13736ee0996e232135d699983a4a8b91b3 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 18 May 2023 17:29:04 +0200 Subject: [PATCH] Update tiny models and pipeline tests (#23446) * fix --------- Co-authored-by: ydshieh --- tests/models/biogpt/test_modeling_biogpt.py | 3 +- tests/models/cpmant/test_modeling_cpmant.py | 6 +- .../models/focalnet/test_modeling_focalnet.py | 8 +- tests/models/gpt_neo/test_modeling_gpt_neo.py | 2 +- .../models/gpt_neox/test_modeling_gpt_neox.py | 2 +- tests/models/rwkv/test_modeling_rwkv.py | 7 +- .../swiftformer/test_modeling_swiftformer.py | 1 - tests/test_pipeline_mixin.py | 27 ++++ tests/utils/tiny_model_summary.json | 143 ++++++++++++++++++ 9 files changed, 187 insertions(+), 12 deletions(-) diff --git a/tests/models/biogpt/test_modeling_biogpt.py b/tests/models/biogpt/test_modeling_biogpt.py index 7e64fc07dad..e5041093088 100644 --- a/tests/models/biogpt/test_modeling_biogpt.py +++ b/tests/models/biogpt/test_modeling_biogpt.py @@ -289,9 +289,10 @@ class BioGptModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix pipeline_model_mapping = ( { "feature-extraction": BioGptModel, + "text-classification": BioGptForSequenceClassification, "text-generation": BioGptForCausalLM, "token-classification": BioGptForTokenClassification, - "text-classification": BioGptForSequenceClassification, + "zero-shot": BioGptForSequenceClassification, } if is_torch_available() else {} diff --git a/tests/models/cpmant/test_modeling_cpmant.py b/tests/models/cpmant/test_modeling_cpmant.py index 61cd0ec17e2..f3ebeaad3e9 100644 --- a/tests/models/cpmant/test_modeling_cpmant.py +++ b/tests/models/cpmant/test_modeling_cpmant.py @@ -21,6 +21,7 @@ from transformers.testing_utils import is_torch_available, require_torch, tooslo from ...generation.test_utils import torch_device from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, ids_tensor +from ...test_pipeline_mixin import PipelineTesterMixin if is_torch_available(): @@ -133,8 +134,11 @@ class CpmAntModelTester: @require_torch -class CpmAntModelTest(ModelTesterMixin, unittest.TestCase): +class CpmAntModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = (CpmAntModel, CpmAntForCausalLM) if is_torch_available() else () + pipeline_model_mapping = ( + {"feature-extraction": CpmAntModel, "text-generation": CpmAntForCausalLM} if is_torch_available() else {} + ) test_pruning = False test_missing_keys = False diff --git a/tests/models/focalnet/test_modeling_focalnet.py b/tests/models/focalnet/test_modeling_focalnet.py index 75127e5fd38..ce96f0ade41 100644 --- a/tests/models/focalnet/test_modeling_focalnet.py +++ b/tests/models/focalnet/test_modeling_focalnet.py @@ -25,6 +25,7 @@ from transformers.utils import cached_property, is_torch_available, is_vision_av from ...test_backbone_common import BackboneTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor +from ...test_pipeline_mixin import PipelineTesterMixin if is_torch_available(): @@ -226,7 +227,7 @@ class FocalNetModelTester: @require_torch -class FocalNetModelTest(ModelTesterMixin, unittest.TestCase): +class FocalNetModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = ( ( FocalNetModel, @@ -237,6 +238,11 @@ class FocalNetModelTest(ModelTesterMixin, unittest.TestCase): if is_torch_available() else () ) + pipeline_model_mapping = ( + {"feature-extraction": FocalNetModel, "image-classification": FocalNetForImageClassification} + if is_torch_available() + else {} + ) fx_compatible = False test_pruning = False diff --git a/tests/models/gpt_neo/test_modeling_gpt_neo.py b/tests/models/gpt_neo/test_modeling_gpt_neo.py index c0334bd05aa..a79cf5b25d7 100644 --- a/tests/models/gpt_neo/test_modeling_gpt_neo.py +++ b/tests/models/gpt_neo/test_modeling_gpt_neo.py @@ -413,8 +413,8 @@ class GPTNeoModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix "feature-extraction": GPTNeoModel, "question-answering": GPTNeoForQuestionAnswering, "text-classification": GPTNeoForSequenceClassification, - "token-classification": GPTNeoForTokenClassification, "text-generation": GPTNeoForCausalLM, + "token-classification": GPTNeoForTokenClassification, "zero-shot": GPTNeoForSequenceClassification, } if is_torch_available() diff --git a/tests/models/gpt_neox/test_modeling_gpt_neox.py b/tests/models/gpt_neox/test_modeling_gpt_neox.py index 927d097691c..c2bbbe58e11 100644 --- a/tests/models/gpt_neox/test_modeling_gpt_neox.py +++ b/tests/models/gpt_neox/test_modeling_gpt_neox.py @@ -239,8 +239,8 @@ class GPTNeoXModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMi "feature-extraction": GPTNeoXModel, "question-answering": GPTNeoXForQuestionAnswering, "text-classification": GPTNeoXForSequenceClassification, - "token-classification": GPTNeoXForTokenClassification, "text-generation": GPTNeoXForCausalLM, + "token-classification": GPTNeoXForTokenClassification, "zero-shot": GPTNeoXForSequenceClassification, } if is_torch_available() diff --git a/tests/models/rwkv/test_modeling_rwkv.py b/tests/models/rwkv/test_modeling_rwkv.py index 4afcc9b41f8..638180cbcf9 100644 --- a/tests/models/rwkv/test_modeling_rwkv.py +++ b/tests/models/rwkv/test_modeling_rwkv.py @@ -262,12 +262,7 @@ class RwkvModelTester: class RwkvModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = (RwkvModel, RwkvForCausalLM) if is_torch_available() else () pipeline_model_mapping = ( - { - "feature-extraction": RwkvModel, - "text-generation": RwkvForCausalLM, - } - if is_torch_available() - else {} + {"feature-extraction": RwkvModel, "text-generation": RwkvForCausalLM} if is_torch_available() else {} ) # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else () fx_compatible = False diff --git a/tests/models/swiftformer/test_modeling_swiftformer.py b/tests/models/swiftformer/test_modeling_swiftformer.py index f2785a6640c..159d5ec0fa9 100644 --- a/tests/models/swiftformer/test_modeling_swiftformer.py +++ b/tests/models/swiftformer/test_modeling_swiftformer.py @@ -139,7 +139,6 @@ class SwiftFormerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestC """ all_model_classes = (SwiftFormerModel, SwiftFormerForImageClassification) if is_torch_available() else () - pipeline_model_mapping = ( {"feature-extraction": SwiftFormerModel, "image-classification": SwiftFormerForImageClassification} if is_torch_available() diff --git a/tests/test_pipeline_mixin.py b/tests/test_pipeline_mixin.py index 904d94e4b2c..05ea27121a5 100644 --- a/tests/test_pipeline_mixin.py +++ b/tests/test_pipeline_mixin.py @@ -263,6 +263,15 @@ class PipelineTesterMixin: ) return + pipeline_test_class_name = pipeline_test_mapping[task]["test"].__name__ + if self.is_pipeline_test_to_skip_more(pipeline_test_class_name, model.config, model, tokenizer, processor): + logger.warning( + f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: test is " + f"currently known to fail for: model `{model_architecture.__name__}` | tokenizer " + f"`{tokenizer_name}` | processor `{processor_name}`." + ) + return + # validate validate_test_components(self, task, model, tokenizer, processor) @@ -440,6 +449,10 @@ class PipelineTesterMixin: def is_pipeline_test_to_skip( self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name ): + """Skip some tests based on the classes or their names without the instantiated objects. + + This is to avoid calling `from_pretrained` (so reducing the runtime) if we already know the tests will fail. + """ # No fix is required for this case. if ( pipeline_test_casse_name == "DocumentQuestionAnsweringPipelineTests" @@ -451,6 +464,20 @@ class PipelineTesterMixin: return False + def is_pipeline_test_to_skip_more(self, pipeline_test_casse_name, config, model, tokenizer, processor): # noqa + """Skip some more tests based on the information from the instantiated objects.""" + # No fix is required for this case. + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer is not None + and getattr(tokenizer, "pad_token", None) is None + and not tokenizer.__class__.__name__.endswith("Fast") + ): + # `QAPipelineTests` doesn't work with a slow tokenizer that has no pad token. + return True + + return False + def validate_test_components(test_case, task, model, tokenizer, processor): # TODO: Move this to tiny model creation script diff --git a/tests/utils/tiny_model_summary.json b/tests/utils/tiny_model_summary.json index 127ef7a9fcf..a186bf17e19 100644 --- a/tests/utils/tiny_model_summary.json +++ b/tests/utils/tiny_model_summary.json @@ -477,6 +477,16 @@ ], "sha": "07073b31da84054fd12226e3cae4cb3beb2547f9" }, + "BioGptForSequenceClassification": { + "tokenizer_classes": [ + "BioGptTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "BioGptForSequenceClassification" + ], + "sha": "8e18ad6218abd795e050dec324a8c827ccedacb4" + }, "BioGptForTokenClassification": { "tokenizer_classes": [ "BioGptTokenizer" @@ -2131,6 +2141,46 @@ ], "sha": "31ebf1b7a0ef1fd5059b98e28e5ab1c366d2c482" }, + "FocalNetBackbone": { + "tokenizer_classes": [], + "processor_classes": [ + "BitImageProcessor" + ], + "model_classes": [ + "FocalNetBackbone" + ], + "sha": "eb8c580969443cb87de7dd9a256deaface03692f" + }, + "FocalNetForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "BitImageProcessor" + ], + "model_classes": [ + "FocalNetForImageClassification" + ], + "sha": "28d30ded26a3213e8fb7011a455afc3aa98b0a95" + }, + "FocalNetForMaskedImageModeling": { + "tokenizer_classes": [], + "processor_classes": [ + "BitImageProcessor" + ], + "model_classes": [ + "FocalNetForMaskedImageModeling" + ], + "sha": "0ea7626d19c9dd2f3113d977f643a1babc720bd3" + }, + "FocalNetModel": { + "tokenizer_classes": [], + "processor_classes": [ + "BitImageProcessor" + ], + "model_classes": [ + "FocalNetModel" + ], + "sha": "107b004e6aa14108a359b7d22bdb9aa141ec05d5" + }, "FunnelBaseModel": { "tokenizer_classes": [ "FunnelTokenizer", @@ -2247,6 +2297,17 @@ ], "sha": "24a8dbb48b1aa0ba2eba44324fcd0c78cca64dd4" }, + "GPT2ForQuestionAnswering": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPT2ForQuestionAnswering" + ], + "sha": "a5bdd6bd4d79feece85ea9a8bd4ee5fe54c1d45b" + }, "GPT2ForSequenceClassification": { "tokenizer_classes": [ "GPT2Tokenizer", @@ -2397,6 +2458,17 @@ ], "sha": "e88934e402c15195dd99b2947632415dd7645268" }, + "GPTNeoForQuestionAnswering": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoForQuestionAnswering" + ], + "sha": "623883e94bd08caf9b3f839b98debeea72d5bc2b" + }, "GPTNeoForSequenceClassification": { "tokenizer_classes": [ "GPT2Tokenizer", @@ -2408,6 +2480,17 @@ ], "sha": "bf2090d5d91a70eb37ba51fbdcf23afc7031fea8" }, + "GPTNeoForTokenClassification": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoForTokenClassification" + ], + "sha": "d5208e73e24a1671219776b50fe5f96e0e4cd218" + }, "GPTNeoModel": { "tokenizer_classes": [ "GPT2Tokenizer", @@ -2429,6 +2512,16 @@ ], "sha": "0229cfaaa843c6b492ac2abffabb00f1ff1936f8" }, + "GPTNeoXForQuestionAnswering": { + "tokenizer_classes": [ + "GPTNeoXTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoXForQuestionAnswering" + ], + "sha": "7d2f08c959c211129952ee03b5562add09fe6864" + }, "GPTNeoXForSequenceClassification": { "tokenizer_classes": [ "GPTNeoXTokenizerFast" @@ -2439,6 +2532,16 @@ ], "sha": "17c4b845ee2e0bb780ca2dea2d59a3d9d5d3c651" }, + "GPTNeoXForTokenClassification": { + "tokenizer_classes": [ + "GPTNeoXTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoXForTokenClassification" + ], + "sha": "3aa4fe8a562f32230041d6d3616aa5ecc3f30192" + }, "GPTNeoXJapaneseForCausalLM": { "tokenizer_classes": [ "GPTNeoXJapaneseTokenizer" @@ -4922,6 +5025,26 @@ ], "sha": "4830db38fd310404c5ab70bd00684eca0bc06ca8" }, + "RwkvForCausalLM": { + "tokenizer_classes": [ + "GPTNeoXTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RwkvForCausalLM" + ], + "sha": "2f452fd46b39e39b1a6a95fa1d8232405bbb3e96" + }, + "RwkvModel": { + "tokenizer_classes": [ + "GPTNeoXTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RwkvModel" + ], + "sha": "88a52c9437dc3c06f65a8252490be7eb91197804" + }, "SEWDForCTC": { "tokenizer_classes": [ "Wav2Vec2CTCTokenizer" @@ -5196,6 +5319,26 @@ ], "sha": "e0a3ac56a4047da3f921638252ead5e44438bbdb" }, + "SwiftFormerForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "SwiftFormerForImageClassification" + ], + "sha": "a249b14a525d29e675b6e4af4baacd9ba7df7598" + }, + "SwiftFormerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "SwiftFormerModel" + ], + "sha": "25ba2d88c770533f8c69811d2a454a00c1d09f5d" + }, "Swin2SRModel": { "tokenizer_classes": [], "processor_classes": [