From 9ac586b3c8636d2216cdfd32d00e40c396e23148 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Fri, 7 Oct 2022 18:01:58 -0400 Subject: [PATCH] Rework pipeline tests (#19366) * Rework pipeline tests * Try to fix Flax tests * Try to put it before * Use a new decorator instead * Remove ignore marker since it doesn't work * Filter pipeline tests * Woopsie * Use the fitlered list * Clean up and fake modif * Remove init * Revert fake modif --- .circleci/config.yml | 52 +++++++++++-------- .github/workflows/self-scheduled.yml | 8 +-- conftest.py | 1 - src/transformers/testing_utils.py | 32 +++++------- .../test_pipelines_audio_classification.py | 10 +--- ..._pipelines_automatic_speech_recognition.py | 2 - tests/pipelines/test_pipelines_common.py | 8 ++- .../test_pipelines_conversational.py | 3 +- ...t_pipelines_document_question_answering.py | 2 - .../test_pipelines_feature_extraction.py | 3 +- tests/pipelines/test_pipelines_fill_mask.py | 10 +--- .../test_pipelines_image_classification.py | 4 +- .../test_pipelines_image_segmentation.py | 11 +--- .../pipelines/test_pipelines_image_to_text.py | 3 +- .../test_pipelines_object_detection.py | 11 +--- .../test_pipelines_question_answering.py | 5 +- .../pipelines/test_pipelines_summarization.py | 3 +- ...test_pipelines_table_question_answering.py | 2 - .../test_pipelines_text2text_generation.py | 3 +- .../test_pipelines_text_classification.py | 3 +- .../test_pipelines_text_generation.py | 4 +- .../test_pipelines_token_classification.py | 11 +--- tests/pipelines/test_pipelines_translation.py | 4 +- ...est_pipelines_visual_question_answering.py | 10 +--- tests/pipelines/test_pipelines_zero_shot.py | 3 +- ...ipelines_zero_shot_image_classification.py | 10 +--- utils/tests_fetcher.py | 26 ++++++++++ 27 files changed, 95 insertions(+), 149 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aef10586cdc..f54a5998088 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,10 +79,19 @@ jobs: path: ~/transformers/tests_fetched_summary.txt - run: | if [ -f test_list.txt ]; then - mv test_list.txt test_preparation/test_list.txt + cp test_list.txt test_preparation/test_list.txt else touch test_preparation/test_list.txt fi + - run: python utils/tests_fetcher.py --filter_pipeline_tests + - run: | + if [ -f test_list.txt ]; then + mv test_list.txt test_preparation/filtered_test_list.txt + else + touch test_preparation/filtered_test_list.txt + fi + - store_artifacts: + path: ~/transformers/test_preparation/filtered_test_list.txt - run: python utils/tests_fetcher.py --filters tests examples | tee examples_tests_fetched_summary.txt - store_artifacts: path: ~/transformers/examples_tests_fetched_summary.txt @@ -97,6 +106,7 @@ jobs: root: test_preparation/ paths: test_list.txt + filtered_test_list.txt examples_test_list.txt # To run all tests for the nightly build @@ -110,6 +120,8 @@ jobs: mkdir test_preparation echo "tests" > test_preparation/test_list.txt echo "tests" > test_preparation/examples_test_list.txt + - run: python utils/tests_fetcher.py --filter_pipeline_tests + - run: mv test_list.txt test_preparation/filtered_test_list.txt - persist_to_workspace: root: test_preparation/ @@ -132,7 +144,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -152,7 +164,7 @@ jobs: key: v0.5-torch_and_tf-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf $(cat test_preparation/test_list.txt) -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt + - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf $(cat test_preparation/filtered_test_list.txt) -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -174,7 +186,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -192,7 +204,7 @@ jobs: key: v0.5-torch_and_flax-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax $(cat test_preparation/test_list.txt) -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt + - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax $(cat test_preparation/filtered_test_list.txt) -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -213,7 +225,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -231,7 +243,7 @@ jobs: key: v0.5-torch-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 3 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_torch $(cat test_preparation/test_list.txt) | tee tests_output.txt + - run: python -m pytest -n 3 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_torch $(cat test_preparation/filtered_test_list.txt) | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -252,7 +264,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -269,7 +281,7 @@ jobs: key: v0.5-tf-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_tf $(cat test_preparation/test_list.txt) | tee tests_output.txt + - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_tf $(cat test_preparation/filtered_test_list.txt) | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -290,7 +302,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -306,7 +318,7 @@ jobs: key: v0.5-flax-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_flax $(cat test_preparation/test_list.txt) | tee tests_output.txt + - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_flax $(cat test_preparation/filtered_test_list.txt) | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -318,7 +330,6 @@ jobs: - image: cimg/python:3.7.12 environment: OMP_NUM_THREADS: 1 - RUN_PIPELINE_TESTS: yes TRANSFORMERS_IS_CI: yes PYTEST_TIMEOUT: 120 resource_class: xlarge @@ -345,7 +356,7 @@ jobs: key: v0.5-torch-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test $(cat test_preparation/test_list.txt) | tee tests_output.txt + - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch tests/pipelines | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -357,7 +368,6 @@ jobs: - image: cimg/python:3.7.12 environment: OMP_NUM_THREADS: 1 - RUN_PIPELINE_TESTS: yes TRANSFORMERS_IS_CI: yes PYTEST_TIMEOUT: 120 resource_class: xlarge @@ -382,7 +392,7 @@ jobs: key: v0.5-tf-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf $(cat test_preparation/test_list.txt) -m is_pipeline_test | tee tests_output.txt + - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf tests/pipelines | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -401,7 +411,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -557,7 +567,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -575,7 +585,7 @@ jobs: key: v0.5-hub-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest --max-worker-restart=0 -sv --make-reports=tests_hub $(cat test_preparation/test_list.txt) -m is_staging_test | tee tests_output.txt + - run: python -m pytest --max-worker-restart=0 -sv --make-reports=tests_hub $(cat test_preparation/filtered_test_list.txt) -m is_staging_test | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt - store_artifacts: @@ -596,7 +606,7 @@ jobs: - attach_workspace: at: ~/transformers/test_preparation - run: | - if [ ! -s test_preparation/test_list.txt ]; then + if [ ! -s test_preparation/filtered_test_list.txt ]; then echo "No tests to run, exiting early!" circleci-agent step halt fi @@ -610,7 +620,7 @@ jobs: key: v0.5-onnx-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - - run: python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_onnx $(cat test_preparation/test_list.txt) -k onnx | tee tests_output.txt + - run: python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_onnx $(cat test_preparation/filtered_test_list.txt) -k onnx | tee tests_output.txt - store_artifacts: path: ~/transformers/tests_output.txt @@ -690,7 +700,7 @@ jobs: steps: - checkout - attach_workspace: - at: ~/transformers/test_preparation + at: ~/transformers/filtered_test_list.txt - run: | if [ ! -s test_preparation/test_list.txt ]; then echo "No tests to run, exiting early!" diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 7de69a573e3..3b91eac50d5 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -256,10 +256,8 @@ jobs: - name: Run all pipeline tests on GPU working-directory: /transformers - env: - RUN_PIPELINE_TESTS: yes run: | - python3 -m pytest -n 1 -v --dist=loadfile -m is_pipeline_test --make-reports=${{ matrix.machine_type }}_tests_torch_pipeline_gpu tests + python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_tests_torch_pipeline_gpu tests/pipelines - name: Failure short reports if: ${{ failure() }} @@ -301,10 +299,8 @@ jobs: - name: Run all pipeline tests on GPU working-directory: /transformers - env: - RUN_PIPELINE_TESTS: yes run: | - python3 -m pytest -n 1 -v --dist=loadfile -m is_pipeline_test --make-reports=${{ matrix.machine_type }}_tests_tf_pipeline_gpu tests + python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_tests_tf_pipeline_gpu tests/pipelines - name: Failure short reports if: ${{ always() }} diff --git a/conftest.py b/conftest.py index e71ada998a6..c3d4f70326d 100644 --- a/conftest.py +++ b/conftest.py @@ -32,7 +32,6 @@ warnings.simplefilter(action="ignore", category=FutureWarning) def pytest_configure(config): - config.addinivalue_line("markers", "is_pipeline_test: mark test to run only when pipeline are tested") config.addinivalue_line( "markers", "is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested" ) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 08409b6e092..5884e642d9b 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -133,7 +133,6 @@ _run_pt_tf_cross_tests = parse_flag_from_env("RUN_PT_TF_CROSS_TESTS", default=Fa _run_pt_flax_cross_tests = parse_flag_from_env("RUN_PT_FLAX_CROSS_TESTS", default=False) _run_custom_tokenizers = parse_flag_from_env("RUN_CUSTOM_TOKENIZERS", default=False) _run_staging = parse_flag_from_env("HUGGINGFACE_CO_STAGING", default=False) -_run_pipeline_tests = parse_flag_from_env("RUN_PIPELINE_TESTS", default=False) _run_git_lfs_tests = parse_flag_from_env("RUN_GIT_LFS_TESTS", default=False) _tf_gpu_memory_limit = parse_int_from_env("TF_GPU_MEMORY_LIMIT", default=None) @@ -176,25 +175,6 @@ def is_pt_flax_cross_test(test_case): return pytest.mark.is_pt_flax_cross_test()(test_case) -def is_pipeline_test(test_case): - """ - Decorator marking a test as a pipeline test. - - Pipeline tests are skipped by default and we can run only them by setting RUN_PIPELINE_TESTS environment variable - to a truthy value and selecting the is_pipeline_test pytest mark. - - """ - if not _run_pipeline_tests: - return unittest.skip("test is pipeline test")(test_case) - else: - try: - import pytest # We don't need a hard dependency on pytest in the main library - except ImportError: - return test_case - else: - return pytest.mark.is_pipeline_test()(test_case) - - def is_staging_test(test_case): """ Decorator marking a test as a staging test. @@ -309,6 +289,18 @@ def require_torch(test_case): return unittest.skipUnless(is_torch_available(), "test requires PyTorch")(test_case) +def require_torch_or_tf(test_case): + """ + Decorator marking a test that requires PyTorch or TensorFlow. + + These tests are skipped when neither PyTorch not TensorFlow is installed. + + """ + return unittest.skipUnless(is_torch_available() or is_tf_available(), "test requires PyTorch or TensorFlow")( + test_case + ) + + def require_intel_extension_for_pytorch(test_case): """ Decorator marking a test that requires Intel Extension for PyTorch. diff --git a/tests/pipelines/test_pipelines_audio_classification.py b/tests/pipelines/test_pipelines_audio_classification.py index df32cf58d35..3f957132fd8 100644 --- a/tests/pipelines/test_pipelines_audio_classification.py +++ b/tests/pipelines/test_pipelines_audio_classification.py @@ -18,19 +18,11 @@ import numpy as np from transformers import MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING from transformers.pipelines import AudioClassificationPipeline, pipeline -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_torch, - require_torchaudio, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_torchaudio, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test @require_torch class AudioClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING diff --git a/tests/pipelines/test_pipelines_automatic_speech_recognition.py b/tests/pipelines/test_pipelines_automatic_speech_recognition.py index 0523639cc4f..d4fcbf5f781 100644 --- a/tests/pipelines/test_pipelines_automatic_speech_recognition.py +++ b/tests/pipelines/test_pipelines_automatic_speech_recognition.py @@ -31,7 +31,6 @@ from transformers.pipelines import AutomaticSpeechRecognitionPipeline, pipeline from transformers.pipelines.audio_utils import chunk_bytes_iter from transformers.pipelines.automatic_speech_recognition import chunk_iter from transformers.testing_utils import ( - is_pipeline_test, is_torch_available, nested_simplify, require_pyctcdecode, @@ -52,7 +51,6 @@ if is_torch_available(): # from .test_pipelines_common import CustomInputPipelineCommonMixin -@is_pipeline_test class AutomaticSpeechRecognitionPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = { k: v diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 34684186b54..0f03a42440d 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -48,13 +48,13 @@ from transformers.testing_utils import ( USER, CaptureLogger, RequestCounter, - is_pipeline_test, is_staging_test, nested_simplify, require_scatter, require_tensorflow_probability, require_tf, require_torch, + require_torch_or_tf, slow, ) from transformers.utils import is_tf_available, is_torch_available @@ -307,7 +307,6 @@ class PipelineTestCaseMeta(type): return type.__new__(mcs, name, bases, dct) -@is_pipeline_test class CommonPipelineTest(unittest.TestCase): @require_torch def test_pipeline_iteration(self): @@ -416,7 +415,6 @@ class CommonPipelineTest(unittest.TestCase): self.assertEqual(len(outputs), 20) -@is_pipeline_test class PipelinePadTest(unittest.TestCase): @require_torch def test_pipeline_padding(self): @@ -498,7 +496,6 @@ class PipelinePadTest(unittest.TestCase): ) -@is_pipeline_test class PipelineUtilsTest(unittest.TestCase): @require_torch def test_pipeline_dataset(self): @@ -795,7 +792,6 @@ class CustomPipeline(Pipeline): return model_outputs["logits"].softmax(-1).numpy() -@is_pipeline_test class CustomPipelineTest(unittest.TestCase): def test_warning_logs(self): transformers_logging.set_verbosity_debug() @@ -835,6 +831,7 @@ class CustomPipelineTest(unittest.TestCase): # Clean registry for next tests. del PIPELINE_REGISTRY.supported_tasks["custom-text-classification"] + @require_torch_or_tf def test_dynamic_pipeline(self): PIPELINE_REGISTRY.register_pipeline( "pair-classification", @@ -886,6 +883,7 @@ class CustomPipelineTest(unittest.TestCase): [{"label": "LABEL_0", "score": 0.505}], ) + @require_torch_or_tf def test_cached_pipeline_has_minimum_calls_to_head(self): # Make sure we have cached the pipeline. _ = pipeline("text-classification", model="hf-internal-testing/tiny-random-bert") diff --git a/tests/pipelines/test_pipelines_conversational.py b/tests/pipelines/test_pipelines_conversational.py index 342a09e2e69..39ad2175dcf 100644 --- a/tests/pipelines/test_pipelines_conversational.py +++ b/tests/pipelines/test_pipelines_conversational.py @@ -29,7 +29,7 @@ from transformers import ( TFAutoModelForCausalLM, pipeline, ) -from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow, torch_device +from transformers.testing_utils import require_tf, require_torch, slow, torch_device from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -37,7 +37,6 @@ from .test_pipelines_common import ANY, PipelineTestCaseMeta DEFAULT_DEVICE_NUM = -1 if torch_device == "cpu" else 0 -@is_pipeline_test class ConversationalPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = dict( list(MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()) diff --git a/tests/pipelines/test_pipelines_document_question_answering.py b/tests/pipelines/test_pipelines_document_question_answering.py index 92d618bfd67..bea8335c5c8 100644 --- a/tests/pipelines/test_pipelines_document_question_answering.py +++ b/tests/pipelines/test_pipelines_document_question_answering.py @@ -18,7 +18,6 @@ from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoToke from transformers.pipelines import pipeline from transformers.pipelines.document_question_answering import apply_tesseract from transformers.testing_utils import ( - is_pipeline_test, nested_simplify, require_detectron2, require_pytesseract, @@ -53,7 +52,6 @@ INVOICE_URL = ( ) -@is_pipeline_test @require_torch @require_vision class DocumentQuestionAnsweringPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): diff --git a/tests/pipelines/test_pipelines_feature_extraction.py b/tests/pipelines/test_pipelines_feature_extraction.py index 42cdb79bb66..7fab49dd604 100644 --- a/tests/pipelines/test_pipelines_feature_extraction.py +++ b/tests/pipelines/test_pipelines_feature_extraction.py @@ -22,12 +22,11 @@ from transformers import ( LxmertConfig, pipeline, ) -from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch +from transformers.testing_utils import nested_simplify, require_tf, require_torch from .test_pipelines_common import PipelineTestCaseMeta -@is_pipeline_test class FeatureExtractionPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_MAPPING tf_model_mapping = TF_MODEL_MAPPING diff --git a/tests/pipelines/test_pipelines_fill_mask.py b/tests/pipelines/test_pipelines_fill_mask.py index d85ab8d7ce3..760c475524f 100644 --- a/tests/pipelines/test_pipelines_fill_mask.py +++ b/tests/pipelines/test_pipelines_fill_mask.py @@ -16,19 +16,11 @@ import unittest from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline from transformers.pipelines import PipelineException -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_torch, - require_torch_gpu, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_torch_gpu, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test class FillMaskPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_MASKED_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_MASKED_LM_MAPPING diff --git a/tests/pipelines/test_pipelines_image_classification.py b/tests/pipelines/test_pipelines_image_classification.py index 8e19d60e80f..59a58ff44d5 100644 --- a/tests/pipelines/test_pipelines_image_classification.py +++ b/tests/pipelines/test_pipelines_image_classification.py @@ -22,10 +22,10 @@ from transformers import ( ) from transformers.pipelines import ImageClassificationPipeline, pipeline from transformers.testing_utils import ( - is_pipeline_test, nested_simplify, require_tf, require_torch, + require_torch_or_tf, require_vision, slow, ) @@ -43,7 +43,7 @@ else: pass -@is_pipeline_test +@require_torch_or_tf @require_vision class ImageClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING diff --git a/tests/pipelines/test_pipelines_image_segmentation.py b/tests/pipelines/test_pipelines_image_segmentation.py index 65656939d0c..3d7d067afa7 100644 --- a/tests/pipelines/test_pipelines_image_segmentation.py +++ b/tests/pipelines/test_pipelines_image_segmentation.py @@ -31,15 +31,7 @@ from transformers import ( is_vision_available, pipeline, ) -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_timm, - require_torch, - require_vision, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_timm, require_torch, require_vision, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -62,7 +54,6 @@ def hashimage(image: Image) -> str: @require_vision @require_timm @require_torch -@is_pipeline_test class ImageSegmentationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = { k: v diff --git a/tests/pipelines/test_pipelines_image_to_text.py b/tests/pipelines/test_pipelines_image_to_text.py index 897c3b2e472..32503033003 100644 --- a/tests/pipelines/test_pipelines_image_to_text.py +++ b/tests/pipelines/test_pipelines_image_to_text.py @@ -16,7 +16,7 @@ import unittest from transformers import MODEL_FOR_VISION_2_SEQ_MAPPING, TF_MODEL_FOR_VISION_2_SEQ_MAPPING, is_vision_available from transformers.pipelines import pipeline -from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, require_vision, slow +from transformers.testing_utils import require_tf, require_torch, require_vision, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -31,7 +31,6 @@ else: pass -@is_pipeline_test @require_vision class ImageToTextPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_VISION_2_SEQ_MAPPING diff --git a/tests/pipelines/test_pipelines_object_detection.py b/tests/pipelines/test_pipelines_object_detection.py index b1d43f8a879..ebefcaab61f 100644 --- a/tests/pipelines/test_pipelines_object_detection.py +++ b/tests/pipelines/test_pipelines_object_detection.py @@ -22,15 +22,7 @@ from transformers import ( is_vision_available, pipeline, ) -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_timm, - require_torch, - require_vision, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_timm, require_torch, require_vision, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -48,7 +40,6 @@ else: @require_vision @require_timm @require_torch -@is_pipeline_test class ObjectDetectionPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_OBJECT_DETECTION_MAPPING diff --git a/tests/pipelines/test_pipelines_question_answering.py b/tests/pipelines/test_pipelines_question_answering.py index 001254aa94b..fb0f6cc8817 100644 --- a/tests/pipelines/test_pipelines_question_answering.py +++ b/tests/pipelines/test_pipelines_question_answering.py @@ -22,12 +22,11 @@ from transformers import ( ) from transformers.data.processors.squad import SquadExample from transformers.pipelines import QuestionAnsweringArgumentHandler, pipeline -from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow +from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_torch_or_tf, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test class QAPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_QUESTION_ANSWERING_MAPPING tf_model_mapping = TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING @@ -345,7 +344,7 @@ between them. It's straightforward to train your models with one before loading self.assertEqual(nested_simplify(outputs), {"score": 0.979, "start": 27, "end": 32, "answer": "Paris"}) -@is_pipeline_test +@require_torch_or_tf class QuestionAnsweringArgumentHandlerTests(unittest.TestCase): def test_argument_handler(self): qa = QuestionAnsweringArgumentHandler() diff --git a/tests/pipelines/test_pipelines_summarization.py b/tests/pipelines/test_pipelines_summarization.py index d797383811c..50e8315a5f1 100644 --- a/tests/pipelines/test_pipelines_summarization.py +++ b/tests/pipelines/test_pipelines_summarization.py @@ -23,7 +23,7 @@ from transformers import ( T5Config, pipeline, ) -from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow, torch_device +from transformers.testing_utils import require_tf, require_torch, slow, torch_device from transformers.tokenization_utils import TruncationStrategy from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -32,7 +32,6 @@ from .test_pipelines_common import ANY, PipelineTestCaseMeta DEFAULT_DEVICE_NUM = -1 if torch_device == "cpu" else 0 -@is_pipeline_test class SummarizationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING diff --git a/tests/pipelines/test_pipelines_table_question_answering.py b/tests/pipelines/test_pipelines_table_question_answering.py index ba7fdaa75c5..089186a4672 100644 --- a/tests/pipelines/test_pipelines_table_question_answering.py +++ b/tests/pipelines/test_pipelines_table_question_answering.py @@ -23,7 +23,6 @@ from transformers import ( pipeline, ) from transformers.testing_utils import ( - is_pipeline_test, require_pandas, require_tensorflow_probability, require_tf, @@ -35,7 +34,6 @@ from transformers.testing_utils import ( from .test_pipelines_common import PipelineTestCaseMeta -@is_pipeline_test class TQAPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): # Putting it there for consistency, but TQA do not have fast tokenizer # which are needed to generate automatic tests diff --git a/tests/pipelines/test_pipelines_text2text_generation.py b/tests/pipelines/test_pipelines_text2text_generation.py index 4490c571622..772190fb63e 100644 --- a/tests/pipelines/test_pipelines_text2text_generation.py +++ b/tests/pipelines/test_pipelines_text2text_generation.py @@ -20,7 +20,7 @@ from transformers import ( Text2TextGenerationPipeline, pipeline, ) -from transformers.testing_utils import is_pipeline_test, require_tf, require_torch +from transformers.testing_utils import require_tf, require_torch from transformers.utils import is_torch_available from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -30,7 +30,6 @@ if is_torch_available(): import torch -@is_pipeline_test class Text2TextGenerationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING diff --git a/tests/pipelines/test_pipelines_text_classification.py b/tests/pipelines/test_pipelines_text_classification.py index 6bbc84989a2..80e8e2559f1 100644 --- a/tests/pipelines/test_pipelines_text_classification.py +++ b/tests/pipelines/test_pipelines_text_classification.py @@ -20,12 +20,11 @@ from transformers import ( TextClassificationPipeline, pipeline, ) -from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow +from transformers.testing_utils import nested_simplify, require_tf, require_torch, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test class TextClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING diff --git a/tests/pipelines/test_pipelines_text_generation.py b/tests/pipelines/test_pipelines_text_generation.py index ac6d122559e..ca0e1011584 100644 --- a/tests/pipelines/test_pipelines_text_generation.py +++ b/tests/pipelines/test_pipelines_text_generation.py @@ -16,17 +16,17 @@ import unittest from transformers import MODEL_FOR_CAUSAL_LM_MAPPING, TF_MODEL_FOR_CAUSAL_LM_MAPPING, TextGenerationPipeline, pipeline from transformers.testing_utils import ( - is_pipeline_test, require_accelerate, require_tf, require_torch, require_torch_gpu, + require_torch_or_tf, ) from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test +@require_torch_or_tf class TextGenerationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_CAUSAL_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_CAUSAL_LM_MAPPING diff --git a/tests/pipelines/test_pipelines_token_classification.py b/tests/pipelines/test_pipelines_token_classification.py index bc4eaef0625..ff86e7106ae 100644 --- a/tests/pipelines/test_pipelines_token_classification.py +++ b/tests/pipelines/test_pipelines_token_classification.py @@ -25,14 +25,7 @@ from transformers import ( pipeline, ) from transformers.pipelines import AggregationStrategy, TokenClassificationArgumentHandler -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_torch, - require_torch_gpu, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_torch_gpu, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -40,7 +33,6 @@ from .test_pipelines_common import ANY, PipelineTestCaseMeta VALID_INPUTS = ["A simple string", ["list of strings", "A simple string that is quite a bit longer"]] -@is_pipeline_test class TokenClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING @@ -770,7 +762,6 @@ class TokenClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTest ) -@is_pipeline_test class TokenClassificationArgumentHandlerTestCase(unittest.TestCase): def setUp(self): self.args_parser = TokenClassificationArgumentHandler() diff --git a/tests/pipelines/test_pipelines_translation.py b/tests/pipelines/test_pipelines_translation.py index 3c5999f36e6..d8de606f698 100644 --- a/tests/pipelines/test_pipelines_translation.py +++ b/tests/pipelines/test_pipelines_translation.py @@ -25,12 +25,11 @@ from transformers import ( TranslationPipeline, pipeline, ) -from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow +from transformers.testing_utils import require_tf, require_torch, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test class TranslationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING @@ -118,7 +117,6 @@ class TranslationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta ) -@is_pipeline_test class TranslationNewFormatPipelineTests(unittest.TestCase): @require_torch @slow diff --git a/tests/pipelines/test_pipelines_visual_question_answering.py b/tests/pipelines/test_pipelines_visual_question_answering.py index d3315681f47..bf3a532b10c 100644 --- a/tests/pipelines/test_pipelines_visual_question_answering.py +++ b/tests/pipelines/test_pipelines_visual_question_answering.py @@ -16,14 +16,7 @@ import unittest from transformers import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, is_vision_available from transformers.pipelines import pipeline -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_torch, - require_vision, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_vision, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -38,7 +31,6 @@ else: pass -@is_pipeline_test @require_torch @require_vision class VisualQuestionAnsweringPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): diff --git a/tests/pipelines/test_pipelines_zero_shot.py b/tests/pipelines/test_pipelines_zero_shot.py index af98ac02017..5c78db1aa96 100644 --- a/tests/pipelines/test_pipelines_zero_shot.py +++ b/tests/pipelines/test_pipelines_zero_shot.py @@ -21,12 +21,11 @@ from transformers import ( ZeroShotClassificationPipeline, pipeline, ) -from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow +from transformers.testing_utils import nested_simplify, require_tf, require_torch, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta -@is_pipeline_test class ZeroShotClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING diff --git a/tests/pipelines/test_pipelines_zero_shot_image_classification.py b/tests/pipelines/test_pipelines_zero_shot_image_classification.py index a5aef5c35bd..d0396f4e9a9 100644 --- a/tests/pipelines/test_pipelines_zero_shot_image_classification.py +++ b/tests/pipelines/test_pipelines_zero_shot_image_classification.py @@ -16,14 +16,7 @@ import unittest from transformers import is_vision_available from transformers.pipelines import pipeline -from transformers.testing_utils import ( - is_pipeline_test, - nested_simplify, - require_tf, - require_torch, - require_vision, - slow, -) +from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_vision, slow from .test_pipelines_common import ANY, PipelineTestCaseMeta @@ -39,7 +32,6 @@ else: @require_vision -@is_pipeline_test class ZeroShotImageClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): # Deactivating auto tests since we don't have a good MODEL_FOR_XX mapping, # and only CLIP would be there for now. diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 0af1a8ad8eb..f9e0f86af5b 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -619,6 +619,25 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None, j json.dump(test_map, fp, ensure_ascii=False) +def filter_pipeline_tests(output_file): + if not os.path.isfile(output_file): + print("No test file found.") + return + with open(output_file, "r", encoding="utf-8") as f: + test_files = f.read().split(" ") + + if len(test_files) == 0: + print("No tests to filter.") + return + if test_files == ["tests"]: + test_files = [os.path.join("tests", f) for f in os.listdir("tests") if f not in ["__init__.py", "pipelines"]] + else: + test_files = [f for f in test_files if not f.startswith(os.path.join("tests", "pipelines"))] + + with open(output_file, "w", encoding="utf-8") as f: + f.write(" ".join(test_files)) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( @@ -645,6 +664,11 @@ if __name__ == "__main__": default=["tests"], help="Only keep the test files matching one of those filters.", ) + parser.add_argument( + "--filter_pipeline_tests", + action="store_true", + help="Will filter the pipeline tests outside of the generated list of tests.", + ) parser.add_argument( "--print_dependencies_of", type=str, @@ -656,6 +680,8 @@ if __name__ == "__main__": print_tree_deps_of(args.print_dependencies_of) elif args.sanity_check: sanity_check() + elif args.filter_pipeline_tests: + filter_pipeline_tests(args.output_file) else: repo = Repo(PATH_TO_TRANFORMERS)