From 2099287a59a5defa80ca8566c61ee5ca06dd55e0 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Thu, 3 Apr 2025 09:57:45 +0100 Subject: [PATCH] [CI] lazy loading external datasets (#37218) --- .../test_pipelines_audio_classification.py | 12 ++++++++--- .../test_pipelines_depth_estimation.py | 18 ++++++++++------- .../test_pipelines_image_classification.py | 18 ++++++++++------- .../test_pipelines_image_segmentation.py | 18 ++++++++++------- .../test_pipelines_object_detection.py | 20 +++++++++++-------- .../test_pipelines_video_classification.py | 14 ++++++++----- 6 files changed, 63 insertions(+), 37 deletions(-) diff --git a/tests/pipelines/test_pipelines_audio_classification.py b/tests/pipelines/test_pipelines_audio_classification.py index 08a361add7f..cea317d0eb0 100644 --- a/tests/pipelines/test_pipelines_audio_classification.py +++ b/tests/pipelines/test_pipelines_audio_classification.py @@ -25,7 +25,6 @@ from transformers import ( ) from transformers.pipelines import AudioClassificationPipeline, pipeline from transformers.testing_utils import ( - _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -46,9 +45,15 @@ if is_torch_available(): class AudioClassificationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING + _dataset = None - if _run_pipeline_tests: - _dataset = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + @classmethod + def _load_dataset(cls): + # Lazy loading of the dataset. Because it is a class method, it will only be loaded once per pytest process. + if cls._dataset is None: + cls._dataset = datasets.load_dataset( + "hf-internal-testing/librispeech_asr_dummy", "clean", split="validation" + ) def get_test_pipeline( self, @@ -99,6 +104,7 @@ class AudioClassificationPipelineTests(unittest.TestCase): @require_torchaudio def run_torchaudio(self, audio_classifier): + self._load_dataset() # test with a local file audio = self._dataset[0]["audio"]["array"] output = audio_classifier(audio) diff --git a/tests/pipelines/test_pipelines_depth_estimation.py b/tests/pipelines/test_pipelines_depth_estimation.py index 73176a43837..a5dcb3ef249 100644 --- a/tests/pipelines/test_pipelines_depth_estimation.py +++ b/tests/pipelines/test_pipelines_depth_estimation.py @@ -21,7 +21,6 @@ from huggingface_hub.utils import insecure_hashlib from transformers import MODEL_FOR_DEPTH_ESTIMATION_MAPPING, is_torch_available, is_vision_available from transformers.pipelines import DepthEstimationPipeline, pipeline from transformers.testing_utils import ( - _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -59,13 +58,17 @@ def hashimage(image: Image) -> str: @require_torch class DepthEstimationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_DEPTH_ESTIMATION_MAPPING + _dataset = None - if _run_pipeline_tests: - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - _dataset = datasets.load_dataset( - "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" - ) + @classmethod + def _load_dataset(cls): + # Lazy loading of the dataset. Because it is a class method, it will only be loaded once per pytest process. + if cls._dataset is None: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + cls._dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) def get_test_pipeline( self, @@ -90,6 +93,7 @@ class DepthEstimationPipelineTests(unittest.TestCase): ] def run_pipeline_test(self, depth_estimator, examples): + self._load_dataset() outputs = depth_estimator("./tests/fixtures/tests_samples/COCO/000000039769.png") self.assertEqual({"predicted_depth": ANY(torch.Tensor), "depth": ANY(Image.Image)}, outputs) diff --git a/tests/pipelines/test_pipelines_image_classification.py b/tests/pipelines/test_pipelines_image_classification.py index e5dce2a03dc..17aec8bf35b 100644 --- a/tests/pipelines/test_pipelines_image_classification.py +++ b/tests/pipelines/test_pipelines_image_classification.py @@ -26,7 +26,6 @@ from transformers import ( ) from transformers.pipelines import ImageClassificationPipeline, pipeline from transformers.testing_utils import ( - _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -59,13 +58,17 @@ else: class ImageClassificationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING + _dataset = None - if _run_pipeline_tests: - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - _dataset = datasets.load_dataset( - "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" - ) + @classmethod + def _load_dataset(cls): + # Lazy loading of the dataset. Because it is a class method, it will only be loaded once per pytest process. + if cls._dataset is None: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + cls._dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) def get_test_pipeline( self, @@ -92,6 +95,7 @@ class ImageClassificationPipelineTests(unittest.TestCase): return image_classifier, examples def run_pipeline_test(self, image_classifier, examples): + self._load_dataset() outputs = image_classifier("./tests/fixtures/tests_samples/COCO/000000039769.png") self.assertEqual( diff --git a/tests/pipelines/test_pipelines_image_segmentation.py b/tests/pipelines/test_pipelines_image_segmentation.py index af251230a6e..62e56556dda 100644 --- a/tests/pipelines/test_pipelines_image_segmentation.py +++ b/tests/pipelines/test_pipelines_image_segmentation.py @@ -37,7 +37,6 @@ from transformers import ( pipeline, ) from transformers.testing_utils import ( - _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -89,13 +88,17 @@ class ImageSegmentationPipelineTests(unittest.TestCase): + (MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING.items() if MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING else []) + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else []) ) + _dataset = None - if _run_pipeline_tests: - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - _dataset = datasets.load_dataset( - "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" - ) + @classmethod + def _load_dataset(cls): + # Lazy loading of the dataset. Because it is a class method, it will only be loaded once per pytest process. + if cls._dataset is None: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + cls._dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) def get_test_pipeline( self, @@ -120,6 +123,7 @@ class ImageSegmentationPipelineTests(unittest.TestCase): ] def run_pipeline_test(self, image_segmenter, examples): + self._load_dataset() outputs = image_segmenter( "./tests/fixtures/tests_samples/COCO/000000039769.png", threshold=0.0, diff --git a/tests/pipelines/test_pipelines_object_detection.py b/tests/pipelines/test_pipelines_object_detection.py index c6fa2c1ec1d..fcc50ca5b2b 100644 --- a/tests/pipelines/test_pipelines_object_detection.py +++ b/tests/pipelines/test_pipelines_object_detection.py @@ -25,8 +25,7 @@ from transformers import ( is_vision_available, pipeline, ) -from transformers.testing_utils import ( # - _run_pipeline_tests, +from transformers.testing_utils import ( compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -57,13 +56,17 @@ else: @require_torch class ObjectDetectionPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_OBJECT_DETECTION_MAPPING + _dataset = None - if _run_pipeline_tests: - # we use revision="refs/pr/1" until the PR is merged - # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 - _dataset = datasets.load_dataset( - "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" - ) + @classmethod + def _load_dataset(cls): + # Lazy loading of the dataset. Because it is a class method, it will only be loaded once per pytest process. + if cls._dataset is None: + # we use revision="refs/pr/1" until the PR is merged + # https://hf.co/datasets/hf-internal-testing/fixtures_image_utils/discussions/1 + cls._dataset = datasets.load_dataset( + "hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1" + ) def get_test_pipeline( self, @@ -85,6 +88,7 @@ class ObjectDetectionPipelineTests(unittest.TestCase): return object_detector, ["./tests/fixtures/tests_samples/COCO/000000039769.png"] def run_pipeline_test(self, object_detector, examples): + self._load_dataset() outputs = object_detector("./tests/fixtures/tests_samples/COCO/000000039769.png", threshold=0.0) self.assertGreater(len(outputs), 0) diff --git a/tests/pipelines/test_pipelines_video_classification.py b/tests/pipelines/test_pipelines_video_classification.py index 9565aa631ee..6dbe324ed3d 100644 --- a/tests/pipelines/test_pipelines_video_classification.py +++ b/tests/pipelines/test_pipelines_video_classification.py @@ -19,7 +19,6 @@ from huggingface_hub import VideoClassificationOutputElement, hf_hub_download from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VideoMAEFeatureExtractor from transformers.pipelines import VideoClassificationPipeline, pipeline from transformers.testing_utils import ( - _run_pipeline_tests, compare_pipeline_output_to_hub_spec, is_pipeline_test, nested_simplify, @@ -39,11 +38,15 @@ from .test_pipelines_common import ANY @require_av class VideoClassificationPipelineTests(unittest.TestCase): model_mapping = MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING + example_video_filepath = None - if _run_pipeline_tests: - example_video_filepath = hf_hub_download( - repo_id="nateraw/video-demo", filename="archery.mp4", repo_type="dataset" - ) + @classmethod + def _load_dataset(cls): + # Lazy loading of the dataset. Because it is a class method, it will only be loaded once per pytest process. + if cls.example_video_filepath is None: + cls.example_video_filepath = hf_hub_download( + repo_id="nateraw/video-demo", filename="archery.mp4", repo_type="dataset" + ) def get_test_pipeline( self, @@ -54,6 +57,7 @@ class VideoClassificationPipelineTests(unittest.TestCase): processor=None, torch_dtype="float32", ): + self._load_dataset() video_classifier = VideoClassificationPipeline( model=model, tokenizer=tokenizer,