mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-02 04:10:06 +06:00
b3b7789cbc
Some checks are pending
Self-hosted runner (benchmark) / Benchmark (aws-g5-4xlarge-cache) (push) Waiting to run
Build documentation / build (push) Waiting to run
New model PR merged notification / Notify new model (push) Waiting to run
Slow tests on important models (on Push - A10) / Get all modified files (push) Waiting to run
Slow tests on important models (on Push - A10) / Slow & FA2 tests (push) Blocked by required conditions
Self-hosted runner (push-caller) / Check if setup was changed (push) Waiting to run
Self-hosted runner (push-caller) / build-docker-containers (push) Blocked by required conditions
Self-hosted runner (push-caller) / Trigger Push CI (push) Blocked by required conditions
Secret Leaks / trufflehog (push) Waiting to run
Update Transformers metadata / build_and_package (push) Waiting to run
* image-classification * depth-estimation * zero-shot-image-classification * image-feature-extraction * image-segmentation * mask-generation * object-detection * zero-shot-object-detection * image-to-image * image-text-to-text * image-to-text * text-classification * text-generation * text-to-audio * text2text_generation * fixup * token-classification * document-qa * video-classification * audio-classification * automatic-speech-recognition * feature-extraction * fill-mask * zero-shot-audio-classification * Add pipeline function typing * Add code generator and checker for pipeline types * Add to makefile * style * Add to CI * Style
228 lines
9.5 KiB
YAML
228 lines
9.5 KiB
YAML
version: 2.1
|
|
setup: true
|
|
orbs:
|
|
continuation: circleci/continuation@0.1.0
|
|
|
|
parameters:
|
|
nightly:
|
|
type: boolean
|
|
default: false
|
|
GHA_Actor:
|
|
type: string
|
|
default: ""
|
|
GHA_Action:
|
|
type: string
|
|
default: ""
|
|
GHA_Event:
|
|
type: string
|
|
default: ""
|
|
GHA_Meta:
|
|
type: string
|
|
default: ""
|
|
|
|
jobs:
|
|
# Ensure running with CircleCI/huggingface
|
|
check_circleci_user:
|
|
docker:
|
|
- image: python:3.10-slim
|
|
resource_class: small
|
|
parallelism: 1
|
|
steps:
|
|
- run: echo $CIRCLE_PROJECT_USERNAME
|
|
- run: |
|
|
if [ "$CIRCLE_PROJECT_USERNAME" = "huggingface" ]; then
|
|
exit 0
|
|
else
|
|
echo "The CI is running under $CIRCLE_PROJECT_USERNAME personal account. Please follow https://support.circleci.com/hc/en-us/articles/360008097173-Troubleshooting-why-pull-requests-are-not-triggering-jobs-on-my-organization- to fix it."; exit -1
|
|
fi
|
|
# Fetch the tests to run
|
|
fetch_tests:
|
|
working_directory: ~/transformers
|
|
docker:
|
|
- image: huggingface/transformers-quality
|
|
parallelism: 1
|
|
steps:
|
|
- checkout
|
|
- run: uv pip install -U -e .
|
|
- run: echo 'export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)"' >> "$BASH_ENV" && source "$BASH_ENV"
|
|
- run: mkdir -p test_preparation
|
|
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
|
|
- run: python utils/tests_fetcher.py --filter_tests
|
|
- run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
|
|
- run: |
|
|
if [ ! -s test_preparation/generated_config.yml ]; then
|
|
echo "No tests to run, exiting early!"
|
|
circleci-agent step halt
|
|
fi
|
|
|
|
- store_artifacts:
|
|
path: test_preparation
|
|
|
|
- run:
|
|
name: "Retrieve Artifact Paths"
|
|
# [reference] https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts
|
|
# `CIRCLE_TOKEN` is defined as an environment variables set within a context, see `https://circleci.com/docs/contexts/`
|
|
command: |
|
|
project_slug="gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
|
|
job_number=${CIRCLE_BUILD_NUM}
|
|
url="https://circleci.com/api/v2/project/${project_slug}/${job_number}/artifacts"
|
|
curl -o test_preparation/artifacts.json ${url} --header "Circle-Token: $CIRCLE_TOKEN"
|
|
- run:
|
|
name: "Prepare pipeline parameters"
|
|
command: |
|
|
python utils/process_test_artifacts.py
|
|
|
|
# To avoid too long generated_config.yaml on the continuation orb, we pass the links to the artifacts as parameters.
|
|
# Otherwise the list of tests was just too big. Explicit is good but for that it was a limitation.
|
|
# We used:
|
|
|
|
# https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts : to get the job artifacts
|
|
# We could not pass a nested dict, which is why we create the test_file_... parameters for every single job
|
|
|
|
- store_artifacts:
|
|
path: test_preparation/transformed_artifacts.json
|
|
- store_artifacts:
|
|
path: test_preparation/artifacts.json
|
|
- continuation/continue:
|
|
parameters: test_preparation/transformed_artifacts.json
|
|
configuration_path: test_preparation/generated_config.yml
|
|
|
|
# To run all tests for the nightly build
|
|
fetch_all_tests:
|
|
working_directory: ~/transformers
|
|
docker:
|
|
- image: huggingface/transformers-quality
|
|
parallelism: 1
|
|
steps:
|
|
- checkout
|
|
- run: uv pip install -U -e .
|
|
- run: echo 'export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)"' >> "$BASH_ENV" && source "$BASH_ENV"
|
|
- run: mkdir -p test_preparation
|
|
- run: python utils/tests_fetcher.py --fetch_all | tee tests_fetched_summary.txt
|
|
- run: python utils/tests_fetcher.py --filter_tests
|
|
- run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
|
|
- run: |
|
|
if [ ! -s test_preparation/generated_config.yml ]; then
|
|
echo "No tests to run, exiting early!"
|
|
circleci-agent step halt
|
|
fi
|
|
|
|
- store_artifacts:
|
|
path: test_preparation
|
|
|
|
- run:
|
|
name: "Retrieve Artifact Paths"
|
|
command: |
|
|
project_slug="gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}"
|
|
job_number=${CIRCLE_BUILD_NUM}
|
|
url="https://circleci.com/api/v2/project/${project_slug}/${job_number}/artifacts"
|
|
curl -o test_preparation/artifacts.json ${url}
|
|
- run:
|
|
name: "Prepare pipeline parameters"
|
|
command: |
|
|
python utils/process_test_artifacts.py
|
|
|
|
# To avoid too long generated_config.yaml on the continuation orb, we pass the links to the artifacts as parameters.
|
|
# Otherwise the list of tests was just too big. Explicit is good but for that it was a limitation.
|
|
# We used:
|
|
|
|
# https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts : to get the job artifacts
|
|
# We could not pass a nested dict, which is why we create the test_file_... parameters for every single job
|
|
|
|
- store_artifacts:
|
|
path: test_preparation/transformed_artifacts.json
|
|
- store_artifacts:
|
|
path: test_preparation/artifacts.json
|
|
- continuation/continue:
|
|
parameters: test_preparation/transformed_artifacts.json
|
|
configuration_path: test_preparation/generated_config.yml
|
|
|
|
check_code_quality:
|
|
working_directory: ~/transformers
|
|
docker:
|
|
- image: huggingface/transformers-quality
|
|
resource_class: large
|
|
environment:
|
|
TRANSFORMERS_IS_CI: yes
|
|
PYTEST_TIMEOUT: 120
|
|
parallelism: 1
|
|
steps:
|
|
- checkout
|
|
- run: uv pip install -e ".[quality]"
|
|
- run:
|
|
name: Show installed libraries and their versions
|
|
command: pip freeze | tee installed.txt
|
|
- store_artifacts:
|
|
path: ~/transformers/installed.txt
|
|
- run: python -c "from transformers import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
|
- run: ruff check examples tests src utils
|
|
- run: ruff format examples tests src utils --check
|
|
- run: python utils/custom_init_isort.py --check_only
|
|
- run: python utils/sort_auto_mappings.py --check_only
|
|
- run: python utils/check_doc_toc.py
|
|
- run: python utils/check_docstrings.py --check_all
|
|
|
|
check_repository_consistency:
|
|
working_directory: ~/transformers
|
|
docker:
|
|
- image: huggingface/transformers-consistency
|
|
resource_class: large
|
|
environment:
|
|
TRANSFORMERS_IS_CI: yes
|
|
PYTEST_TIMEOUT: 120
|
|
parallelism: 1
|
|
steps:
|
|
- checkout
|
|
- run: uv pip install -e ".[quality]"
|
|
- run:
|
|
name: Show installed libraries and their versions
|
|
command: pip freeze | tee installed.txt
|
|
- store_artifacts:
|
|
path: ~/transformers/installed.txt
|
|
- run: python utils/check_copies.py
|
|
- run: python utils/check_modular_conversion.py
|
|
- run: python utils/check_dummies.py
|
|
- run: python utils/check_repo.py
|
|
- run: python utils/check_inits.py
|
|
- run: python utils/check_pipeline_typing.py
|
|
- run: python utils/check_config_docstrings.py
|
|
- run: python utils/check_config_attributes.py
|
|
- run: python utils/check_doctest_list.py
|
|
- run: make deps_table_check_updated
|
|
- run: python utils/update_metadata.py --check-only
|
|
- run: python utils/check_docstrings.py
|
|
|
|
workflows:
|
|
version: 2
|
|
setup_and_quality:
|
|
when:
|
|
and:
|
|
- equal: [<<pipeline.project.git_url>>, https://github.com/huggingface/transformers]
|
|
- not: <<pipeline.parameters.nightly>>
|
|
jobs:
|
|
- check_circleci_user
|
|
- check_code_quality
|
|
- check_repository_consistency
|
|
- fetch_tests
|
|
|
|
setup_and_quality_2:
|
|
when:
|
|
not:
|
|
equal: [<<pipeline.project.git_url>>, https://github.com/huggingface/transformers]
|
|
jobs:
|
|
- check_circleci_user
|
|
- check_code_quality
|
|
- check_repository_consistency
|
|
- fetch_tests:
|
|
# [reference] https://circleci.com/docs/contexts/
|
|
context:
|
|
- TRANSFORMERS_CONTEXT
|
|
|
|
nightly:
|
|
when: <<pipeline.parameters.nightly>>
|
|
jobs:
|
|
- check_circleci_user
|
|
- check_code_quality
|
|
- check_repository_consistency
|
|
- fetch_all_tests
|