riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-03 21:00:08 +06:00
Code Issues Actions 25 Packages Projects Releases Wiki Activity

consistent job / pytest report / artifact name correspondence (#30392)

Browse Source 
* better names

* run better names

* update

* update

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar 2024-04-24 22:32:42 +02:00 committed by GitHub
parent 6ad9c8f743
commit fbb41cd420
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 145 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.github/workflows/model_jobs.yml vendored
View File

@ -28,7 +28,7 @@ env:
CUDA_VISIBLE_DEVICES: 0,1 CUDA_VISIBLE_DEVICES: 0,1
jobs: jobs:
model_job: run_models_gpu:
name: " " name: " "
strategy: strategy:
fail-fast: false fail-fast: false
@ -80,23 +80,23 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /transformers working-directory: /transformers
run: python3 -m pytest -rs -v --make-reports=${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }} tests/${{ matrix.folders }} run: python3 -m pytest -rs -v --make-reports=${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt run: cat /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: Run test - name: Run test
shell: bash shell: bash
run: | run: |
mkdir -p /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }} mkdir -p /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
echo "hello" > /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}/hello.txt echo "hello" > /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
echo "${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }}" echo "${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
- name: "Test suite reports artifacts: ${{ inputs.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports" - name: "Test suite reports artifacts: ${{ inputs.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ inputs.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports name: ${{ inputs.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ inputs.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

18
.github/workflows/self-new-model-pr-caller.yml vendored
View File

@ -36,7 +36,7 @@ jobs:
python -m pip install GitPython python -m pip install GitPython
echo "new_model=$(python utils/check_if_new_model_added.py | tail -n 1)" >> $GITHUB_OUTPUT echo "new_model=$(python utils/check_if_new_model_added.py | tail -n 1)" >> $GITHUB_OUTPUT
run_new_model_tests: run_models_gpu:
name: Run all tests for the new model name: Run all tests for the new model
# Triggered if it is a new model PR and the required label is added # Triggered if it is a new model PR and the required label is added
if: ${{ needs.check_for_new_model.outputs.new_model != '' && contains(github.event.pull_request.labels.*.name, 'single-model-run-slow') }} if: ${{ needs.check_for_new_model.outputs.new_model != '' && contains(github.event.pull_request.labels.*.name, 'single-model-run-slow') }}
@ -90,23 +90,23 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /transformers working-directory: /transformers
run: python3 -m pytest -v -rs --make-reports=${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} tests/${{ matrix.folders }} run: python3 -m pytest -v -rs --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: Make sure report directory exists - name: Make sure report directory exists
shell: bash shell: bash
run: | run: |
mkdir -p /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} mkdir -p /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
echo "hello" > /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/hello.txt echo "hello" > /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
echo "${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}" echo "${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

16
.github/workflows/self-nightly-scheduled.yml vendored
View File

@ -2,7 +2,7 @@ name: Self-hosted runner (nightly-ci)
# Note that each job's dependencies go into a corresponding docker file. # Note that each job's dependencies go into a corresponding docker file.
# #
# For example for `run_all_tests_torch_cuda_extensions_gpu` the docker image is # For example for `run_torch_cuda_extensions_gpu` the docker image is
# `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at # `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at
# `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile` # `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile`
@ -183,7 +183,7 @@ jobs:
name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_nightly name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_nightly
path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}
run_all_tests_torch_cuda_extensions_gpu: run_torch_cuda_extensions_gpu:
name: Torch CUDA extension tests name: Torch CUDA extension tests
strategy: strategy:
fail-fast: false fail-fast: false
@ -231,19 +231,19 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /workspace/transformers working-directory: /workspace/transformers
run: | run: |
python -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended python -m pytest -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_nightly" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports_postfix_nightly"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_nightly name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports_postfix_nightly
path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu path: /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
send_results: send_results:
name: Send results to webhook name: Send results to webhook
@ -253,7 +253,7 @@ jobs:
setup, setup,
run_tests_single_gpu, run_tests_single_gpu,
run_tests_multi_gpu, run_tests_multi_gpu,
run_all_tests_torch_cuda_extensions_gpu run_torch_cuda_extensions_gpu
] ]
steps: steps:
- name: Preliminary job status - name: Preliminary job status

16
.github/workflows/self-past.yml vendored
View File

@ -2,7 +2,7 @@ name: Self-hosted runner (past-ci)
# Note that each job's dependencies go into a corresponding docker file. # Note that each job's dependencies go into a corresponding docker file.
# #
# For example for `run_all_tests_torch_cuda_extensions_gpu` the docker image is # For example for `run_torch_cuda_extensions_gpu` the docker image is
# `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at # `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at
# `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile` # `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile`
@ -228,7 +228,7 @@ jobs:
name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }} name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}
path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}
run_all_tests_torch_cuda_extensions_gpu: run_torch_cuda_extensions_gpu:
name: Torch CUDA extension tests name: Torch CUDA extension tests
if: inputs.framework == 'pytorch' if: inputs.framework == 'pytorch'
strategy: strategy:
@ -286,19 +286,19 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /transformers working-directory: /transformers
run: | run: |
python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }} name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }}
path: /transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
send_results: send_results:
name: Send results to webhook name: Send results to webhook
@ -308,7 +308,7 @@ jobs:
setup, setup,
run_tests_single_gpu, run_tests_single_gpu,
run_tests_multi_gpu, run_tests_multi_gpu,
run_all_tests_torch_cuda_extensions_gpu run_torch_cuda_extensions_gpu
] ]
steps: steps:
- name: Preliminary job status - name: Preliminary job status

14
.github/workflows/self-push-amd.yml vendored
View File

@ -145,7 +145,7 @@ jobs:
echo "matrix=$keys" >> $GITHUB_OUTPUT echo "matrix=$keys" >> $GITHUB_OUTPUT
echo "test_map=$test_map" >> $GITHUB_OUTPUT echo "test_map=$test_map" >> $GITHUB_OUTPUT
run_tests_amdgpu: run_models_gpu:
name: Model tests name: Model tests
needs: setup_gpu needs: setup_gpu
# `dummy` means there is no test to run # `dummy` means there is no test to run
@ -230,19 +230,19 @@ jobs:
- name: Run all non-slow selected tests on GPU - name: Run all non-slow selected tests on GPU
working-directory: /transformers working-directory: /transformers
run: | run: |
python3 -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} ${{ fromJson(needs.setup_gpu.outputs.test_map)[matrix.folders] }} python3 -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports ${{ fromJson(needs.setup_gpu.outputs.test_map)[matrix.folders] }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
send_results: send_results:
name: Send results to webhook name: Send results to webhook
@ -252,7 +252,7 @@ jobs:
check_runner_status, check_runner_status,
check_runners, check_runners,
setup_gpu, setup_gpu,
run_tests_amdgpu, run_models_gpu,
# run_tests_torch_cuda_extensions_single_gpu, # run_tests_torch_cuda_extensions_single_gpu,
# run_tests_torch_cuda_extensions_multi_gpu # run_tests_torch_cuda_extensions_multi_gpu
] ]

20
.github/workflows/self-push.yml vendored
View File

@ -385,19 +385,19 @@ jobs:
working-directory: /workspace/transformers working-directory: /workspace/transformers
# TODO: Here we pass all tests in the 2 folders for simplicity. It's better to pass only the identified tests. # TODO: Here we pass all tests in the 2 folders for simplicity. It's better to pass only the identified tests.
run: | run: |
python -m pytest -n 1 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended python -m pytest -n 1 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu path: /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
run_tests_torch_cuda_extensions_multi_gpu: run_tests_torch_cuda_extensions_multi_gpu:
name: Torch CUDA extension tests name: Torch CUDA extension tests
@ -475,19 +475,19 @@ jobs:
working-directory: /workspace/transformers working-directory: /workspace/transformers
# TODO: Here we pass all tests in the 2 folders for simplicity. It's better to pass only the identified tests. # TODO: Here we pass all tests in the 2 folders for simplicity. It's better to pass only the identified tests.
run: | run: |
python -m pytest -n 1 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended python -m pytest -n 1 --dist=loadfile -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu path: /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
send_results: send_results:
name: Send results to webhook name: Send results to webhook

68
.github/workflows/self-scheduled-amd.yml vendored
View File

@ -108,7 +108,7 @@ jobs:
run: | run: |
python3 utils/print_env.py python3 utils/print_env.py
run_tests_single_gpu: run_models_gpu_single_gpu:
name: Single GPU tests name: Single GPU tests
strategy: strategy:
max-parallel: 1 # For now, not to parallelize. Can change later if it works well. max-parallel: 1 # For now, not to parallelize. Can change later if it works well.
@ -162,21 +162,21 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /transformers working-directory: /transformers
run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} tests/${{ matrix.folders }} run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
run_tests_multi_gpu: run_models_gpu_multi_gpu:
name: Multi GPU tests name: Multi GPU tests
strategy: strategy:
max-parallel: 1 max-parallel: 1
@ -230,19 +230,19 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /transformers working-directory: /transformers
run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} tests/${{ matrix.folders }} run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }}/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
run_examples_gpu: run_examples_gpu:
name: Examples tests name: Examples tests
@ -287,19 +287,19 @@ jobs:
working-directory: /transformers working-directory: /transformers
run: | run: |
pip install -r examples/pytorch/_tests_requirements.txt pip install -r examples/pytorch/_tests_requirements.txt
python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_examples_gpu examples/pytorch python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_examples_gpu_test_reports examples/pytorch
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_examples_gpu/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_examples_gpu" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_examples_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_examples_gpu name: ${{ matrix.machine_type }}_run_examples_gpu_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_examples_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports
run_pipelines_torch_gpu: run_pipelines_torch_gpu:
name: PyTorch pipelines tests name: PyTorch pipelines tests
@ -343,21 +343,21 @@ jobs:
- name: Run all pipeline tests on GPU - name: Run all pipeline tests on GPU
working-directory: /transformers working-directory: /transformers
run: | run: |
python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_tests_torch_pipeline_gpu tests/pipelines python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports tests/pipelines
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_torch_pipeline_gpu/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_pipeline_gpu" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_pipeline_gpu name: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_torch_pipeline_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports
run_tests_torch_deepspeed_gpu: run_torch_cuda_extensions_gpu:
name: Torch ROCm deepspeed tests name: Torch ROCm deepspeed tests
strategy: strategy:
fail-fast: false fail-fast: false
@ -400,19 +400,19 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /transformers working-directory: /transformers
run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_torch_deepspeed_gpu tests/deepspeed tests/extended run: python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_torch_deepspeed_gpu/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_deepspeed_gpu_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_deepspeed_gpu_test_reports name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_torch_deepspeed_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
run_extract_warnings: run_extract_warnings:
name: Extract warnings in CI artifacts name: Extract warnings in CI artifacts
@ -422,11 +422,11 @@ jobs:
check_runner_status, check_runner_status,
check_runners, check_runners,
setup, setup,
run_tests_single_gpu, run_models_gpu_single_gpu,
run_tests_multi_gpu, run_models_gpu_multi_gpu,
run_examples_gpu, run_examples_gpu,
run_pipelines_torch_gpu, run_pipelines_torch_gpu,
run_tests_torch_deepspeed_gpu run_torch_cuda_extensions_gpu
] ]
steps: steps:
- name: Checkout transformers - name: Checkout transformers
@ -471,11 +471,11 @@ jobs:
check_runner_status, check_runner_status,
check_runners, check_runners,
setup, setup,
run_tests_single_gpu, run_models_gpu_single_gpu,
run_tests_multi_gpu, run_models_gpu_multi_gpu,
run_examples_gpu, run_examples_gpu,
run_pipelines_torch_gpu, run_pipelines_torch_gpu,
run_tests_torch_deepspeed_gpu, run_torch_cuda_extensions_gpu,
run_extract_warnings run_extract_warnings
] ]
steps: steps:

6
.github/workflows/self-scheduled-caller.yml vendored
View File

@ -14,7 +14,7 @@ jobs:
name: Model CI name: Model CI
uses: ./.github/workflows/self-scheduled.yml uses: ./.github/workflows/self-scheduled.yml
with: with:
job: run_tests_gpu job: run_models_gpu
slack_report_channel: "#transformers-ci-daily-models" slack_report_channel: "#transformers-ci-daily-models"
secrets: inherit secrets: inherit
@ -46,7 +46,7 @@ jobs:
name: DeepSpeed CI name: DeepSpeed CI
uses: ./.github/workflows/self-scheduled.yml uses: ./.github/workflows/self-scheduled.yml
with: with:
job: run_all_tests_torch_cuda_extensions_gpu job: run_torch_cuda_extensions_gpu
slack_report_channel: "#transformers-ci-daily-deepspeed" slack_report_channel: "#transformers-ci-daily-deepspeed"
secrets: inherit secrets: inherit
@ -54,6 +54,6 @@ jobs:
name: Quantization CI name: Quantization CI
uses: ./.github/workflows/self-scheduled.yml uses: ./.github/workflows/self-scheduled.yml
with: with:
job: run_tests_quantization_torch_gpu job: run_quantization_torch_gpu
slack_report_channel: "#transformers-ci-daily-quantization" slack_report_channel: "#transformers-ci-daily-quantization"
secrets: inherit secrets: inherit

82
.github/workflows/self-scheduled.yml vendored
View File

@ -2,7 +2,7 @@ name: Self-hosted runner (scheduled)
# Note that each job's dependencies go into a corresponding docker file. # Note that each job's dependencies go into a corresponding docker file.
# #
# For example for `run_all_tests_torch_cuda_extensions_gpu` the docker image is # For example for `run_torch_cuda_extensions_gpu` the docker image is
# `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at # `huggingface/transformers-pytorch-deepspeed-latest-gpu`, which can be found at
# `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile` # `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile`
@ -33,7 +33,7 @@ env:
jobs: jobs:
setup: setup:
if: contains(fromJSON('["run_tests_gpu", "run_tests_quantization_torch_gpu"]'), inputs.job) if: contains(fromJSON('["run_models_gpu", "run_quantization_torch_gpu"]'), inputs.job)
name: Setup name: Setup
strategy: strategy:
matrix: matrix:
@ -64,7 +64,7 @@ jobs:
run: pip freeze run: pip freeze
- id: set-matrix - id: set-matrix
if: ${{ inputs.job == 'run_tests_gpu' }} if: ${{ inputs.job == 'run_models_gpu' }}
name: Identify models to test name: Identify models to test
working-directory: /transformers/tests working-directory: /transformers/tests
run: | run: |
@ -72,7 +72,7 @@ jobs:
echo "slice_ids=$(python3 -c 'd = list(range(${{ env.NUM_SLICES }})); print(d)')" >> $GITHUB_OUTPUT echo "slice_ids=$(python3 -c 'd = list(range(${{ env.NUM_SLICES }})); print(d)')" >> $GITHUB_OUTPUT
- id: set-matrix-quantization - id: set-matrix-quantization
if: ${{ inputs.job == 'run_tests_quantization_torch_gpu' }} if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
name: Identify quantization method to test name: Identify quantization method to test
working-directory: /transformers/tests working-directory: /transformers/tests
run: | run: |
@ -82,8 +82,8 @@ jobs:
run: | run: |
nvidia-smi nvidia-smi
run_tests_gpu: run_models_gpu:
if: ${{ inputs.job == 'run_tests_gpu' }} if: ${{ inputs.job == 'run_models_gpu' }}
name: " " name: " "
needs: setup needs: setup
strategy: strategy:
@ -134,19 +134,19 @@ jobs:
- name: Run all pipeline tests on GPU - name: Run all pipeline tests on GPU
working-directory: /transformers working-directory: /transformers
run: | run: |
python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_tests_torch_pipeline_gpu tests/pipelines python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports tests/pipelines
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_torch_pipeline_gpu/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_pipeline_gpu" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_pipeline_gpu name: ${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_torch_pipeline_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_pipelines_torch_gpu_test_reports
run_pipelines_tf_gpu: run_pipelines_tf_gpu:
if: ${{ inputs.job == 'run_pipelines_tf_gpu' }} if: ${{ inputs.job == 'run_pipelines_tf_gpu' }}
@ -185,19 +185,19 @@ jobs:
- name: Run all pipeline tests on GPU - name: Run all pipeline tests on GPU
working-directory: /transformers working-directory: /transformers
run: | run: |
python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_tests_tf_pipeline_gpu tests/pipelines python3 -m pytest -n 1 -v --dist=loadfile --make-reports=${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports tests/pipelines
- name: Failure short reports - name: Failure short reports
if: ${{ always() }} if: ${{ always() }}
run: | run: |
cat /transformers/reports/${{ matrix.machine_type }}_tests_tf_pipeline_gpu/failures_short.txt cat /transformers/reports/${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_tf_pipeline_gpu" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_tf_pipeline_gpu name: ${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_tf_pipeline_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_pipelines_tf_gpu_test_reports
run_examples_gpu: run_examples_gpu:
if: ${{ inputs.job == 'run_examples_gpu' }} if: ${{ inputs.job == 'run_examples_gpu' }}
@ -236,22 +236,22 @@ jobs:
working-directory: /transformers working-directory: /transformers
run: | run: |
pip install -r examples/pytorch/_tests_requirements.txt pip install -r examples/pytorch/_tests_requirements.txt
python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_examples_gpu examples/pytorch python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_examples_gpu_test_reports examples/pytorch
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_examples_gpu/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_examples_gpu" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_examples_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_examples_gpu name: ${{ matrix.machine_type }}_run_examples_gpu_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_examples_gpu path: /transformers/reports/${{ matrix.machine_type }}_run_examples_gpu_test_reports
run_all_tests_torch_cuda_extensions_gpu: run_torch_cuda_extensions_gpu:
if: ${{ inputs.job == 'run_all_tests_torch_cuda_extensions_gpu' }} if: ${{ inputs.job == 'run_torch_cuda_extensions_gpu' }}
name: Torch CUDA extension tests name: Torch CUDA extension tests
strategy: strategy:
fail-fast: false fail-fast: false
@ -296,22 +296,22 @@ jobs:
- name: Run all tests on GPU - name: Run all tests on GPU
working-directory: /workspace/transformers working-directory: /workspace/transformers
run: | run: |
python -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended python -m pytest -v --make-reports=${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports tests/deepspeed tests/extended
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt run: cat /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports name: ${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu path: /workspace/transformers/reports/${{ matrix.machine_type }}_run_torch_cuda_extensions_gpu_test_reports
run_tests_quantization_torch_gpu: run_quantization_torch_gpu:
if: ${{ inputs.job == 'run_tests_quantization_torch_gpu' }} if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
name: " " name: " "
needs: setup needs: setup
strategy: strategy:
@ -357,26 +357,26 @@ jobs:
- name: Run quantization tests on GPU - name: Run quantization tests on GPU
working-directory: /transformers working-directory: /transformers
run: | run: |
python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_quantization_torch_gpu_${{ matrix.folders }} tests/${{ matrix.folders }} python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Failure short reports - name: Failure short reports
if: ${{ failure() }} if: ${{ failure() }}
continue-on-error: true continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_tests_quantization_torch_gpu_${{ matrix.folders }}/failures_short.txt run: cat /transformers/reports/${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_tests_quantization_torch_gpu_${{ env.matrix_folders }}" - name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }} if: ${{ always() }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: ${{ matrix.machine_type }}_run_tests_quantization_torch_gpu_${{ env.matrix_folders }} name: ${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_tests_quantization_torch_gpu_${{ matrix.folders }} path: /transformers/reports/${{ matrix.machine_type }}_run_quantization_torch_gpu_${{ matrix.folders }}_test_reports
run_extract_warnings: run_extract_warnings:
# Let's only do this for the job `run_tests_gpu` to simplify the (already complex) logic. # Let's only do this for the job `run_models_gpu` to simplify the (already complex) logic.
if: ${{ always() && inputs.job == 'run_tests_gpu' }} if: ${{ always() && inputs.job == 'run_models_gpu' }}
name: Extract warnings in CI artifacts name: Extract warnings in CI artifacts
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
needs: [setup, run_tests_gpu] needs: [setup, run_models_gpu]
steps: steps:
- name: Checkout transformers - name: Checkout transformers
uses: actions/checkout@v4 uses: actions/checkout@v4
@ -416,12 +416,12 @@ jobs:
name: Slack Report name: Slack Report
needs: [ needs: [
setup, setup,
run_tests_gpu, run_models_gpu,
run_pipelines_torch_gpu, run_pipelines_torch_gpu,
run_pipelines_tf_gpu, run_pipelines_tf_gpu,
run_examples_gpu, run_examples_gpu,
run_all_tests_torch_cuda_extensions_gpu, run_torch_cuda_extensions_gpu,
run_tests_quantization_torch_gpu, run_quantization_torch_gpu,
run_extract_warnings run_extract_warnings
] ]
if: ${{ always() }} if: ${{ always() }}

6
.github/workflows/slack-report.yml vendored
View File

@ -35,7 +35,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/download-artifact@v4 - uses: actions/download-artifact@v4
- name: Send message to Slack - name: Send message to Slack
if: ${{ inputs.job != 'run_tests_quantization_torch_gpu' }} if: ${{ inputs.job != 'run_quantization_torch_gpu' }}
env: env:
CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }} CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }} CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}
@ -61,7 +61,7 @@ jobs:
# Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack. # Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack.
- name: Failure table artifacts - name: Failure table artifacts
# Only the model testing job is concerned for this step # Only the model testing job is concerned for this step
if: ${{ inputs.job == 'run_tests_gpu' }} if: ${{ inputs.job == 'run_models_gpu' }}
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: prev_ci_results name: prev_ci_results
@ -70,7 +70,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/download-artifact@v4 - uses: actions/download-artifact@v4
- name: Send message to Slack for quantization workflow - name: Send message to Slack for quantization workflow
if: ${{ inputs.job == 'run_tests_quantization_torch_gpu' }} if: ${{ inputs.job == 'run_quantization_torch_gpu' }}
env: env:
CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }} CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }}

20
utils/notification_service.py
View File

@ -992,13 +992,13 @@ if __name__ == "__main__":
"job_link": {}, "job_link": {},
} }
for model in models for model in models
if f"run_all_tests_gpu_{model}_test_reports" in available_artifacts if f"run_models_gpu_{model}_test_reports" in available_artifacts
} }
unclassified_model_failures = [] unclassified_model_failures = []
for model in model_results.keys(): for model in model_results.keys():
for artifact_path in available_artifacts[f"run_all_tests_gpu_{model}_test_reports"].paths: for artifact_path in available_artifacts[f"run_models_gpu_{model}_test_reports"].paths:
artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"]) artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"])
if "stats" in artifact: if "stats" in artifact:
# Link to the GitHub Action job # Link to the GitHub Action job
@ -1052,10 +1052,10 @@ if __name__ == "__main__":
# Additional runs # Additional runs
additional_files = { additional_files = {
"PyTorch pipelines": "run_tests_torch_pipeline_gpu", "PyTorch pipelines": "run_pipelines_torch_gpu_test_reports",
"TensorFlow pipelines": "run_tests_tf_pipeline_gpu", "TensorFlow pipelines": "run_pipelines_tf_gpu_test_reports",
"Examples directory": "run_examples_gpu", "Examples directory": "run_examples_gpu_test_reports",
"Torch CUDA extension tests": "run_tests_torch_cuda_extensions_gpu_test_reports", "Torch CUDA extension tests": "run_torch_cuda_extensions_gpu_test_reports",
} }
if ci_event in ["push", "Nightly CI"] or ci_event.startswith("Past CI"): if ci_event in ["push", "Nightly CI"] or ci_event.startswith("Past CI"):
@ -1075,7 +1075,7 @@ if __name__ == "__main__":
"run_pipelines_torch_gpu": "PyTorch pipelines", "run_pipelines_torch_gpu": "PyTorch pipelines",
"run_pipelines_tf_gpu": "TensorFlow pipelines", "run_pipelines_tf_gpu": "TensorFlow pipelines",
"run_examples_gpu": "Examples directory", "run_examples_gpu": "Examples directory",
"run_all_tests_torch_cuda_extensions_gpu": "Torch CUDA extension tests", "run_torch_cuda_extensions_gpu": "Torch CUDA extension tests",
} }
# Remove some entries in `additional_files` if they are not concerned. # Remove some entries in `additional_files` if they are not concerned.
@ -1133,10 +1133,10 @@ if __name__ == "__main__":
) )
# Let's only check the warning for the model testing job. Currently, the job `run_extract_warnings` is only run # Let's only check the warning for the model testing job. Currently, the job `run_extract_warnings` is only run
# when `inputs.job` (in the workflow file) is `run_tests_gpu`. The reason is: otherwise we need to save several # when `inputs.job` (in the workflow file) is `run_models_gpu`. The reason is: otherwise we need to save several
# artifacts with different names which complicates the logic for an insignificant part of the CI workflow reporting. # artifacts with different names which complicates the logic for an insignificant part of the CI workflow reporting.
selected_warnings = [] selected_warnings = []
if job_name == "run_tests_gpu": if job_name == "run_models_gpu":
if "warnings_in_ci" in available_artifacts: if "warnings_in_ci" in available_artifacts:
directory = available_artifacts["warnings_in_ci"].paths[0]["path"] directory = available_artifacts["warnings_in_ci"].paths[0]["path"]
with open(os.path.join(directory, "selected_warnings.json")) as fp: with open(os.path.join(directory, "selected_warnings.json")) as fp:
@ -1147,7 +1147,7 @@ if __name__ == "__main__":
# Only the model testing job is concerned: this condition is to avoid other jobs to upload the empty list as # Only the model testing job is concerned: this condition is to avoid other jobs to upload the empty list as
# results. # results.
if job_name == "run_tests_gpu": if job_name == "run_models_gpu":
with open("prev_ci_results/model_results.json", "w", encoding="UTF-8") as fp: with open("prev_ci_results/model_results.json", "w", encoding="UTF-8") as fp:
json.dump(model_results, fp, indent=4, ensure_ascii=False) json.dump(model_results, fp, indent=4, ensure_ascii=False)

4
utils/notification_service_quantization.py
View File

@ -200,7 +200,7 @@ if __name__ == "__main__":
"job_link": {}, "job_link": {},
} }
for quant in quantization_matrix for quant in quantization_matrix
if f"run_tests_quantization_torch_gpu_{quant}" in available_artifacts if f"run_quantization_torch_gpu_{ quant }_test_reports" in available_artifacts
} }
github_actions_jobs = get_jobs( github_actions_jobs = get_jobs(
@ -217,7 +217,7 @@ if __name__ == "__main__":
break break
for quant in quantization_results.keys(): for quant in quantization_results.keys():
for artifact_path in available_artifacts[f"run_tests_quantization_torch_gpu_{quant}"].paths: for artifact_path in available_artifacts[f"run_quantization_torch_gpu_{ quant }_test_reports"].paths:
artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"]) artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"])
if "stats" in artifact: if "stats" in artifact:
# Link to the GitHub Action job # Link to the GitHub Action job

2
utils/split_model_tests.py
View File

@ -18,7 +18,7 @@ The main use case is a GitHub Actions workflow file calling this script to get t
to split the list of jobs to run into multiple slices each containing a smaller number of jobs. This way, we can bypass to split the list of jobs to run into multiple slices each containing a smaller number of jobs. This way, we can bypass
the maximum of 256 jobs in a matrix. the maximum of 256 jobs in a matrix.
See the `setup` and `run_tests_gpu` jobs defined in the workflow file `.github/workflows/self-scheduled.yml` for more See the `setup` and `run_models_gpu` jobs defined in the workflow file `.github/workflows/self-scheduled.yml` for more
details. details.
Usage: Usage: