mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-30 17:52:35 +06:00
39acfe84ba
* add deepspeed scheduled test for amd * fix image * add dockerfile * add comment * enable tests * trigger * remove trigger for this branch * trigger * change runner env to trigger the docker build image test * use new docker image * remove test suffix from docker image tag * replace test docker image with original image * push new image * Trigger * add back amd tests * fix typo * add amd tests back * fix * comment until docker image build scheduled test fix * remove deprecated deepspeed build option * upgrade torch * update docker & make tests pass * Update docker/transformers-pytorch-deepspeed-amd-gpu/Dockerfile * fix * tmp disable test * precompile deepspeed to avoid timeout during tests * fix comment * trigger deepspeed tests with new image * comment tests * trigger * add sklearn dependency to fix slow tests * enable back other tests * final update --------- Co-authored-by: Felix Marty <felix@hf.co> Co-authored-by: Félix Marty <9808326+fxmarty@users.noreply.github.com> Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
310 lines
10 KiB
YAML
310 lines
10 KiB
YAML
name: Build docker images (scheduled)
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- build_ci_docker_image*
|
|
repository_dispatch:
|
|
workflow_call:
|
|
inputs:
|
|
image_postfix:
|
|
required: true
|
|
type: string
|
|
schedule:
|
|
- cron: "17 0 * * *"
|
|
|
|
concurrency:
|
|
group: docker-images-builds
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
latest-docker:
|
|
name: "Latest PyTorch + TensorFlow [dev]"
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Cleanup disk
|
|
run: |
|
|
sudo ls -l /usr/local/lib/
|
|
sudo ls -l /usr/share/
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
-
|
|
name: Check out code
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-all-latest-gpu
|
|
build-args: |
|
|
REF=main
|
|
push: true
|
|
tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }}
|
|
# Push CI images still need to be re-built daily
|
|
-
|
|
name: Build and push (for Push CI) in a daily basis
|
|
# This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
|
|
# The later case is useful for manual image building for debugging purpose. Use another tag in this case!
|
|
if: inputs.image_postfix != '-push-ci'
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-all-latest-gpu
|
|
build-args: |
|
|
REF=main
|
|
push: true
|
|
tags: huggingface/transformers-all-latest-gpu-push-ci
|
|
|
|
latest-torch-deepspeed-docker:
|
|
name: "Latest PyTorch + DeepSpeed"
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Cleanup disk
|
|
run: |
|
|
sudo ls -l /usr/local/lib/
|
|
sudo ls -l /usr/share/
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
-
|
|
name: Check out code
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-pytorch-deepspeed-latest-gpu
|
|
build-args: |
|
|
REF=main
|
|
push: true
|
|
tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }}
|
|
|
|
# Can't build 2 images in a single job `latest-torch-deepspeed-docker` (for `nvcr.io/nvidia`)
|
|
latest-torch-deepspeed-docker-for-push-ci-daily-build:
|
|
name: "Latest PyTorch + DeepSpeed (Push CI - Daily Build)"
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Cleanup disk
|
|
run: |
|
|
sudo ls -l /usr/local/lib/
|
|
sudo ls -l /usr/share/
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
-
|
|
name: Check out code
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
# Push CI images still need to be re-built daily
|
|
-
|
|
name: Build and push (for Push CI) in a daily basis
|
|
# This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
|
|
# The later case is useful for manual image building for debugging purpose. Use another tag in this case!
|
|
if: inputs.image_postfix != '-push-ci'
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-pytorch-deepspeed-latest-gpu
|
|
build-args: |
|
|
REF=main
|
|
push: true
|
|
tags: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci
|
|
|
|
doc-builder:
|
|
name: "Doc builder"
|
|
# Push CI doesn't need this image
|
|
if: inputs.image_postfix != '-push-ci'
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
-
|
|
name: Check out code
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-doc-builder
|
|
push: true
|
|
tags: huggingface/transformers-doc-builder
|
|
|
|
latest-pytorch:
|
|
name: "Latest PyTorch [dev]"
|
|
# Push CI doesn't need this image
|
|
if: inputs.image_postfix != '-push-ci'
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Cleanup disk
|
|
run: |
|
|
sudo ls -l /usr/local/lib/
|
|
sudo ls -l /usr/share/
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo du -sh /usr/local/lib/
|
|
sudo du -sh /usr/share/
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
-
|
|
name: Check out code
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-pytorch-gpu
|
|
build-args: |
|
|
REF=main
|
|
push: true
|
|
tags: huggingface/transformers-pytorch-gpu
|
|
|
|
# Need to be fixed with the help from Guillaume.
|
|
# latest-pytorch-amd:
|
|
# name: "Latest PyTorch (AMD) [dev]"
|
|
# runs-on: [self-hosted, docker-gpu, amd-gpu, single-gpu, mi210]
|
|
# steps:
|
|
# - name: Set up Docker Buildx
|
|
# uses: docker/setup-buildx-action@v3
|
|
# - name: Check out code
|
|
# uses: actions/checkout@v3
|
|
# - name: Login to DockerHub
|
|
# uses: docker/login-action@v3
|
|
# with:
|
|
# username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
# password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
# - name: Build and push
|
|
# uses: docker/build-push-action@v5
|
|
# with:
|
|
# context: ./docker/transformers-pytorch-amd-gpu
|
|
# build-args: |
|
|
# REF=main
|
|
# push: true
|
|
# tags: huggingface/transformers-pytorch-amd-gpu${{ inputs.image_postfix }}
|
|
# # Push CI images still need to be re-built daily
|
|
# -
|
|
# name: Build and push (for Push CI) in a daily basis
|
|
# # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
|
|
# # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
|
|
# if: inputs.image_postfix != '-push-ci'
|
|
# uses: docker/build-push-action@v5
|
|
# with:
|
|
# context: ./docker/transformers-pytorch-amd-gpu
|
|
# build-args: |
|
|
# REF=main
|
|
# push: true
|
|
# tags: huggingface/transformers-pytorch-amd-gpu-push-ci
|
|
|
|
latest-tensorflow:
|
|
name: "Latest TensorFlow [dev]"
|
|
# Push CI doesn't need this image
|
|
if: inputs.image_postfix != '-push-ci'
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
-
|
|
name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
-
|
|
name: Check out code
|
|
uses: actions/checkout@v3
|
|
-
|
|
name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
-
|
|
name: Build and push
|
|
uses: docker/build-push-action@v5
|
|
with:
|
|
context: ./docker/transformers-tensorflow-gpu
|
|
build-args: |
|
|
REF=main
|
|
push: true
|
|
tags: huggingface/transformers-tensorflow-gpu
|
|
|
|
# latest-pytorch-deepspeed-amd:
|
|
# name: "PyTorch + DeepSpeed (AMD) [dev]"
|
|
|
|
# runs-on: [self-hosted, docker-gpu, amd-gpu, single-gpu, mi210]
|
|
# steps:
|
|
# - name: Set up Docker Buildx
|
|
# uses: docker/setup-buildx-action@v3
|
|
# - name: Check out code
|
|
# uses: actions/checkout@v3
|
|
# - name: Login to DockerHub
|
|
# uses: docker/login-action@v3
|
|
# with:
|
|
# username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
# password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
|
# - name: Build and push
|
|
# uses: docker/build-push-action@v5
|
|
# with:
|
|
# context: ./docker/transformers-pytorch-deepspeed-amd-gpu
|
|
# build-args: |
|
|
# REF=main
|
|
# push: true
|
|
# tags: huggingface/transformers-pytorch-deepspeed-amd-gpu${{ inputs.image_postfix }}
|
|
# # Push CI images still need to be re-built daily
|
|
# -
|
|
# name: Build and push (for Push CI) in a daily basis
|
|
# # This condition allows `schedule` events, or `push` events that trigger this workflow NOT via `workflow_call`.
|
|
# # The later case is useful for manual image building for debugging purpose. Use another tag in this case!
|
|
# if: inputs.image_postfix != '-push-ci'
|
|
# uses: docker/build-push-action@v5
|
|
# with:
|
|
# context: ./docker/transformers-pytorch-deepspeed-amd-gpu
|
|
# build-args: |
|
|
# REF=main
|
|
# push: true
|
|
# tags: huggingface/transformers-pytorch-deepspeed-amd-gpu-push-ci
|