Update build ci image [push-ci-image] (#30933)

* [build-ci-image] * correct branch * push ci image * [build-ci-image] * update scheduled as well * [push-ci-image] * [build-ci-image] * [push-ci-image] * update deps * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * oups [build-ci-image] * [push-ci-image] * fix * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * updated * [build-ci-image] update tag * [build-ci-image] * [build-ci-image] * fix tag * [build-ci-image] * [build-ci-image] * [build-ci-image] * [build-ci-image] * github name * commit_title? * fetch * update * it not found * dev * dev * [push-ci-image] * dev * dev * update * dev * dev print dev commit message dev * dev ? dev * dev * dev * dev * dev * [build-ci-image] * [build-ci-image] * [push-ci-image] * revert unwanted * revert convert as well * no you are not important * [build-ci-image] * Update .circleci/config.yml * pin tf probability dev
2025-07-03 12:50:06 +06:00 · 2024-05-22 10:52:59 +02:00 · 2024-05-22 10:52:59 +02:00 · 8e8786e5f0
commit 8e8786e5f0
parent 673440d073
13 changed files with 43 additions and 18 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -31,6 +31,7 @@ jobs:
        steps:
            - checkout
            - run: uv pip install -U -e .
+            - run: echo 'export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)"' >> "$BASH_ENV" && source "$BASH_ENV"
            - run: mkdir -p test_preparation
            - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
            - store_artifacts:
@ -80,7 +81,7 @@ jobs:
                  path: ~/transformers/test_preparation/filtered_test_list.txt
            - store_artifacts:
                  path: test_preparation/examples_test_list.txt
-            - run: python .circleci/create_circleci_config.py --fetcher_folder test_preparation
+            - run: export "GIT_COMMIT_MESSAGE=$(git show -s --format=%s)" && echo $GIT_COMMIT_MESSAGE && python .circleci/create_circleci_config.py --fetcher_folder test_preparation
            - run: |
                  if [ ! -s test_preparation/generated_config.yml ]; then
                      echo "No tests to run, exiting early!"
--- a/.circleci/create_circleci_config.py
+++ b/.circleci/create_circleci_config.py
@ -72,6 +72,12 @@ class CircleCIJob:
        if self.docker_image is None:
            # Let's avoid changing the default list and make a copy.
            self.docker_image = copy.deepcopy(DEFAULT_DOCKER_IMAGE)
+        else:
+            # BIG HACK WILL REMOVE ONCE FETCHER IS UPDATED
+            print(os.environ.get("GIT_COMMIT_MESSAGE"))
+            if "[build-ci-image]" in os.environ.get("GIT_COMMIT_MESSAGE", "") or "dev" in  os.environ.get("GIT_COMMIT_MESSAGE", ""):
+                self.docker_image[0]["image"] = f"{self.docker_image[0]['image']}:dev"
+            print(f"Using {self.docker_image} docker image")
        if self.install_steps is None:
            self.install_steps = []
        if self.pytest_options is None:
--- a/.github/workflows/build-ci-docker-images.yml
+++ b/.github/workflows/build-ci-docker-images.yml
@ -3,7 +3,7 @@ name: Build pr ci-docker
 on:
  push:
    branches:
-      - change-ci # for now let's only build on this branch
+      - push-ci-image # for now let's only build on this branch
  repository_dispatch:
  workflow_call:
    inputs:
@ -22,7 +22,7 @@ jobs:
  build:
    runs-on: ubuntu-22.04

-    if: ${{ contains(github.event.head_commit.message, '[push-ci-image]') && '!cancelled()' }}
+    if: ${{ contains(github.event.head_commit.message, '[build-ci-image]') || contains(github.event.head_commit.message, '[push-ci-image]') && '!cancelled()' || github.event_name == 'schedule' }}

    strategy:
      matrix:
@ -30,6 +30,16 @@ jobs:
    continue-on-error: true 

    steps:
+      - 
+        name: Set tag
+        run: |
+              if [ ${{contains(github.event.head_commit.message, '[build-ci-image]')}} ]; then
+                  echo "TAG=huggingface/transformers-${{ matrix.file }}:dev" >> "$GITHUB_ENV" 
+                  echo "setting it to DEV!"
+              else
+                  echo "TAG=huggingface/transformers-${{ matrix.file }}" >> "$GITHUB_ENV"
+                  
+              fi
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
@ -50,5 +60,5 @@ jobs:
          build-args: |
            REF=${{ github.sha }}
          file: "./docker/${{ matrix.file }}.dockerfile"
-          push: true
-          tags: huggingface/transformers-${{ matrix.file }}
+          push: ${{ contains(github.event.head_commit.message, 'ci-image]') ||  github.event_name == 'schedule' }}
+          tags: ${{ env.TAG }}
--- a/docker/consistency.dockerfile
+++ b/docker/consistency.dockerfile
@ -1,12 +1,13 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
 USER root
+ARG REF=main
 RUN apt-get update && apt-get install -y time git pkg-config make git-lfs
 ENV VIRTUAL_ENV=/usr/local
 RUN pip install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython
 RUN uv pip install --no-cache-dir --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-cache-dir tensorflow-cpu tf-keras
-RUN uv pip install --no-cache-dir "transformers[flax,quality,vision,testing]"
+RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,quality,vision,testing]" 
 RUN git lfs install

 RUN pip uninstall -y transformers
--- a/docker/jax-light.dockerfile
+++ b/docker/jax-light.dockerfile
@ -1,9 +1,10 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv &&  uv venv && uv pip install --no-cache-dir -U pip setuptools
-RUN pip install --no-cache-dir "scipy<1.13" "transformers[flax,testing,sentencepiece,flax-speech,vision]"
+RUN pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,testing,sentencepiece,flax-speech,vision]"
 RUN pip uninstall -y transformers
 RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
--- a/docker/pipeline-tf.dockerfile
+++ b/docker/pipeline-tf.dockerfile
@ -1,9 +1,10 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake g++
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
-RUN pip install --no-cache-dir "transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]"
+RUN pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]"
 RUN uv pip install --no-cache-dir  "protobuf==3.20.3" tensorflow_probability
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
--- a/docker/pipeline-torch.dockerfile
+++ b/docker/pipeline-torch.dockerfile
@ -1,10 +1,11 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git pkg-config openssh-client git
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu 
-RUN uv pip install --no-cache-dir librosa "transformers[sklearn,sentencepiece,vision,testing]"
+RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]"
 RUN pip uninstall -y transformers
--- a/docker/quality.dockerfile
+++ b/docker/quality.dockerfile
@ -1,8 +1,9 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update && apt-get install -y time git 
 ENV VIRTUAL_ENV=/usr/local
 RUN pip install uv &&  uv venv
-RUN uv pip install --no-cache-dir -U pip setuptools GitPython transformers "ruff==0.1.5" urllib3
+RUN uv pip install --no-cache-dir -U pip setuptools GitPython "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[ruff]" urllib3
 RUN apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/*
--- a/docker/tf-light.dockerfile
+++ b/docker/tf-light.dockerfile
@ -1,11 +1,12 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ pkg-config openssh-client git
 RUN apt-get install -y  cmake
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
-RUN pip install  --upgrade --no-cache-dir "transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]"
+RUN pip install  --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]"
 RUN uv pip install --no-cache-dir  "protobuf==3.20.3" 
 RUN pip uninstall -y transformers
 RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean
--- a/docker/torch-jax-light.dockerfile
+++ b/docker/torch-jax-light.dockerfile
@ -1,12 +1,13 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update &&  apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
 RUN uv pip install --no-deps accelerate
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
-RUN pip install --no-cache-dir "scipy<1.13" "transformers[flax, audio, sklearn,sentencepiece,vision,testing]"
+RUN pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,audio,sklearn,sentencepiece,vision,testing]"


 # RUN pip install --no-cache-dir "scipy<1.13" "transformers[flax,testing,sentencepiece,flax-speech,vision]"
--- a/docker/torch-light.dockerfile
+++ b/docker/torch-light.dockerfile
@ -1,10 +1,11 @@
 FROM python:3.10-slim
 ENV PYTHONDONTWRITEBYTECODE=1
+ARG REF=main
 USER root
 RUN apt-get update &&  apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git git-lfs
 ENV VIRTUAL_ENV=/usr/local
 RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools
 RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu
 RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu 
-RUN uv pip install --no-cache-dir librosa "transformers[sklearn,sentencepiece,vision,testing]"
+RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]"
 RUN pip uninstall -y transformers
--- a/setup.py
+++ b/setup.py
@ -173,10 +173,10 @@ _deps = [
    "tensorflow-cpu>2.9,<2.16",
    "tensorflow>2.9,<2.16",
    "tensorflow-text<2.16",
-    "tensorflow-probability<2.16",
+    "tensorflow-probability<0.24",
    "tf2onnx",
    "timeout-decorator",
-    "timm",
+    "timm<=0.9.16",
    "tokenizers>=0.19,<0.20",
    "torch",
    "torchaudio",
@ -339,7 +339,7 @@ extras["testing"] = (
 )

 extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
-
+extras["ruff"] = deps_list("ruff")
 extras["quality"] = deps_list("datasets", "isort", "ruff", "GitPython", "urllib3")

 extras["all"] = (
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@ -78,10 +78,10 @@ deps = {
    "tensorflow-cpu": "tensorflow-cpu>2.9,<2.16",
    "tensorflow": "tensorflow>2.9,<2.16",
    "tensorflow-text": "tensorflow-text<2.16",
-    "tensorflow-probability": "tensorflow-probability<2.16",
+    "tensorflow-probability": "tensorflow-probability<0.24",
    "tf2onnx": "tf2onnx",
    "timeout-decorator": "timeout-decorator",
-    "timm": "timm",
+    "timm": "timm<=0.9.16",
    "tokenizers": "tokenizers>=0.19,<0.20",
    "torch": "torch",
    "torchaudio": "torchaudio",