From a41677a68bcdb1deb26df44f94f8c0cd2eeb2de7 Mon Sep 17 00:00:00 2001 From: Sai-Suraj-27 Date: Tue, 25 Mar 2025 22:42:51 +0530 Subject: [PATCH] Updated docker files to use `uv` for installing packages (#36957) * Updated docker files to use uv pip install as uv is blazingly fast. * Removed -y flag for uv pip uninstall. * Passed --no-build-isolation flag --------- Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> --- docker/README.md | 4 ++-- docker/consistency.dockerfile | 6 +++--- docker/custom-tokenizers.dockerfile | 2 +- docker/examples-tf.dockerfile | 4 ++-- docker/examples-torch.dockerfile | 4 ++-- docker/exotic-models.dockerfile | 8 ++++---- docker/jax-light.dockerfile | 6 +++--- docker/pipeline-tf.dockerfile | 4 ++-- docker/pipeline-torch.dockerfile | 4 ++-- docker/quality.dockerfile | 2 +- docker/tf-light.dockerfile | 4 ++-- docker/torch-jax-light.dockerfile | 6 +++--- docker/torch-light.dockerfile | 4 ++-- docker/torch-tf-light.dockerfile | 8 ++++---- 14 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docker/README.md b/docker/README.md index 2a71ab6fb6e..5410a2839e3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,8 +2,8 @@ In this folder you will find various docker files, and some subfolders. - dockerfiles (ex: `consistency.dockerfile`) present under `~/docker` are used for our "fast" CIs. You should be able to use them for tasks that only need CPU. For example `torch-light` is a very light weights container (703MiB). -- subfloder contain dockerfiles used for our `slow` CIs, which *can* be used for GPU tasks, but they are **BIG** as they were not specifically designed for a single model / single task. Thus the `~/docker/transformers-pytorch-gpu` includes additional dependencies to allow us to run ALL model tests (say `librosa` or `tesseract`, which you do not need to run LLMs) +- subfolders contain dockerfiles used for our `slow` CIs, which *can* be used for GPU tasks, but they are **BIG** as they were not specifically designed for a single model / single task. Thus the `~/docker/transformers-pytorch-gpu` includes additional dependencies to allow us to run ALL model tests (say `librosa` or `tesseract`, which you do not need to run LLMs) Note that in both case, you need to run `uv pip install -e .`, which should take around 5 seconds. We do it outside the dockerfile for the need of our CI: we checkout a new branch each time, and the `transformers` code is thus updated. -We are open to contribution, and invite the community to create dockerfiles with potential arguments that properly choose extras depending on the model's dependencies! :hugs: \ No newline at end of file +We are open to contribution, and invite the community to create dockerfiles with potential arguments that properly choose extras depending on the model's dependencies! :hugs: diff --git a/docker/consistency.dockerfile b/docker/consistency.dockerfile index a564d76c9bb..5b35a5f85dc 100644 --- a/docker/consistency.dockerfile +++ b/docker/consistency.dockerfile @@ -5,12 +5,12 @@ ARG REF=main RUN apt-get update && apt-get install -y time git g++ pkg-config make git-lfs ENV UV_PYTHON=/usr/local/bin/python RUN pip install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython -RUN pip install --no-cache-dir --upgrade 'torch' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir --upgrade 'torch' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu # tensorflow pin matching setup.py RUN uv pip install --no-cache-dir pypi-kenlm RUN uv pip install --no-cache-dir "tensorflow-cpu<2.16" "tf-keras<2.16" RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,quality,testing,torch-speech,vision]" RUN git lfs install -RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean \ No newline at end of file +RUN uv pip uninstall transformers +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile index 2261c44099c..a0a9f5ea23b 100644 --- a/docker/custom-tokenizers.dockerfile +++ b/docker/custom-tokenizers.dockerfile @@ -21,7 +21,7 @@ RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https: RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic unidic-lite # spacy is not used so not tested. Causes to failures. TODO fix later RUN python3 -m unidic download -RUN pip uninstall -y transformers +RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* RUN apt remove -y g++ cmake xz-utils libprotobuf-dev protobuf-compiler diff --git a/docker/examples-tf.dockerfile b/docker/examples-tf.dockerfile index 16899e2a446..306d00fdea5 100644 --- a/docker/examples-tf.dockerfile +++ b/docker/examples-tf.dockerfile @@ -7,7 +7,7 @@ RUN apt-get install -y g++ cmake ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv RUN uv pip install --no-cache-dir -U pip setuptools albumentations seqeval -RUN pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" +RUN uv pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" -RUN pip uninstall -y transformers +RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/examples-torch.dockerfile b/docker/examples-torch.dockerfile index ec868cb10a1..e1029f6ddf1 100644 --- a/docker/examples-torch.dockerfile +++ b/docker/examples-torch.dockerfile @@ -5,8 +5,8 @@ USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]" seqeval albumentations jiwer -RUN pip uninstall -y transformers +RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/exotic-models.dockerfile b/docker/exotic-models.dockerfile index dd40476064a..32491888e80 100644 --- a/docker/exotic-models.dockerfile +++ b/docker/exotic-models.dockerfile @@ -5,13 +5,13 @@ USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git libgl1-mesa-glx libgl1 g++ tesseract-ocr ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir --no-deps timm accelerate RUN pip install -U --upgrade-strategy eager --no-cache-dir pytesseract python-Levenshtein opencv-python nltk # RUN uv pip install --no-cache-dir natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels -RUN pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[testing, vision]" 'scikit-learn' 'torch-stft' 'nose' 'dataset' +RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[testing, vision]" 'scikit-learn' 'torch-stft' 'nose' 'dataset' # RUN git clone https://github.com/facebookresearch/detectron2.git # RUN python3 -m pip install --no-cache-dir -e detectron2 -RUN pip install 'git+https://github.com/facebookresearch/detectron2.git@92ae9f0b92aba5867824b4f12aa06a22a60a45d3' -RUN pip uninstall -y transformers +RUN uv pip install 'git+https://github.com/facebookresearch/detectron2.git@92ae9f0b92aba5867824b4f12aa06a22a60a45d3' --no-build-isolation +RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/jax-light.dockerfile b/docker/jax-light.dockerfile index df1e1144c0e..c2a73e98ca9 100644 --- a/docker/jax-light.dockerfile +++ b/docker/jax-light.dockerfile @@ -5,6 +5,6 @@ USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,testing,sentencepiece,flax-speech,vision]" -RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean \ No newline at end of file +RUN uv pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,testing,sentencepiece,flax-speech,vision]" +RUN uv pip uninstall transformers +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/pipeline-tf.dockerfile b/docker/pipeline-tf.dockerfile index dd38b52acbb..61a442a5594 100644 --- a/docker/pipeline-tf.dockerfile +++ b/docker/pipeline-tf.dockerfile @@ -5,6 +5,6 @@ USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake g++ ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]" +RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" tensorflow_probability -RUN apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/pipeline-torch.dockerfile b/docker/pipeline-torch.dockerfile index 2b4014b4fff..10b6450b2df 100644 --- a/docker/pipeline-torch.dockerfile +++ b/docker/pipeline-torch.dockerfile @@ -5,7 +5,7 @@ USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git pkg-config openssh-client git ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]" -RUN pip uninstall -y transformers \ No newline at end of file +RUN uv pip uninstall transformers diff --git a/docker/quality.dockerfile b/docker/quality.dockerfile index 7e6999e40f2..e2421efe00b 100644 --- a/docker/quality.dockerfile +++ b/docker/quality.dockerfile @@ -6,4 +6,4 @@ RUN apt-get update && apt-get install -y time git ENV UV_PYTHON=/usr/local/bin/python RUN pip install uv && uv venv RUN uv pip install --no-cache-dir -U pip setuptools GitPython "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[ruff]" urllib3 -RUN apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/tf-light.dockerfile b/docker/tf-light.dockerfile index e58db2aa00d..0206c740639 100644 --- a/docker/tf-light.dockerfile +++ b/docker/tf-light.dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-de RUN apt-get install -y cmake ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" +RUN uv pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" -RUN pip uninstall -y transformers +RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/torch-jax-light.dockerfile b/docker/torch-jax-light.dockerfile index b779fcbfac9..6394bc76afc 100644 --- a/docker/torch-jax-light.dockerfile +++ b/docker/torch-jax-light.dockerfile @@ -6,11 +6,11 @@ RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-deps accelerate -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu -RUN pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,audio,sklearn,sentencepiece,vision,testing]" +RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,audio,sklearn,sentencepiece,vision,testing]" # RUN pip install --no-cache-dir "scipy<1.13" "transformers[flax,testing,sentencepiece,flax-speech,vision]" -RUN pip uninstall -y transformers +RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/torch-light.dockerfile b/docker/torch-light.dockerfile index 5e71afe978d..630a0087f6f 100644 --- a/docker/torch-light.dockerfile +++ b/docker/torch-light.dockerfile @@ -5,7 +5,7 @@ USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git git-lfs ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing,tiktoken,num2words]" -RUN pip uninstall -y transformers +RUN uv pip uninstall transformers diff --git a/docker/torch-tf-light.dockerfile b/docker/torch-tf-light.dockerfile index 0556b79ffdb..63512328f12 100644 --- a/docker/torch-tf-light.dockerfile +++ b/docker/torch-tf-light.dockerfile @@ -7,13 +7,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-de ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu -RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN git lfs install RUN uv pip install --no-cache-dir pypi-kenlm -RUN pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,sentencepiece,vision,testing]" +RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,sentencepiece,vision,testing]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" librosa -RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean \ No newline at end of file +RUN uv pip uninstall transformers +RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean