mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-04 21:30:07 +06:00
Save site-packages
as cache in CircleCI job (#24424)
* fix * fix * Upgrade complete! --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
parent
2834c17ad2
commit
2c977e4a90
@ -31,8 +31,8 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- run: pip install --upgrade pip
|
- run: pip install --upgrade pip
|
||||||
- run: pip install GitPython
|
- run: pip install -U GitPython
|
||||||
- run: pip install .
|
- run: pip install -U .
|
||||||
- run: mkdir -p test_preparation
|
- run: mkdir -p test_preparation
|
||||||
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
|
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
|
||||||
- store_artifacts:
|
- store_artifacts:
|
||||||
@ -105,8 +105,8 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- run: pip install --upgrade pip
|
- run: pip install --upgrade pip
|
||||||
- run: pip install GitPython
|
- run: pip install -U GitPython
|
||||||
- run: pip install .
|
- run: pip install -U .
|
||||||
- run: |
|
- run: |
|
||||||
mkdir test_preparation
|
mkdir test_preparation
|
||||||
echo -n "tests" > test_preparation/test_list.txt
|
echo -n "tests" > test_preparation/test_list.txt
|
||||||
@ -138,12 +138,20 @@ jobs:
|
|||||||
keys:
|
keys:
|
||||||
- v0.6-code_quality-{{ checksum "setup.py" }}
|
- v0.6-code_quality-{{ checksum "setup.py" }}
|
||||||
- v0.6-code-quality
|
- v0.6-code-quality
|
||||||
|
- restore_cache:
|
||||||
|
keys:
|
||||||
|
- v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
|
||||||
|
- v0.6-code-quality-site-packages
|
||||||
- run: pip install --upgrade pip
|
- run: pip install --upgrade pip
|
||||||
- run: pip install .[all,quality]
|
- run: pip install -U .[all,quality]
|
||||||
- save_cache:
|
- save_cache:
|
||||||
key: v0.6-code_quality-{{ checksum "setup.py" }}
|
key: v0.6-code_quality-{{ checksum "setup.py" }}
|
||||||
paths:
|
paths:
|
||||||
- '~/.cache/pip'
|
- '~/.cache/pip'
|
||||||
|
- save_cache:
|
||||||
|
key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
|
||||||
|
paths:
|
||||||
|
- '~/.pyenv/versions/'
|
||||||
- run:
|
- run:
|
||||||
name: Show installed libraries and their versions
|
name: Show installed libraries and their versions
|
||||||
command: pip freeze | tee installed.txt
|
command: pip freeze | tee installed.txt
|
||||||
@ -171,12 +179,20 @@ jobs:
|
|||||||
keys:
|
keys:
|
||||||
- v0.6-repository_consistency-{{ checksum "setup.py" }}
|
- v0.6-repository_consistency-{{ checksum "setup.py" }}
|
||||||
- v0.6-repository_consistency
|
- v0.6-repository_consistency
|
||||||
|
- restore_cache:
|
||||||
|
keys:
|
||||||
|
- v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
|
||||||
|
- v0.6-repository_consistency-site-packages
|
||||||
- run: pip install --upgrade pip
|
- run: pip install --upgrade pip
|
||||||
- run: pip install .[all,quality]
|
- run: pip install -U .[all,quality]
|
||||||
- save_cache:
|
- save_cache:
|
||||||
key: v0.6-repository_consistency-{{ checksum "setup.py" }}
|
key: v0.6-repository_consistency-{{ checksum "setup.py" }}
|
||||||
paths:
|
paths:
|
||||||
- '~/.cache/pip'
|
- '~/.cache/pip'
|
||||||
|
- save_cache:
|
||||||
|
key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
|
||||||
|
paths:
|
||||||
|
- '~/.pyenv/versions/'
|
||||||
- run:
|
- run:
|
||||||
name: Show installed libraries and their versions
|
name: Show installed libraries and their versions
|
||||||
command: pip freeze | tee installed.txt
|
command: pip freeze | tee installed.txt
|
||||||
|
@ -106,10 +106,18 @@ class CircleCIJob:
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"restore_cache": {
|
||||||
|
"keys": [
|
||||||
|
f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
|
||||||
|
f"v{self.cache_version}-{self.cache_name}-site-packages",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
]
|
]
|
||||||
steps.extend([{"run": l} for l in self.install_steps])
|
steps.extend([{"run": l} for l in self.install_steps])
|
||||||
# TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
|
# TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
|
||||||
steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install git+https://github.com/huggingface/huggingface_hub.git@c36817701ecd4694b290178846176da2e195d838"}})
|
steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@c36817701ecd4694b290178846176da2e195d838"}})
|
||||||
steps.append(
|
steps.append(
|
||||||
{
|
{
|
||||||
"save_cache": {
|
"save_cache": {
|
||||||
@ -118,6 +126,14 @@ class CircleCIJob:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
steps.append(
|
||||||
|
{
|
||||||
|
"save_cache": {
|
||||||
|
"key": f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
|
||||||
|
"paths": ["~/.pyenv/versions/"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
|
steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
|
||||||
steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
|
steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
|
||||||
|
|
||||||
@ -229,9 +245,9 @@ torch_and_tf_job = CircleCIJob(
|
|||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
|
||||||
"git lfs install",
|
"git lfs install",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
|
"pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
|
||||||
"pip install tensorflow_probability",
|
"pip install -U tensorflow_probability",
|
||||||
"pip install git+https://github.com/huggingface/accelerate",
|
"pip install -U git+https://github.com/huggingface/accelerate",
|
||||||
],
|
],
|
||||||
marker="is_pt_tf_cross_test",
|
marker="is_pt_tf_cross_test",
|
||||||
pytest_options={"rA": None, "durations": 0},
|
pytest_options={"rA": None, "durations": 0},
|
||||||
@ -243,9 +259,9 @@ torch_and_flax_job = CircleCIJob(
|
|||||||
additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
|
additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
|
||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
||||||
"pip install --upgrade pip",
|
"pip install -U --upgrade pip",
|
||||||
"pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
|
"pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
|
||||||
"pip install git+https://github.com/huggingface/accelerate",
|
"pip install -U git+https://github.com/huggingface/accelerate",
|
||||||
],
|
],
|
||||||
marker="is_pt_flax_cross_test",
|
marker="is_pt_flax_cross_test",
|
||||||
pytest_options={"rA": None, "durations": 0},
|
pytest_options={"rA": None, "durations": 0},
|
||||||
@ -257,8 +273,8 @@ torch_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
|
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
|
||||||
"pip install git+https://github.com/huggingface/accelerate",
|
"pip install -U git+https://github.com/huggingface/accelerate",
|
||||||
],
|
],
|
||||||
parallelism=1,
|
parallelism=1,
|
||||||
pytest_num_workers=3,
|
pytest_num_workers=3,
|
||||||
@ -270,8 +286,8 @@ tf_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
|
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
|
||||||
"pip install tensorflow_probability",
|
"pip install -U tensorflow_probability",
|
||||||
],
|
],
|
||||||
parallelism=1,
|
parallelism=1,
|
||||||
pytest_num_workers=6,
|
pytest_num_workers=6,
|
||||||
@ -284,7 +300,7 @@ flax_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[flax,testing,sentencepiece,flax-speech,vision]",
|
"pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
|
||||||
],
|
],
|
||||||
parallelism=1,
|
parallelism=1,
|
||||||
pytest_options={"rA": None},
|
pytest_options={"rA": None},
|
||||||
@ -297,7 +313,7 @@ pipelines_torch_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
|
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
|
||||||
],
|
],
|
||||||
pytest_options={"rA": None},
|
pytest_options={"rA": None},
|
||||||
marker="is_pipeline_test",
|
marker="is_pipeline_test",
|
||||||
@ -310,8 +326,8 @@ pipelines_tf_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y cmake",
|
"sudo apt-get -y update && sudo apt-get install -y cmake",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,tf-cpu,testing,sentencepiece,vision]",
|
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
|
||||||
"pip install tensorflow_probability",
|
"pip install -U tensorflow_probability",
|
||||||
],
|
],
|
||||||
pytest_options={"rA": None},
|
pytest_options={"rA": None},
|
||||||
marker="is_pipeline_test",
|
marker="is_pipeline_test",
|
||||||
@ -334,7 +350,7 @@ custom_tokenizers_job = CircleCIJob(
|
|||||||
"sudo make install\n",
|
"sudo make install\n",
|
||||||
},
|
},
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
|
"pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
|
||||||
"python -m unidic download",
|
"python -m unidic download",
|
||||||
],
|
],
|
||||||
parallelism=None,
|
parallelism=None,
|
||||||
@ -353,8 +369,8 @@ examples_torch_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,torch,sentencepiece,testing,torch-speech]",
|
"pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
|
||||||
"pip install -r examples/pytorch/_tests_requirements.txt",
|
"pip install -U -r examples/pytorch/_tests_requirements.txt",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -365,8 +381,8 @@ examples_tensorflow_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y cmake",
|
"sudo apt-get -y update && sudo apt-get install -y cmake",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[sklearn,tensorflow,sentencepiece,testing]",
|
"pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
|
||||||
"pip install -r examples/tensorflow/_tests_requirements.txt",
|
"pip install -U -r examples/tensorflow/_tests_requirements.txt",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -376,8 +392,8 @@ examples_flax_job = CircleCIJob(
|
|||||||
cache_name="flax_examples",
|
cache_name="flax_examples",
|
||||||
install_steps=[
|
install_steps=[
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[flax,testing,sentencepiece]",
|
"pip install -U .[flax,testing,sentencepiece]",
|
||||||
"pip install -r examples/flax/_tests_requirements.txt",
|
"pip install -U -r examples/flax/_tests_requirements.txt",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -389,7 +405,7 @@ hub_job = CircleCIJob(
|
|||||||
'git config --global user.email "ci@dummy.com"',
|
'git config --global user.email "ci@dummy.com"',
|
||||||
'git config --global user.name "ci"',
|
'git config --global user.name "ci"',
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[torch,sentencepiece,testing]",
|
"pip install -U .[torch,sentencepiece,testing]",
|
||||||
],
|
],
|
||||||
marker="is_staging_test",
|
marker="is_staging_test",
|
||||||
pytest_num_workers=1,
|
pytest_num_workers=1,
|
||||||
@ -401,7 +417,7 @@ onnx_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y cmake",
|
"sudo apt-get -y update && sudo apt-get install -y cmake",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
|
"pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
|
||||||
],
|
],
|
||||||
pytest_options={"k onnx": None},
|
pytest_options={"k onnx": None},
|
||||||
pytest_num_workers=1,
|
pytest_num_workers=1,
|
||||||
@ -413,13 +429,13 @@ exotic_models_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[torch,testing,vision]",
|
"pip install -U .[torch,testing,vision]",
|
||||||
"pip install torchvision",
|
"pip install -U torchvision",
|
||||||
"pip install scipy",
|
"pip install -U scipy",
|
||||||
"pip install 'git+https://github.com/facebookresearch/detectron2.git'",
|
"pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
|
||||||
"sudo apt install tesseract-ocr",
|
"sudo apt install tesseract-ocr",
|
||||||
"pip install pytesseract",
|
"pip install -U pytesseract",
|
||||||
"pip install natten",
|
"pip install -U natten",
|
||||||
],
|
],
|
||||||
tests_to_run=[
|
tests_to_run=[
|
||||||
"tests/models/*layoutlmv*",
|
"tests/models/*layoutlmv*",
|
||||||
@ -435,7 +451,7 @@ repo_utils_job = CircleCIJob(
|
|||||||
"repo_utils",
|
"repo_utils",
|
||||||
install_steps=[
|
install_steps=[
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install .[quality,testing,torch]",
|
"pip install -U .[quality,testing,torch]",
|
||||||
],
|
],
|
||||||
parallelism=None,
|
parallelism=None,
|
||||||
pytest_num_workers=1,
|
pytest_num_workers=1,
|
||||||
@ -456,10 +472,10 @@ doc_test_job = CircleCIJob(
|
|||||||
install_steps=[
|
install_steps=[
|
||||||
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
|
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
|
||||||
"pip install --upgrade pip",
|
"pip install --upgrade pip",
|
||||||
"pip install -e .[dev]",
|
"pip install -U -e .[dev]",
|
||||||
"pip install git+https://github.com/huggingface/accelerate",
|
"pip install -U git+https://github.com/huggingface/accelerate",
|
||||||
"pip install --upgrade pytest pytest-sugar",
|
"pip install --upgrade pytest pytest-sugar",
|
||||||
"pip install natten",
|
"pip install -U natten",
|
||||||
"find -name __pycache__ -delete",
|
"find -name __pycache__ -delete",
|
||||||
"find . -name \*.pyc -delete",
|
"find . -name \*.pyc -delete",
|
||||||
# Add an empty file to keep the test step running correctly even no file is selected to be tested.
|
# Add an empty file to keep the test step running correctly even no file is selected to be tested.
|
||||||
|
Loading…
Reference in New Issue
Block a user