riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-04 21:30:07 +06:00
Code Issues Actions 30 Packages Projects Releases Wiki Activity

Save site-packages as cache in CircleCI job (#24424)

Browse Source 
* fix

* fix

* Upgrade complete!

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar 2023-06-22 23:16:35 +02:00 committed by GitHub
parent 2834c17ad2
commit 2c977e4a90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 72 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.circleci/config.yml
View File

@ -31,8 +31,8 @@ jobs:
steps: steps:
- checkout - checkout
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install GitPython - run: pip install -U GitPython
- run: pip install . - run: pip install -U .
- run: mkdir -p test_preparation - run: mkdir -p test_preparation
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
- store_artifacts: - store_artifacts:
@ -105,8 +105,8 @@ jobs:
steps: steps:
- checkout - checkout
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install GitPython - run: pip install -U GitPython
- run: pip install . - run: pip install -U .
- run: | - run: |
mkdir test_preparation mkdir test_preparation
echo -n "tests" > test_preparation/test_list.txt echo -n "tests" > test_preparation/test_list.txt
@ -138,12 +138,20 @@ jobs:
keys: keys:
- v0.6-code_quality-{{ checksum "setup.py" }} - v0.6-code_quality-{{ checksum "setup.py" }}
- v0.6-code-quality - v0.6-code-quality
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.6-code-quality-site-packages
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install .[all,quality] - run: pip install -U .[all,quality]
- save_cache: - save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }} key: v0.6-code_quality-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run: - run:
name: Show installed libraries and their versions name: Show installed libraries and their versions
command: pip freeze | tee installed.txt command: pip freeze | tee installed.txt
@ -171,12 +179,20 @@ jobs:
keys: keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }} - v0.6-repository_consistency-{{ checksum "setup.py" }}
- v0.6-repository_consistency - v0.6-repository_consistency
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.6-repository_consistency-site-packages
- run: pip install --upgrade pip - run: pip install --upgrade pip
- run: pip install .[all,quality] - run: pip install -U .[all,quality]
- save_cache: - save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }} key: v0.6-repository_consistency-{{ checksum "setup.py" }}
paths: paths:
- '~/.cache/pip' - '~/.cache/pip'
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run: - run:
name: Show installed libraries and their versions name: Show installed libraries and their versions
command: pip freeze | tee installed.txt command: pip freeze | tee installed.txt

84
.circleci/create_circleci_config.py
View File

@ -106,10 +106,18 @@ class CircleCIJob:
] ]
} }
}, },
{
"restore_cache": {
"keys": [
f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
f"v{self.cache_version}-{self.cache_name}-site-packages",
]
}
},
] ]
steps.extend([{"run": l} for l in self.install_steps]) steps.extend([{"run": l} for l in self.install_steps])
# TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub` # TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install git+https://github.com/huggingface/huggingface_hub.git@c36817701ecd4694b290178846176da2e195d838"}}) steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@c36817701ecd4694b290178846176da2e195d838"}})
steps.append( steps.append(
{ {
"save_cache": { "save_cache": {
@ -118,6 +126,14 @@ class CircleCIJob:
} }
} }
) )
steps.append(
{
"save_cache": {
"key": f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
"paths": ["~/.pyenv/versions/"],
}
}
)
steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}}) steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}}) steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
@ -229,9 +245,9 @@ torch_and_tf_job = CircleCIJob(
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
"git lfs install", "git lfs install",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]", "pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install tensorflow_probability", "pip install -U tensorflow_probability",
"pip install git+https://github.com/huggingface/accelerate", "pip install -U git+https://github.com/huggingface/accelerate",
], ],
marker="is_pt_tf_cross_test", marker="is_pt_tf_cross_test",
pytest_options={"rA": None, "durations": 0}, pytest_options={"rA": None, "durations": 0},
@ -243,9 +259,9 @@ torch_and_flax_job = CircleCIJob(
additional_env={"RUN_PT_FLAX_CROSS_TESTS": True}, additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip", "pip install -U --upgrade pip",
"pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]", "pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install git+https://github.com/huggingface/accelerate", "pip install -U git+https://github.com/huggingface/accelerate",
], ],
marker="is_pt_flax_cross_test", marker="is_pt_flax_cross_test",
pytest_options={"rA": None, "durations": 0}, pytest_options={"rA": None, "durations": 0},
@ -257,8 +273,8 @@ torch_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]", "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install git+https://github.com/huggingface/accelerate", "pip install -U git+https://github.com/huggingface/accelerate",
], ],
parallelism=1, parallelism=1,
pytest_num_workers=3, pytest_num_workers=3,
@ -270,8 +286,8 @@ tf_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]", "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install tensorflow_probability", "pip install -U tensorflow_probability",
], ],
parallelism=1, parallelism=1,
pytest_num_workers=6, pytest_num_workers=6,
@ -284,7 +300,7 @@ flax_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[flax,testing,sentencepiece,flax-speech,vision]", "pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
], ],
parallelism=1, parallelism=1,
pytest_options={"rA": None}, pytest_options={"rA": None},
@ -297,7 +313,7 @@ pipelines_torch_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]", "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
], ],
pytest_options={"rA": None}, pytest_options={"rA": None},
marker="is_pipeline_test", marker="is_pipeline_test",
@ -310,8 +326,8 @@ pipelines_tf_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake", "sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,tf-cpu,testing,sentencepiece,vision]", "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install tensorflow_probability", "pip install -U tensorflow_probability",
], ],
pytest_options={"rA": None}, pytest_options={"rA": None},
marker="is_pipeline_test", marker="is_pipeline_test",
@ -334,7 +350,7 @@ custom_tokenizers_job = CircleCIJob(
"sudo make install\n", "sudo make install\n",
}, },
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]", "pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"python -m unidic download", "python -m unidic download",
], ],
parallelism=None, parallelism=None,
@ -353,8 +369,8 @@ examples_torch_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,torch,sentencepiece,testing,torch-speech]", "pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -r examples/pytorch/_tests_requirements.txt", "pip install -U -r examples/pytorch/_tests_requirements.txt",
], ],
) )
@ -365,8 +381,8 @@ examples_tensorflow_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake", "sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[sklearn,tensorflow,sentencepiece,testing]", "pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -r examples/tensorflow/_tests_requirements.txt", "pip install -U -r examples/tensorflow/_tests_requirements.txt",
], ],
) )
@ -376,8 +392,8 @@ examples_flax_job = CircleCIJob(
cache_name="flax_examples", cache_name="flax_examples",
install_steps=[ install_steps=[
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[flax,testing,sentencepiece]", "pip install -U .[flax,testing,sentencepiece]",
"pip install -r examples/flax/_tests_requirements.txt", "pip install -U -r examples/flax/_tests_requirements.txt",
], ],
) )
@ -389,7 +405,7 @@ hub_job = CircleCIJob(
'git config --global user.email "ci@dummy.com"', 'git config --global user.email "ci@dummy.com"',
'git config --global user.name "ci"', 'git config --global user.name "ci"',
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[torch,sentencepiece,testing]", "pip install -U .[torch,sentencepiece,testing]",
], ],
marker="is_staging_test", marker="is_staging_test",
pytest_num_workers=1, pytest_num_workers=1,
@ -401,7 +417,7 @@ onnx_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake", "sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]", "pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
], ],
pytest_options={"k onnx": None}, pytest_options={"k onnx": None},
pytest_num_workers=1, pytest_num_workers=1,
@ -413,13 +429,13 @@ exotic_models_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[torch,testing,vision]", "pip install -U .[torch,testing,vision]",
"pip install torchvision", "pip install -U torchvision",
"pip install scipy", "pip install -U scipy",
"pip install 'git+https://github.com/facebookresearch/detectron2.git'", "pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
"sudo apt install tesseract-ocr", "sudo apt install tesseract-ocr",
"pip install pytesseract", "pip install -U pytesseract",
"pip install natten", "pip install -U natten",
], ],
tests_to_run=[ tests_to_run=[
"tests/models/*layoutlmv*", "tests/models/*layoutlmv*",
@ -435,7 +451,7 @@ repo_utils_job = CircleCIJob(
"repo_utils", "repo_utils",
install_steps=[ install_steps=[
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install .[quality,testing,torch]", "pip install -U .[quality,testing,torch]",
], ],
parallelism=None, parallelism=None,
pytest_num_workers=1, pytest_num_workers=1,
@ -456,10 +472,10 @@ doc_test_job = CircleCIJob(
install_steps=[ install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg", "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
"pip install --upgrade pip", "pip install --upgrade pip",
"pip install -e .[dev]", "pip install -U -e .[dev]",
"pip install git+https://github.com/huggingface/accelerate", "pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade pytest pytest-sugar", "pip install --upgrade pytest pytest-sugar",
"pip install natten", "pip install -U natten",
"find -name __pycache__ -delete", "find -name __pycache__ -delete",
"find . -name \*.pyc -delete", "find . -name \*.pyc -delete",
# Add an empty file to keep the test step running correctly even no file is selected to be tested. # Add an empty file to keep the test step running correctly even no file is selected to be tested.