diff --git a/.circleci/config.yml b/.circleci/config.yml
index b7a0652b9fa..99d0e05c645 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,8 +31,8 @@ jobs:
         steps:
             - checkout
             - run: pip install --upgrade pip
-            - run: pip install GitPython
-            - run: pip install .
+            - run: pip install -U GitPython
+            - run: pip install -U .
             - run: mkdir -p test_preparation
             - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
             - store_artifacts:
@@ -105,8 +105,8 @@ jobs:
         steps:
             - checkout
             - run: pip install --upgrade pip
-            - run: pip install GitPython
-            - run: pip install .
+            - run: pip install -U GitPython
+            - run: pip install -U .
             - run: |
                   mkdir test_preparation
                   echo -n "tests" > test_preparation/test_list.txt
@@ -138,12 +138,20 @@ jobs:
                   keys:
                       - v0.6-code_quality-{{ checksum "setup.py" }}
                       - v0.6-code-quality
+            - restore_cache:
+                  keys:
+                      - v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
+                      - v0.6-code-quality-site-packages
             - run: pip install --upgrade pip
-            - run: pip install .[all,quality]
+            - run: pip install -U .[all,quality]
             - save_cache:
                   key: v0.6-code_quality-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
+            - save_cache:
+                  key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
+                  paths:
+                      - '~/.pyenv/versions/'
             - run:
                 name: Show installed libraries and their versions
                 command: pip freeze | tee installed.txt
@@ -171,12 +179,20 @@ jobs:
                   keys:
                       - v0.6-repository_consistency-{{ checksum "setup.py" }}
                       - v0.6-repository_consistency
+            - restore_cache:
+                  keys:
+                      - v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
+                      - v0.6-repository_consistency-site-packages
             - run: pip install --upgrade pip
-            - run: pip install .[all,quality]
+            - run: pip install -U .[all,quality]
             - save_cache:
                   key: v0.6-repository_consistency-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
+            - save_cache:
+                  key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
+                  paths:
+                      - '~/.pyenv/versions/'
             - run:
                 name: Show installed libraries and their versions
                 command: pip freeze | tee installed.txt
diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py
index 0b7923ef676..85552297b1c 100644
--- a/.circleci/create_circleci_config.py
+++ b/.circleci/create_circleci_config.py
@@ -106,10 +106,18 @@ class CircleCIJob:
                     ]
                 }
             },
+            {
+                "restore_cache": {
+                    "keys": [
+                        f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
+                        f"v{self.cache_version}-{self.cache_name}-site-packages",
+                    ]
+                }
+            },
         ]
         steps.extend([{"run": l} for l in self.install_steps])
         # TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
-        steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install git+https://github.com/huggingface/huggingface_hub.git@c36817701ecd4694b290178846176da2e195d838"}})
+        steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@c36817701ecd4694b290178846176da2e195d838"}})
         steps.append(
             {
                 "save_cache": {
@@ -118,6 +126,14 @@ class CircleCIJob:
                 }
             }
         )
+        steps.append(
+            {
+                "save_cache": {
+                    "key": f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
+                    "paths": ["~/.pyenv/versions/"],
+                }
+            }
+        )
         steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
         steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
 
@@ -229,9 +245,9 @@ torch_and_tf_job = CircleCIJob(
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
         "git lfs install",
         "pip install --upgrade pip",
-        "pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
-        "pip install tensorflow_probability",
-        "pip install git+https://github.com/huggingface/accelerate",
+        "pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
+        "pip install -U tensorflow_probability",
+        "pip install -U git+https://github.com/huggingface/accelerate",
     ],
     marker="is_pt_tf_cross_test",
     pytest_options={"rA": None, "durations": 0},
@@ -243,9 +259,9 @@ torch_and_flax_job = CircleCIJob(
     additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
-        "pip install --upgrade pip",
-        "pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
-        "pip install git+https://github.com/huggingface/accelerate",
+        "pip install -U --upgrade pip",
+        "pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
+        "pip install -U git+https://github.com/huggingface/accelerate",
     ],
     marker="is_pt_flax_cross_test",
     pytest_options={"rA": None, "durations": 0},
@@ -257,8 +273,8 @@ torch_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
         "pip install --upgrade pip",
-        "pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
-        "pip install git+https://github.com/huggingface/accelerate",
+        "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
+        "pip install -U git+https://github.com/huggingface/accelerate",
     ],
     parallelism=1,
     pytest_num_workers=3,
@@ -270,8 +286,8 @@ tf_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
         "pip install --upgrade pip",
-        "pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
-        "pip install tensorflow_probability",
+        "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
+        "pip install -U tensorflow_probability",
     ],
     parallelism=1,
     pytest_num_workers=6,
@@ -284,7 +300,7 @@ flax_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
         "pip install --upgrade pip",
-        "pip install .[flax,testing,sentencepiece,flax-speech,vision]",
+        "pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
     ],
     parallelism=1,
     pytest_options={"rA": None},
@@ -297,7 +313,7 @@ pipelines_torch_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
         "pip install --upgrade pip",
-        "pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
+        "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
     ],
     pytest_options={"rA": None},
     marker="is_pipeline_test",
@@ -310,8 +326,8 @@ pipelines_tf_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y cmake",
         "pip install --upgrade pip",
-        "pip install .[sklearn,tf-cpu,testing,sentencepiece,vision]",
-        "pip install tensorflow_probability",
+        "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
+        "pip install -U tensorflow_probability",
     ],
     pytest_options={"rA": None},
     marker="is_pipeline_test",
@@ -334,7 +350,7 @@ custom_tokenizers_job = CircleCIJob(
                 "sudo make install\n",
         },
         "pip install --upgrade pip",
-        "pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
+        "pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
         "python -m unidic download",
     ],
     parallelism=None,
@@ -353,8 +369,8 @@ examples_torch_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
         "pip install --upgrade pip",
-        "pip install .[sklearn,torch,sentencepiece,testing,torch-speech]",
-        "pip install -r examples/pytorch/_tests_requirements.txt",
+        "pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
+        "pip install -U -r examples/pytorch/_tests_requirements.txt",
     ],
 )
 
@@ -365,8 +381,8 @@ examples_tensorflow_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y cmake",
         "pip install --upgrade pip",
-        "pip install .[sklearn,tensorflow,sentencepiece,testing]",
-        "pip install -r examples/tensorflow/_tests_requirements.txt",
+        "pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
+        "pip install -U -r examples/tensorflow/_tests_requirements.txt",
     ],
 )
 
@@ -376,8 +392,8 @@ examples_flax_job = CircleCIJob(
     cache_name="flax_examples",
     install_steps=[
         "pip install --upgrade pip",
-        "pip install .[flax,testing,sentencepiece]",
-        "pip install -r examples/flax/_tests_requirements.txt",
+        "pip install -U .[flax,testing,sentencepiece]",
+        "pip install -U -r examples/flax/_tests_requirements.txt",
     ],
 )
 
@@ -389,7 +405,7 @@ hub_job = CircleCIJob(
         'git config --global user.email "ci@dummy.com"',
         'git config --global user.name "ci"',
         "pip install --upgrade pip",
-        "pip install .[torch,sentencepiece,testing]",
+        "pip install -U .[torch,sentencepiece,testing]",
     ],
     marker="is_staging_test",
     pytest_num_workers=1,
@@ -401,7 +417,7 @@ onnx_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y cmake",
         "pip install --upgrade pip",
-        "pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
+        "pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
     ],
     pytest_options={"k onnx": None},
     pytest_num_workers=1,
@@ -413,13 +429,13 @@ exotic_models_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
         "pip install --upgrade pip",
-        "pip install .[torch,testing,vision]",
-        "pip install torchvision",
-        "pip install scipy",
-        "pip install 'git+https://github.com/facebookresearch/detectron2.git'",
+        "pip install -U .[torch,testing,vision]",
+        "pip install -U torchvision",
+        "pip install -U scipy",
+        "pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
         "sudo apt install tesseract-ocr",
-        "pip install pytesseract",
-        "pip install natten",
+        "pip install -U pytesseract",
+        "pip install -U natten",
     ],
     tests_to_run=[
         "tests/models/*layoutlmv*",
@@ -435,7 +451,7 @@ repo_utils_job = CircleCIJob(
     "repo_utils",
     install_steps=[
         "pip install --upgrade pip",
-        "pip install .[quality,testing,torch]",
+        "pip install -U .[quality,testing,torch]",
     ],
     parallelism=None,
     pytest_num_workers=1,
@@ -456,10 +472,10 @@ doc_test_job = CircleCIJob(
     install_steps=[
         "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
         "pip install --upgrade pip",
-        "pip install -e .[dev]",
-        "pip install git+https://github.com/huggingface/accelerate",
+        "pip install -U -e .[dev]",
+        "pip install -U git+https://github.com/huggingface/accelerate",
         "pip install --upgrade pytest pytest-sugar",
-        "pip install natten",
+        "pip install -U natten",
         "find -name __pycache__ -delete",
         "find . -name \*.pyc -delete",
         # Add an empty file to keep the test step running correctly even no file is selected to be tested.