diff --git a/setup.py b/setup.py
index 1eaee04a67b..abafc03bc6f 100644
--- a/setup.py
+++ b/setup.py
@@ -135,6 +135,7 @@ _deps = [
     "librosa",
     "nltk",
     "natten>=0.14.6",
+    "numba<0.57.0",  # Can be removed once unpinned.
     "numpy>=1.17",
     "onnxconverter-common",
     "onnxruntime-tools>=1.4.2",
@@ -286,7 +287,8 @@ extras["sigopt"] = deps_list("sigopt")
 extras["integrations"] = extras["optuna"] + extras["ray"] + extras["sigopt"]
 
 extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
-extras["audio"] = deps_list("librosa", "pyctcdecode", "phonemizer", "kenlm")
+# numba can be removed here once unpinned
+extras["audio"] = deps_list("librosa", "pyctcdecode", "phonemizer", "kenlm", "numba")
 # `pip install ".[speech]"` is deprecated and `pip install ".[torch-speech]"` should be used instead
 extras["speech"] = deps_list("torchaudio") + extras["audio"]
 extras["torch-speech"] = deps_list("torchaudio") + extras["audio"]
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index 68ad7a1587f..bae19acd3e1 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -36,6 +36,7 @@ deps = {
     "librosa": "librosa",
     "nltk": "nltk",
     "natten": "natten>=0.14.6",
+    "numba": "numba<0.57.0",
     "numpy": "numpy>=1.17",
     "onnxconverter-common": "onnxconverter-common",
     "onnxruntime-tools": "onnxruntime-tools>=1.4.2",