diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py
index 7360ce02f28..ea730bc00d0 100644
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -1762,11 +1762,17 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
     @lru_cache
     def _compile_jinja_template(self, chat_template):
         try:
+            import jinja2
             from jinja2.exceptions import TemplateError
             from jinja2.sandbox import ImmutableSandboxedEnvironment
         except ImportError:
             raise ImportError("apply_chat_template requires jinja2 to be installed.")
 
+        if version.parse(jinja2.__version__) <= version.parse("3.0.0"):
+            raise ImportError(
+                "apply_chat_template requires jinja2>=3.0.0 to be installed. Your version is " f"{jinja2.__version__}."
+            )
+
         def raise_exception(message):
             raise TemplateError(message)