diff --git a/docs/source/conf.py b/docs/source/conf.py
index 91c226b755d..d0f7339c1cc 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'2.3.0'
+release = u'2.4.0'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/setup.py b/setup.py
index 6f1bf280f9f..2f82c3f240b 100644
--- a/setup.py
+++ b/setup.py
@@ -73,7 +73,7 @@ extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "sciki
 
 setup(
     name="transformers",
-    version="2.3.0",
+    version="2.4.0",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index 13990695ce2..a1888a4424a 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "2.3.0"
+__version__ = "2.4.0"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.