diff --git a/.circleci/config.yml b/.circleci/config.yml
index e54d92ab957..dfb7de5634f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,6 +1,6 @@
 version: 2
 jobs:
-    build_py3:
+    build_py3_torch:
         working_directory: ~/pytorch-transformers
         docker:
             - image: circleci/python:3.5
@@ -8,14 +8,29 @@ jobs:
         parallelism: 1
         steps:
             - checkout
+            - run: sudo pip install torch
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov
             - run: sudo pip install tensorboardX scikit-learn
-            - run: sudo pip install tensorflow==2.0.0-rc0
             - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov
             - run: python -m pytest -sv ./examples/
             - run: codecov
-    build_py2:
+    build_py3_tf:
+        working_directory: ~/pytorch-transformers
+        docker:
+            - image: circleci/python:3.5
+        resource_class: xlarge
+        parallelism: 1
+        steps:
+            - checkout
+            - run: sudo pip install tensorflow==2.0.0-rc0
+            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install pytest codecov pytest-cov
+            - run: sudo pip install tensorboardX scikit-learn
+            - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov
+            - run: python -m pytest -sv ./examples/
+            - run: codecov
+    build_py2_torch:
         working_directory: ~/pytorch-transformers
         resource_class: large
         parallelism: 1
@@ -23,9 +38,22 @@ jobs:
             - image: circleci/python:2.7
         steps:
             - checkout
+            - run: sudo pip install torch
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov
+            - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov
+            - run: codecov
+    build_py2_tf:
+        working_directory: ~/pytorch-transformers
+        resource_class: large
+        parallelism: 1
+        docker:
+            - image: circleci/python:2.7
+        steps:
+            - checkout
             - run: sudo pip install tensorflow==2.0.0-rc0
+            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install pytest codecov pytest-cov
             - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov
             - run: codecov
     deploy_doc:
@@ -49,6 +77,8 @@ workflows:
     version: 2
     build_and_test:
         jobs:
-            - build_py3
-            - build_py2
+            - build_py3_torch
+            - build_py3_tf
+            - build_py2_torch
+            - build_py2_tf
             - deploy_doc: *workflow_filters
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 01dca79d23b..9c43abc6d76 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,3 @@
-# PyTorch
-torch>=1.0.0
 # progress bars in model download and training scripts
 tqdm
 # Accessing files from S3 directly.
diff --git a/setup.py b/setup.py
index c31bf6eaeb3..903f1d8cac3 100644
--- a/setup.py
+++ b/setup.py
@@ -49,8 +49,7 @@ setup(
     url="https://github.com/huggingface/pytorch-transformers",
     packages=find_packages(exclude=["*.tests", "*.tests.*",
                                     "tests.*", "tests"]),
-    install_requires=['torch>=1.0.0',
-                      'numpy',
+    install_requires=['numpy',
                       'boto3',
                       'requests',
                       'tqdm',