diff --git a/pyproject.toml b/pyproject.toml
index ef3be4606ea..7905afac5d6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,5 +83,6 @@ line-length = 119
 lines-after-imports = 2
 known-first-party = ["transformers"]
 
-[tool.pytest]
-doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS"
+# This is ignored, maybe because of the header? If someone finds a fix, we can uncomment and remove setup.cfg
+# [tool.pytest]
+# doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS"
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 00000000000..5f47c5c6be6
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[tool:pytest]
+doctest_optionflags=NUMBER NORMALIZE_WHITESPACE ELLIPSIS