diff --git a/tests/models/xglm/test_modeling_tf_xglm.py b/tests/models/xglm/test_modeling_tf_xglm.py
index 88e961fca7f..7ddab81a261 100644
--- a/tests/models/xglm/test_modeling_tf_xglm.py
+++ b/tests/models/xglm/test_modeling_tf_xglm.py
@@ -239,6 +239,7 @@ class TFXGLMModelLanguageGenerationTest(unittest.TestCase):
         self.assertListEqual(expected_output_sentence, batch_out_sentence)
         self.assertListEqual(expected_output_sentence, [non_padded_sentence, padded_sentence])
 
+    @slow
     def test_loss_with_padding(self):
         tokenizer = XGLMTokenizer.from_pretrained("facebook/xglm-564M")
         model = TFXGLMForCausalLM.from_pretrained("facebook/xglm-564M")
diff --git a/tests/models/xglm/test_modeling_xglm.py b/tests/models/xglm/test_modeling_xglm.py
index 92210d04615..88f3c13497e 100644
--- a/tests/models/xglm/test_modeling_xglm.py
+++ b/tests/models/xglm/test_modeling_xglm.py
@@ -449,6 +449,7 @@ class XGLMModelLanguageGenerationTest(unittest.TestCase):
                 torch.isnan(outputs.logits[0]).any().item()
             )  # the first logits could contain NaNs if it fails
 
+    @slow
     def test_loss_with_padding(self):
         tokenizer = XGLMTokenizer.from_pretrained("facebook/xglm-564M")
         model = XGLMForCausalLM.from_pretrained("facebook/xglm-564M")