diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py
index 2c5b8cca1cd..b9359251925 100644
--- a/src/transformers/models/mask2former/image_processing_mask2former.py
+++ b/src/transformers/models/mask2former/image_processing_mask2former.py
@@ -1026,9 +1026,10 @@ class Mask2FormerImageProcessor(BaseImageProcessor):
                     )
                     current_segment_id += 1
                     instance_maps.append(pred_masks[j])
-                    # Return segmentation map in run-length encoding (RLE) format
-                    if return_coco_annotation:
-                        segmentation = convert_segmentation_to_rle(segmentation)
+
+            # Return segmentation map in run-length encoding (RLE) format
+            if return_coco_annotation:
+                segmentation = convert_segmentation_to_rle(segmentation)
 
             # Return a concatenated tensor of binary instance maps
             if return_binary_maps and len(instance_maps) != 0:
diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py
index a151700f97d..b5880566748 100644
--- a/src/transformers/models/maskformer/image_processing_maskformer.py
+++ b/src/transformers/models/maskformer/image_processing_maskformer.py
@@ -1081,9 +1081,10 @@ class MaskFormerImageProcessor(BaseImageProcessor):
                     )
                     current_segment_id += 1
                     instance_maps.append(pred_masks[j])
-                    # Return segmentation map in run-length encoding (RLE) format
-                    if return_coco_annotation:
-                        segmentation = convert_segmentation_to_rle(segmentation)
+
+            # Return segmentation map in run-length encoding (RLE) format
+            if return_coco_annotation:
+                segmentation = convert_segmentation_to_rle(segmentation)
 
             # Return a concatenated tensor of binary instance maps
             if return_binary_maps and len(instance_maps) != 0: