Fix slack report failing for doctest (#27042)

* fix slack report for doctest

* separate reports

* style

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

utils/notification_service.py

@@ -314,7 +314,7 @@ class Message:
         return entries_changed
 
     @property
-    def model_failures(self) -> Dict:
+    def model_failures(self) -> List[Dict]:
         # Obtain per-model failures
         def per_model_sum(model_category_dict):
             return dicts_to_sum(model_category_dict["failed"].values())

utils/notification_service_doc_tests.py

@@ -19,7 +19,7 @@ import os
 import re
 import time
 from fnmatch import fnmatch
-from typing import Dict
+from typing import Dict, List
 
 import requests
 from slack_sdk import WebClient
@@ -132,30 +132,42 @@ class Message:
         }
 
     @property
-    def category_failures(self) -> Dict:
+    def category_failures(self) -> List[Dict]:
+        failure_blocks = []
+
+        MAX_ERROR_TEXT = 3000 - len("The following examples had failures:\n\n\n\n") - len("[Truncated]\n")
         line_length = 40
         category_failures = {k: v["failed"] for k, v in doc_test_results.items() if isinstance(v, dict)}
 
-        report = ""
+        def single_category_failures(category, failures):
-        for category, failures in category_failures.items():
+            text = ""
             if len(failures) == 0:
+                return ""
+            text += f"*{category} failures*:".ljust(line_length // 2).rjust(line_length // 2) + "\n"
+
+            for idx, failure in enumerate(failures):
+                new_text = text + f"`{failure}`\n"
+                if len(new_text) > MAX_ERROR_TEXT:
+                    text = text + "[Truncated]\n"
+                    break
+                text = new_text
+
+            return text
+
+        for category, failures in category_failures.items():
+            report = single_category_failures(category, failures)
+            if len(report) == 0:
                 continue
+            block = {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": f"The following examples had failures:\n\n\n{report}\n",
+                },
+            }
+            failure_blocks.append(block)
 
-            if report != "":
+        return failure_blocks
-                report += "\n\n"
-
-            report += f"*{category} failures*:".ljust(line_length // 2).rjust(line_length // 2) + "\n"
-            report += "`"
-            report += "`\n`".join(failures)
-            report += "`"
-
-        return {
-            "type": "section",
-            "text": {
-                "type": "mrkdwn",
-                "text": f"The following examples had failures:\n\n\n{report}\n",
-            },
-        }
 
     @property
     def payload(self) -> str:
@@ -165,7 +177,7 @@ class Message:
             blocks.append(self.failures)
 
         if self.n_failures > 0:
-            blocks.extend([self.category_failures])
+            blocks.extend(self.category_failures)
 
         if self.n_failures == 0:
             blocks.append(self.no_failures)
@@ -211,10 +223,19 @@ class Message:
         )
 
     def get_reply_blocks(self, job_name, job_link, failures, text):
-        failures_text = ""
+        # `text` must be less than 3001 characters in Slack SDK
+        # keep some room for adding "[Truncated]" when necessary
+        MAX_ERROR_TEXT = 3000 - len("[Truncated]")
+
+        failure_text = ""
         for key, value in failures.items():
-            value = value[:200] + " [Truncated]" if len(value) > 250 else value
+            new_text = failure_text + f"*{key}*\n_{value}_\n\n"
-            failures_text += f"*{key}*\n_{value}_\n\n"
+            if len(new_text) > MAX_ERROR_TEXT:
+                # `failure_text` here has length <= 3000
+                failure_text = failure_text + "[Truncated]"
+                break
+            # `failure_text` here has length <= MAX_ERROR_TEXT
+            failure_text = new_text
 
         title = job_name
         content = {"type": "section", "text": {"type": "mrkdwn", "text": text}}
@@ -229,7 +250,7 @@ class Message:
         return [
             {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
             content,
-            {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
+            {"type": "section", "text": {"type": "mrkdwn", "text": failure_text}},
         ]
 
     def post_reply(self):