From c1af180dfeb7777c82a00f6f0865af31ef67f970 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 21 Mar 2022 11:33:18 +0100 Subject: [PATCH] Add Slack notification support for doc tests (#16253) * up * up * up * fix * yeh * ups * Empty test commit * correct quicktour * correct * correct * up * up * uP * uP * up * up * uP * up * up * up * up * up * up * up * up * up * up * Update src/transformers/models/van/modeling_van.py * finish * apply suggestions * remove folder * revert to daily testing --- .github/workflows/doctests.yml | 41 ++- docs/source/quicktour.mdx | 5 +- utils/documentation_tests.txt | 64 ++-- utils/notification_service_doc_tests.py | 379 ++++++++++++++++++++++++ 4 files changed, 444 insertions(+), 45 deletions(-) create mode 100644 utils/notification_service_doc_tests.py diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index b0a961c612a..917945a0631 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -26,32 +26,55 @@ jobs: options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ steps: - uses: actions/checkout@v2 - with: - repository: 'huggingface/transformers' - path: transformers - - name: NVIDIA-SMI run: | nvidia-smi - name: GPU visibility - working-directory: transformers run: | utils/print_env_pt.py TF_CPP_MIN_LOG_LEVEL=3 python3 -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))" TF_CPP_MIN_LOG_LEVEL=3 python3 -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))" - name: Prepare files for doctests - working-directory: transformers run: | python3 utils/prepare_for_doc_test.py src docs - name: Run doctests - working-directory: transformers run: | - python3 -m pytest --doctest-modules $(cat utils/documentation_tests.txt) -sv --doctest-continue-on-failure --doctest-glob="*.mdx" + python3 -m pytest -v --make-reports doc_tests_gpu --doctest-modules $(cat utils/documentation_tests.txt) -sv --doctest-continue-on-failure --doctest-glob="*.mdx" - name: Clean files after doctests - working-directory: transformers run: | python3 utils/prepare_for_doc_test.py src docs --remove_new_line + + - name: Failure short reports + if: ${{ failure() }} + continue-on-error: true + run: cat reports/doc_tests_gpu/failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: doc_tests_gpu_test_reports + path: reports/doc_tests_gpu + + + send_results: + name: Send results to webhook + runs-on: ubuntu-latest + if: always() + needs: [run_doctests] + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + - name: Send message to Slack + env: + CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }} + CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_DOCS }} + CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY_DOCS }} + CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} + run: | + pip install slack_sdk + python utils/notification_service_doc_tests.py diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx index 8d26f915514..1fc4f8b865d 100644 --- a/docs/source/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -308,10 +308,7 @@ The model outputs the final activations in the `logits` attribute. Apply the sof >>> import tensorflow as tf >>> tf_predictions = tf.nn.softmax(tf_outputs.logits, axis=-1) ->>> print(tf.math.round(tf_predictions * 10**4) / 10**4) -tf.Tensor( -[[0.0021 0.0018 0.0116 0.2121 0.7725] - [0.2084 0.1826 0.1969 0.1755 0.2365]], shape=(2, 5), dtype=float32) +>>> tf_predictions # doctest: +IGNORE_RESULT ``` diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index c425e5fbd7d..56b68930b49 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -1,36 +1,36 @@ +docs/source/quicktour.mdx +docs/source/task_summary.mdx +src/transformers/generation_utils.py +src/transformers/models/bart/modeling_bart.py +src/transformers/models/beit/modeling_beit.py +src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py +src/transformers/models/blenderbot/modeling_blenderbot.py +src/transformers/models/blenderbot_small/modeling_blenderbot_small.py +src/transformers/models/convnext/modeling_convnext.py +src/transformers/models/data2vec/modeling_data2vec_audio.py +src/transformers/models/deit/modeling_deit.py +src/transformers/models/hubert/modeling_hubert.py +src/transformers/models/marian/modeling_marian.py +src/transformers/models/mbart/modeling_mbart.py +src/transformers/models/pegasus/modeling_pegasus.py +src/transformers/models/plbart/modeling_plbart.py +src/transformers/models/poolformer/modeling_poolformer.py +src/transformers/models/resnet/modeling_resnet.py +src/transformers/models/segformer/modeling_segformer.py +src/transformers/models/sew/modeling_sew.py +src/transformers/models/sew_d/modeling_sew_d.py +src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +src/transformers/models/speech_to_text/modeling_speech_to_text.py +src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py +src/transformers/models/swin/modeling_swin.py +src/transformers/models/unispeech/modeling_unispeech.py +src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +src/transformers/models/van/modeling_van.py +src/transformers/models/vilt/modeling_vilt.py +src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +src/transformers/models/vit/modeling_vit.py +src/transformers/models/vit_mae/modeling_vit_mae.py src/transformers/models/wav2vec2/modeling_wav2vec2.py src/transformers/models/wav2vec2/tokenization_wav2vec2.py src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py -src/transformers/models/hubert/modeling_hubert.py src/transformers/models/wavlm/modeling_wavlm.py -src/transformers/models/unispeech/modeling_unispeech.py -src/transformers/models/unispeech_sat/modeling_unispeech_sat.py -src/transformers/models/sew/modeling_sew.py -src/transformers/models/sew_d/modeling_sew_d.py -src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py -src/transformers/models/speech_to_text/modeling_speech_to_text.py -src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py -src/transformers/models/data2vec/modeling_data2vec_audio.py -src/transformers/models/vit/modeling_vit.py -src/transformers/models/beit/modeling_beit.py -src/transformers/models/deit/modeling_deit.py -src/transformers/models/swin/modeling_swin.py -src/transformers/models/convnext/modeling_convnext.py -src/transformers/models/poolformer/modeling_poolformer.py -src/transformers/models/vit_mae/modeling_vit_mae.py -src/transformers/models/vilt/modeling_vilt.py -src/transformers/models/van/modeling_van.py -src/transformers/models/segformer/modeling_segformer.py -src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py -src/transformers/models/bart/modeling_bart.py -src/transformers/models/mbart/modeling_mbart.py -src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py -src/transformers/models/marian/modeling_marian.py -src/transformers/models/pegasus/modeling_pegasus.py -src/transformers/models/blenderbot/modeling_blenderbot.py -src/transformers/models/blenderbot_small/modeling_blenderbot_small.py -src/transformers/models/plbart/modeling_plbart.py -src/transformers/generation_utils.py -docs/source/quicktour.mdx -docs/source/task_summary.mdx -src/transformers/models/resnet/modeling_resnet.py diff --git a/utils/notification_service_doc_tests.py b/utils/notification_service_doc_tests.py new file mode 100644 index 00000000000..58ceb567adb --- /dev/null +++ b/utils/notification_service_doc_tests.py @@ -0,0 +1,379 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import json +import math +import os +import re +import time +from fnmatch import fnmatch +from typing import Dict + +import requests +from slack_sdk import WebClient + + +client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"]) + + +def handle_test_results(test_results): + expressions = test_results.split(" ") + + failed = 0 + success = 0 + + # When the output is short enough, the output is surrounded by = signs: "== OUTPUT ==" + # When it is too long, those signs are not present. + time_spent = expressions[-2] if "=" in expressions[-1] else expressions[-1] + + for i, expression in enumerate(expressions): + if "failed" in expression: + failed += int(expressions[i - 1]) + if "passed" in expression: + success += int(expressions[i - 1]) + + return failed, success, time_spent + + +def extract_first_line_failure(failures_short_lines): + failures = {} + file = None + in_error = False + for line in failures_short_lines.split("\n"): + if re.search("_ \[doctest\]", line): + in_error = True + file = line.split(" ")[2] + elif in_error and not line.split(" ")[0].isdigit(): + failures[file] = line + in_error = False + + return failures + + +class Message: + def __init__(self, title: str, doc_test_results: Dict): + self.title = title + + self._time_spent = doc_test_results["time_spent"].split(",")[0] + self.n_success = doc_test_results["success"] + self.n_failures = doc_test_results["failures"] + self.n_tests = self.n_success + self.n_failures + + # Failures and success of the modeling tests + self.doc_test_results = doc_test_results + + @property + def time(self) -> str: + time_spent = [self._time_spent] + total_secs = 0 + + for time in time_spent: + time_parts = time.split(":") + + # Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute. + if len(time_parts) == 1: + time_parts = [0, 0, time_parts[0]] + + hours, minutes, seconds = int(time_parts[0]), int(time_parts[1]), float(time_parts[2]) + total_secs += hours * 3600 + minutes * 60 + seconds + + hours, minutes, seconds = total_secs // 3600, (total_secs % 3600) // 60, total_secs % 60 + return f"{int(hours)}h{int(minutes)}m{int(seconds)}s" + + @property + def header(self) -> Dict: + return {"type": "header", "text": {"type": "plain_text", "text": self.title}} + + @property + def no_failures(self) -> Dict: + return { + "type": "section", + "text": { + "type": "plain_text", + "text": f"🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.", + "emoji": True, + }, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check Action results", "emoji": True}, + "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}", + }, + } + + @property + def failures(self) -> Dict: + return { + "type": "section", + "text": { + "type": "plain_text", + "text": f"There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in {self.time}.", + "emoji": True, + }, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check Action results", "emoji": True}, + "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}", + }, + } + + @property + def category_failures(self) -> Dict: + line_length = 40 + category_failures = {k: v["failed"] for k, v in doc_test_results.items() if isinstance(v, dict)} + + report = "" + for category, failures in category_failures.items(): + if len(failures) == 0: + continue + + if report != "": + report += "\n\n" + + report += f"*{category} failures*:".ljust(line_length // 2).rjust(line_length // 2) + "\n" + report += "`" + report += "`\n`".join(failures) + report += "`" + + return { + "type": "section", + "text": { + "type": "mrkdwn", + "text": f"The following examples had failures:\n\n\n{report}\n", + }, + } + + @property + def payload(self) -> str: + blocks = [self.header] + + if self.n_failures > 0: + blocks.append(self.failures) + + if self.n_failures > 0: + blocks.extend([self.category_failures]) + + if self.no_failures == 0: + blocks.append(self.no_failures) + + return json.dumps(blocks) + + @staticmethod + def error_out(): + payload = [ + { + "type": "section", + "text": { + "type": "plain_text", + "text": "There was an issue running the tests.", + }, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check Action results", "emoji": True}, + "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}", + }, + } + ] + + print("Sending the following payload") + print(json.dumps({"blocks": json.loads(payload)})) + + client.chat_postMessage( + channel=os.environ["CI_SLACK_CHANNEL_ID_DAILY"], + text="There was an issue running the tests.", + blocks=payload, + ) + + def post(self): + print("Sending the following payload") + print(json.dumps({"blocks": json.loads(self.payload)})) + + text = f"{self.n_failures} failures out of {self.n_tests} tests," if self.n_failures else "All tests passed." + + self.thread_ts = client.chat_postMessage( + channel=os.environ["CI_SLACK_CHANNEL_ID_DAILY"], + blocks=self.payload, + text=text, + ) + + def get_reply_blocks(self, job_name, job_link, failures, text): + failures_text = "" + for key, value in failures.items(): + value = value[:200] + " [Truncated]" if len(value) > 250 else value + failures_text += f"*{key}*\n_{value}_\n\n" + + title = job_name + content = {"type": "section", "text": {"type": "mrkdwn", "text": text}} + + if job_link is not None: + content["accessory"] = { + "type": "button", + "text": {"type": "plain_text", "text": "GitHub Action job", "emoji": True}, + "url": job_link, + } + + return [ + {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}}, + content, + {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}}, + ] + + def post_reply(self): + if self.thread_ts is None: + raise ValueError("Can only post reply if a post has been made.") + + job_link = self.doc_test_results.pop("job_link") + self.doc_test_results.pop("failures") + self.doc_test_results.pop("success") + self.doc_test_results.pop("time_spent") + + sorted_dict = sorted(self.doc_test_results.items(), key=lambda t: t[0]) + for job, job_result in sorted_dict: + if len(job_result["failures"]): + text = f"*Num failures* :{len(job_result['failed'])} \n" + failures = job_result["failures"] + blocks = self.get_reply_blocks(job, job_link, failures, text=text) + + print("Sending the following reply") + print(json.dumps({"blocks": blocks})) + + client.chat_postMessage( + channel=os.environ["CI_SLACK_CHANNEL_ID_DAILY"], + text=f"Results for {job}", + blocks=blocks, + thread_ts=self.thread_ts["ts"], + ) + + time.sleep(1) + + +def get_job_links(): + run_id = os.environ["GITHUB_RUN_ID"] + url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100" + result = requests.get(url).json() + jobs = {} + + try: + jobs.update({job["name"]: job["html_url"] for job in result["jobs"]}) + pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100) + + for i in range(pages_to_iterate_over): + result = requests.get(url + f"&page={i + 2}").json() + jobs.update({job["name"]: job["html_url"] for job in result["jobs"]}) + + return jobs + except Exception as e: + print("Unknown error, could not fetch links.", e) + + return {} + + +def retrieve_artifact(name: str): + _artifact = {} + + if os.path.exists(name): + files = os.listdir(name) + for file in files: + try: + with open(os.path.join(name, file)) as f: + _artifact[file.split(".")[0]] = f.read() + except UnicodeDecodeError as e: + raise ValueError(f"Could not open {os.path.join(name, file)}.") from e + + return _artifact + + +def retrieve_available_artifacts(): + class Artifact: + def __init__(self, name: str): + self.name = name + self.paths = [] + + def __str__(self): + return self.name + + def add_path(self, path: str): + self.paths.append({"name": self.name, "path": path}) + + _available_artifacts: Dict[str, Artifact] = {} + + directories = filter(os.path.isdir, os.listdir()) + for directory in directories: + artifact_name = directory + if artifact_name not in _available_artifacts: + _available_artifacts[artifact_name] = Artifact(artifact_name) + + _available_artifacts[artifact_name].add_path(directory) + + return _available_artifacts + + +if __name__ == "__main__": + + github_actions_job_links = get_job_links() + available_artifacts = retrieve_available_artifacts() + + docs = collections.OrderedDict( + [ + ("*.py", "API Examples"), + ("*.mdx", "MDX Examples"), + ] + ) + + # This dict will contain all the information relative to each doc test category: + # - failed: list of failed tests + # - failures: dict in the format 'test': 'error_message' + doc_test_results = { + v: { + "failed": [], + "failures": {}, + } + for v in docs.values() + } + + # Link to the GitHub Action job + doc_test_results["job_link"] = github_actions_job_links.get("run_doctests") + + artifact_path = available_artifacts["doc_tests_gpu_test_reports"].paths[0] + artifact = retrieve_artifact(artifact_path["name"]) + if "stats" in artifact: + failed, success, time_spent = handle_test_results(artifact["stats"]) + doc_test_results["failures"] = failed + doc_test_results["success"] = success + doc_test_results["time_spent"] = time_spent[1:-1] + ", " + + all_failures = extract_first_line_failure(artifact["failures_short"]) + for line in artifact["summary_short"].split("\n"): + if re.search("FAILED", line): + + line = line.replace("FAILED ", "") + line = line.split()[0].replace("\n", "") + + if "::" in line: + file_path, test = line.split("::") + else: + file_path, test = line, line + + for file_regex in docs.keys(): + if fnmatch(file_path, file_regex): + category = docs[file_regex] + doc_test_results[category]["failed"].append(test) + + failure = all_failures[test] if test in all_failures else "N/A" + doc_test_results[category]["failures"][test] = failure + break + + message = Message("🤗 Results of the doc tests.", doc_test_results) + message.post() + message.post_reply()