diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 59fc111af13..374b57a9047 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -398,6 +398,56 @@ jobs: name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu + run_extract_warnings: + name: Extract warnings in CI artifacts + runs-on: ubuntu-latest + if: always() + needs: [ + check_runner_status, + check_runners, + setup, + run_tests_single_gpu, + run_tests_multi_gpu, + run_examples_gpu, + run_pipelines_tf_gpu, + run_pipelines_torch_gpu, + run_all_tests_torch_cuda_extensions_gpu + ] + steps: + - name: Checkout transformers + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Install transformers + run: pip install transformers + + - name: Show installed libraries and their versions + run: pip freeze + + - name: Create output directory + run: mkdir warnings_in_ci + + - uses: actions/download-artifact@v2 + with: + path: warnings_in_ci + + - name: Show artifacts + run: echo "$(python3 -c 'import os; d = os.listdir(); print(d)')" + working-directory: warnings_in_ci + + - name: Extract warnings in CI artifacts + run: | + python3 utils/extract_warnings.py --workflow_run_id ${{ github.run_id }} --output_dir warnings_in_ci --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }} --from_gh + echo "$(python3 -c 'import os; import json; fp = open("warnings_in_ci/selected_warnings.json"); d = json.load(fp); d = "\n".join(d) ;print(d)')" + + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: warnings_in_ci + path: warnings_in_ci/selected_warnings.json + send_results: name: Send results to webhook runs-on: ubuntu-latest @@ -411,7 +461,8 @@ jobs: run_examples_gpu, run_pipelines_tf_gpu, run_pipelines_torch_gpu, - run_all_tests_torch_cuda_extensions_gpu + run_all_tests_torch_cuda_extensions_gpu, + run_extract_warnings ] steps: - name: Preliminary job status diff --git a/utils/extract_warnings.py b/utils/extract_warnings.py index bc795c53f74..1f46bc5538f 100644 --- a/utils/extract_warnings.py +++ b/utils/extract_warnings.py @@ -11,40 +11,54 @@ from transformers import logging logger = logging.get_logger(__name__) -def extract_warnings_from_single_artifact(artifact_zip_path, targets): +def extract_warnings_from_single_artifact(artifact_path, targets): """Extract warnings from a downloaded artifact (in .zip format)""" selected_warnings = set() buffer = [] - try: - with zipfile.ZipFile(artifact_zip_path) as z: - for filename in z.namelist(): - if not os.path.isdir(filename): - # read the file - if filename != "warnings.txt": - continue - with z.open(filename) as f: - for line in f: - line = line.decode("UTF-8") - if "warnings summary (final)" in line: - continue - # This means we are outside the body of a warning - elif not line.startswith(" "): - # process a single warning and move it to `selected_warnings`. - if len(buffer) > 0: - warning = "\n".join(buffer) - # Only keep the warnings specified in `targets` - if any(f": {x}: " in warning for x in targets): - selected_warnings.add(warning) - buffer = [] - continue - else: - line = line.strip() - buffer.append(line) - except Exception: - logger.warning( - f"{artifact_zip_path} is either an invalid zip file or something else wrong. This file is skipped." - ) + def parse_line(fp): + for line in fp: + if isinstance(line, bytes): + line = line.decode("UTF-8") + if "warnings summary (final)" in line: + continue + # This means we are outside the body of a warning + elif not line.startswith(" "): + # process a single warning and move it to `selected_warnings`. + if len(buffer) > 0: + warning = "\n".join(buffer) + # Only keep the warnings specified in `targets` + if any(f": {x}: " in warning for x in targets): + selected_warnings.add(warning) + buffer.clear() + continue + else: + line = line.strip() + buffer.append(line) + + if from_gh: + for filename in os.listdir(artifact_path): + file_path = os.path.join(artifact_path, filename) + if not os.path.isdir(file_path): + # read the file + if filename != "warnings.txt": + continue + with open(file_path) as fp: + parse_line(fp) + else: + try: + with zipfile.ZipFile(artifact_path) as z: + for filename in z.namelist(): + if not os.path.isdir(filename): + # read the file + if filename != "warnings.txt": + continue + with z.open(filename) as fp: + parse_line(fp) + except Exception: + logger.warning( + f"{artifact_path} is either an invalid zip file or something else wrong. This file is skipped." + ) return selected_warnings @@ -54,7 +68,7 @@ def extract_warnings(artifact_dir, targets): selected_warnings = set() - paths = [os.path.join(artifact_dir, p) for p in os.listdir(artifact_dir) if p.endswith(".zip")] + paths = [os.path.join(artifact_dir, p) for p in os.listdir(artifact_dir) if (p.endswith(".zip") or from_gh)] for p in paths: selected_warnings.update(extract_warnings_from_single_artifact(p, targets)) @@ -81,30 +95,41 @@ if __name__ == "__main__": parser.add_argument( "--token", default=None, type=str, required=True, help="A token that has actions:read permission." ) + # optional parameters parser.add_argument( "--targets", default="DeprecationWarning,UserWarning,FutureWarning", type=list_str, help="Comma-separated list of target warning(s) which we want to extract.", ) + parser.add_argument( + "--from_gh", + action="store_true", + help="If running from a GitHub action workflow and collecting warnings from its artifacts.", + ) args = parser.parse_args() - os.makedirs(args.output_dir, exist_ok=True) + from_gh = args.from_gh + if from_gh: + # The artifacts have to be downloaded using `actions/download-artifact@v2` + pass + else: + os.makedirs(args.output_dir, exist_ok=True) - # get download links - artifacts = get_artifacts_links(args.workflow_run_id) - with open(os.path.join(args.output_dir, "artifacts.json"), "w", encoding="UTF-8") as fp: - json.dump(artifacts, fp, ensure_ascii=False, indent=4) + # get download links + artifacts = get_artifacts_links(args.workflow_run_id) + with open(os.path.join(args.output_dir, "artifacts.json"), "w", encoding="UTF-8") as fp: + json.dump(artifacts, fp, ensure_ascii=False, indent=4) - # download artifacts - for idx, (name, url) in enumerate(artifacts.items()): - print(name) - print(url) - print("=" * 80) - download_artifact(name, url, args.output_dir, args.token) - # Be gentle to GitHub - time.sleep(1) + # download artifacts + for idx, (name, url) in enumerate(artifacts.items()): + print(name) + print(url) + print("=" * 80) + download_artifact(name, url, args.output_dir, args.token) + # Be gentle to GitHub + time.sleep(1) # extract warnings from artifacts selected_warnings = extract_warnings(args.output_dir, args.targets) diff --git a/utils/notification_service.py b/utils/notification_service.py index d4b5479aecd..da315dc56ae 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -98,7 +98,9 @@ def dicts_to_sum(objects: Union[Dict[str, Dict], List[dict]]): class Message: - def __init__(self, title: str, ci_title: str, model_results: Dict, additional_results: Dict): + def __init__( + self, title: str, ci_title: str, model_results: Dict, additional_results: Dict, selected_warnings: List = None + ): self.title = title self.ci_title = ci_title @@ -136,6 +138,10 @@ class Message: self.thread_ts = None + if selected_warnings is None: + selected_warnings = [] + self.selected_warnings = selected_warnings + @property def time(self) -> str: all_results = [*self.model_results.values(), *self.additional_results.values()] @@ -198,6 +204,22 @@ class Message: }, } + @property + def warnings(self) -> Dict: + return { + "type": "section", + "text": { + "type": "plain_text", + "text": f"There were {len(self.selected_warnings)} warnings being selected.", + "emoji": True, + }, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check warnings", "emoji": True}, + "url": f"{github_actions_job_links['Extract warnings in CI artifacts']}", + }, + } + @staticmethod def get_device_report(report, rjust=6): if "single" in report and "multi" in report: @@ -384,6 +406,9 @@ class Message: if self.n_model_failures == 0 and self.n_additional_failures == 0: blocks.append(self.no_failures) + if len(self.selected_warnings) > 0: + blocks.append(self.warnings) + return json.dumps(blocks) @staticmethod @@ -910,7 +935,13 @@ if __name__ == "__main__": {"line": line, "trace": stacktraces.pop(0)} ) - message = Message(title, ci_title, model_results, additional_results) + selected_warnings = [] + if "warnings_in_ci" in available_artifacts: + directory = available_artifacts["warnings_in_ci"].paths[0]["path"] + with open(os.path.join(directory, "selected_warnings.json")) as fp: + selected_warnings = json.load(fp) + + message = Message(title, ci_title, model_results, additional_results, selected_warnings=selected_warnings) # send report only if there is any failure (for push CI) if message.n_failures or ci_event != "push":