mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-03 21:00:08 +06:00
b1375177fc
* update for job link * stye --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
99 lines
3.6 KiB
Python
99 lines
3.6 KiB
Python
"""An internal script to process `new_model_failures_with_bad_commit.json` produced by `utils/check_bad_commit.py`.
|
|
|
|
This is used by `.github/workflows/check_failed_model_tests.yml` to produce a slack report of the following form
|
|
|
|
```
|
|
<{url}|New failed tests>
|
|
{
|
|
"GH_ydshieh": {
|
|
"vit": 1
|
|
}
|
|
}
|
|
```
|
|
"""
|
|
|
|
import datetime
|
|
import json
|
|
import os
|
|
from collections import Counter
|
|
from copy import deepcopy
|
|
|
|
from huggingface_hub import HfApi
|
|
|
|
|
|
if __name__ == "__main__":
|
|
api = HfApi()
|
|
|
|
with open("new_model_failures_with_bad_commit.json") as fp:
|
|
data = json.load(fp)
|
|
|
|
with open("ci_results_run_models_gpu/model_job_links.json") as fp:
|
|
model_job_links = json.load(fp)
|
|
|
|
# TODO: extend
|
|
team_members = [
|
|
"ydshieh",
|
|
"zucchini-nlp",
|
|
"ArthurZucker",
|
|
"gante",
|
|
"LysandreJik",
|
|
"molbap",
|
|
"qubvel",
|
|
"Rocketknight1",
|
|
"muellerzr",
|
|
"SunMarc",
|
|
]
|
|
|
|
# Counting the number of failures grouped by authors
|
|
new_data = {}
|
|
for model, model_result in data.items():
|
|
for device, failed_tests in model_result.items():
|
|
for failed_test in failed_tests:
|
|
author = failed_test["author"]
|
|
|
|
if author not in team_members:
|
|
author = failed_test["merged_by"]
|
|
|
|
if author not in new_data:
|
|
new_data[author] = Counter()
|
|
new_data[author].update([model])
|
|
for author in new_data:
|
|
new_data[author] = dict(new_data[author])
|
|
|
|
# Group by author
|
|
new_data_full = {author: deepcopy(data) for author in new_data}
|
|
for author, _data in new_data_full.items():
|
|
for model, model_result in _data.items():
|
|
for device, failed_tests in model_result.items():
|
|
# prepare job_link and add it to each entry of new failed test information.
|
|
# need to change from `single-gpu` to `single` and same for `multi-gpu` to match `job_link`.
|
|
job_link = model_job_links[model][device.replace("-gpu", "")]
|
|
failed_tests = [x for x in failed_tests if x["author"] == author or x["merged_by"] == author]
|
|
for x in failed_tests:
|
|
x.update({"job_link": job_link})
|
|
model_result[device] = failed_tests
|
|
_data[model] = {k: v for k, v in model_result.items() if len(v) > 0}
|
|
new_data_full[author] = {k: v for k, v in _data.items() if len(v) > 0}
|
|
|
|
# Upload to Hub and get the url
|
|
with open("new_model_failures_with_bad_commit_grouped_by_authors.json", "w") as fp:
|
|
json.dump(new_data_full, fp, ensure_ascii=False, indent=4)
|
|
commit_info = api.upload_file(
|
|
path_or_fileobj="new_model_failures_with_bad_commit_grouped_by_authors.json",
|
|
path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_run_models_gpu/new_model_failures_with_bad_commit_grouped_by_authors.json",
|
|
repo_id="hf-internal-testing/transformers_daily_ci",
|
|
repo_type="dataset",
|
|
token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
|
|
)
|
|
url = f"https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/{commit_info.oid}/{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_run_models_gpu/new_model_failures_with_bad_commit_grouped_by_authors.json"
|
|
|
|
# Add `GH_` prefix as keyword mention
|
|
output = {}
|
|
for author, item in new_data.items():
|
|
author = f"GH_{author}"
|
|
output[author] = item
|
|
|
|
report = f"<{url}|New failed tests>\\n\\n"
|
|
report += json.dumps(output, indent=4).replace('"', '\\"').replace("\n", "\\n")
|
|
print(report)
|