diff --git a/utils/get_test_reports.py b/utils/get_test_reports.py new file mode 100644 index 00000000000..7113b8f5a1b --- /dev/null +++ b/utils/get_test_reports.py @@ -0,0 +1,78 @@ +import os +import argparse +import subprocess +from pathlib import Path + +# Mapping from suite name to test directory under `tests/` +SUITE_TO_PATH = { + "run_models_gpu": "models", + "run_pipelines_torch_gpu": "pipelines", + "run_examples_gpu": "examples", + "run_torch_cuda_extensions_gpu": "utils/torch_cuda_extensions", +} + +def is_valid_test_dir(path: Path) -> bool: + return path.is_dir() and not path.name.startswith("__") and not path.name.startswith(".") + +def run_pytest(suite: str, subdir: Path, root_test_dir: Path, machine_type: str, dry_run: bool): + relative_path = subdir.relative_to(root_test_dir) + report_name = f"{machine_type}_{suite}_{relative_path}_test_reports" + + cmd = [ + "python3", "-m", "pytest", + "-rsfE", "-v", + f"--make-reports={report_name}", + str(subdir), + "-m", "not not_device_test" + ] + + print(f"Suite: {suite} | Running on: {relative_path}") + print("Command:", " ".join(cmd)) + if not dry_run: + subprocess.run(cmd, check=False) + +def handle_suite(suite: str, test_root: Path, machine_type: str, dry_run: bool): + if suite not in SUITE_TO_PATH: + print(f"Unknown suite: {suite}") + return + + subpath = SUITE_TO_PATH[suite] + full_path = test_root / subpath + + if not full_path.exists(): + print(f"Test folder does not exist: {full_path}") + return + + if full_path.is_file(): + # Top-level test file + run_pytest(suite, full_path, test_root, machine_type, dry_run) + else: + # Recurse into each valid subdirectory + for subdir in sorted(full_path.iterdir()): + if is_valid_test_dir(subdir): + run_pytest(suite, subdir, test_root, machine_type, dry_run) + +def main(): + parser = argparse.ArgumentParser(description="Run selected test suites recursively.") + parser.add_argument("folder", help="Path to test root folder (e.g., ./tests)") + parser.add_argument("--suites", nargs="+", required=True, help="List of test suite names to run") + parser.add_argument("--machine-type", default="single-gpu", help="Machine type for report names") + parser.add_argument("--enable-slow", action="store_true", help="Run slow tests instead of skipping them") + parser.add_argument("--dry-run", action="store_true", help="Only print commands without running them") + args = parser.parse_args() + + if args.enable_slow: + os.environ["RUN_SLOW"] = "yes" + + test_root = Path(args.folder).resolve() + if not test_root.exists(): + print(f"Root test folder not found: {test_root}") + return + + for suite in args.suites: + handle_suite(suite, test_root, args.machine_type, args.dry_run) + +if __name__ == "__main__": + main() + +# python3 utils/get_test_reports.py ./tests --suites run_models_gpu run_pipelines_torch_gpu --machine-type multi-gpu \ No newline at end of file