riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-01 18:51:14 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add test for parse_json_file and change typing to os.PathLike (#30183)

Browse Source 
* Add test for parse_json_file

* Change Path to PathLike

* Fix `Import block is un-sorted or un-formatted`

* revert parse_json_file

* Fix ruff format

* Add parse_json_file test
This commit is contained in:
Xu Song 2024-04-15 16:34:36 +08:00 committed by GitHub
parent b109257f4f
commit 8fd2de933c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/transformers/hf_argparser.py
View File

@ -14,6 +14,7 @@
import dataclasses import dataclasses
import json import json
import os
import sys import sys
import types import types
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError
@ -376,7 +377,9 @@ class HfArgumentParser(ArgumentParser):
raise ValueError(f"Some keys are not used by the HfArgumentParser: {sorted(unused_keys)}") raise ValueError(f"Some keys are not used by the HfArgumentParser: {sorted(unused_keys)}")
return tuple(outputs) return tuple(outputs)
def parse_json_file(self, json_file: Union[str, Path], allow_extra_keys: bool = False) -> Tuple[DataClass, ...]: def parse_json_file(
self, json_file: Union[str, os.PathLike], allow_extra_keys: bool = False
) -> Tuple[DataClass, ...]:
""" """
Alternative helper method that does not use `argparse` at all, instead loading a json file and populating the Alternative helper method that does not use `argparse` at all, instead loading a json file and populating the
dataclass types. dataclass types.
@ -398,7 +401,9 @@ class HfArgumentParser(ArgumentParser):
outputs = self.parse_dict(data, allow_extra_keys=allow_extra_keys) outputs = self.parse_dict(data, allow_extra_keys=allow_extra_keys)
return tuple(outputs) return tuple(outputs)
def parse_yaml_file(self, yaml_file: Union[str, Path], allow_extra_keys: bool = False) -> Tuple[DataClass, ...]: def parse_yaml_file(
self, yaml_file: Union[str, os.PathLike], allow_extra_keys: bool = False
) -> Tuple[DataClass, ...]:
""" """
Alternative helper method that does not use `argparse` at all, instead loading a yaml file and populating the Alternative helper method that does not use `argparse` at all, instead loading a yaml file and populating the
dataclass types. dataclass types.

2
tests/utils/test_hf_argparser.py
View File

@ -379,7 +379,7 @@ class HfArgumentParserTest(unittest.TestCase):
os.mkdir(temp_local_path) os.mkdir(temp_local_path)
with open(temp_local_path + ".json", "w+") as f: with open(temp_local_path + ".json", "w+") as f:
json.dump(args_dict_for_json, f) json.dump(args_dict_for_json, f)
parsed_args = parser.parse_yaml_file(Path(temp_local_path + ".json"))[0] parsed_args = parser.parse_json_file(Path(temp_local_path + ".json"))[0]
args = BasicExample(**args_dict_for_json) args = BasicExample(**args_dict_for_json)
self.assertEqual(parsed_args, args) self.assertEqual(parsed_args, args)