riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-04 05:10:06 +06:00
Code Issues Actions 20 Packages Projects Releases Wiki Activity

[modular] CLI allows positional arguments, and more defaults names for the optional arg (#38979)

Browse Source 
* More defaults

* Update modular_model_converter.py
This commit is contained in:
Cyril Vallez 2025-06-23 12:40:01 +02:00 committed by GitHub
parent 334bf913dc
commit 74f5e4a1fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/source/en/modular_transformers.md
View File

@ -24,7 +24,7 @@ A linter "unravels" the modular file into a `modeling.py` file to preserve the s
Run the command below to automatically generate a `modeling.py` file from a modular file. Run the command below to automatically generate a `modeling.py` file from a modular file.
```bash ```bash
python utils/modular_model_converter.py --files_to_parse src/transformers/models/<your_model>/modular_<your_model>.py python utils/modular_model_converter.py --files-to-parse src/transformers/models/<your_model>/modular_<your_model>.py
``` ```
For example: For example:

28
utils/modular_model_converter.py
View File

@ -1778,19 +1778,31 @@ def save_modeling_file(modular_file, converted_file):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
# Same arg as both positional and optional, just for convenience
parser.add_argument( parser.add_argument(
"files",
nargs="*",
help="A list of `modular_xxxx` files that should be converted to single model file",
)
parser.add_argument(
"--files-to-parse",
"--files_to_parse", "--files_to_parse",
"--files",
"-f",
default=["all"], default=["all"],
nargs="+", nargs="+",
help="A list of `modular_xxxx` files that should be converted to single model file", help="A list of `modular_xxxx` files that should be converted to single model file",
) )
args = parser.parse_args() args = parser.parse_args()
if args.files_to_parse == ["all"]: # Both arg represent the same data, but as positional and optional
args.files_to_parse = glob.glob("src/transformers/models/**/modular_*.py", recursive=True) files_to_parse = args.files if len(args.files) > 0 else args.files_to_parse
if args.files_to_parse == ["examples"]:
args.files_to_parse = glob.glob("examples/**/modular_*.py", recursive=True) if files_to_parse == ["all"]:
files_to_parse = glob.glob("src/transformers/models/**/modular_*.py", recursive=True)
if files_to_parse == ["examples"]:
files_to_parse = glob.glob("examples/**/modular_*.py", recursive=True)
else: else:
for i, model_name in enumerate(args.files_to_parse): for i, model_name in enumerate(files_to_parse):
if os.sep not in model_name: if os.sep not in model_name:
full_path = os.path.join("src", "transformers", "models", model_name, f"modular_{model_name}.py") full_path = os.path.join("src", "transformers", "models", model_name, f"modular_{model_name}.py")
# If it does not exist, try in the examples section # If it does not exist, try in the examples section
@ -1799,10 +1811,10 @@ if __name__ == "__main__":
# We did not find it anywhere # We did not find it anywhere
if not os.path.isfile(full_path): if not os.path.isfile(full_path):
raise ValueError(f"Cannot find a modular file for {model_name}. Please provide the full path.") raise ValueError(f"Cannot find a modular file for {model_name}. Please provide the full path.")
args.files_to_parse[i] = full_path files_to_parse[i] = full_path
priority_list, _ = find_priority_list(args.files_to_parse) priority_list, _ = find_priority_list(files_to_parse)
assert len(priority_list) == len(args.files_to_parse), "Some files will not be converted" assert len(priority_list) == len(files_to_parse), "Some files will not be converted"
for file_name in priority_list: for file_name in priority_list:
print(f"Converting {file_name} to a single model single file format") print(f"Converting {file_name} to a single model single file format")