mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-04 13:20:12 +06:00
parent
f5620a7634
commit
a7f5479b45
@ -1,40 +1,48 @@
|
|||||||
import ast
|
import ast
|
||||||
from collections import defaultdict, deque
|
from collections import defaultdict
|
||||||
|
|
||||||
|
|
||||||
# Function to perform topological sorting
|
# Function to perform topological sorting
|
||||||
def topological_sort(dependencies):
|
def topological_sort(dependencies):
|
||||||
# Create a graph and in-degree count for each node
|
new_dependencies = {}
|
||||||
graph = defaultdict(list)
|
graph = defaultdict(list)
|
||||||
in_degree = defaultdict(int)
|
|
||||||
|
|
||||||
# Build the graph
|
|
||||||
for node, deps in dependencies.items():
|
for node, deps in dependencies.items():
|
||||||
for dep in deps:
|
for dep in deps:
|
||||||
graph[dep].append(node) # node depends on dep
|
if "example" not in node and "auto" not in dep:
|
||||||
in_degree[node] += 1 # increase in-degree of node
|
graph[dep.split(".")[-2]].append(node.split("/")[-2])
|
||||||
|
new_dependencies[node.split("/")[-2]] = node
|
||||||
|
|
||||||
# Add all nodes with zero in-degree to the queue
|
# Create a graph and in-degree count for each node
|
||||||
zero_in_degree_queue = deque([node for node in dependencies if in_degree[node] == 0])
|
def filter_one_by_one(filtered_list, reverse):
|
||||||
|
if len(reverse) == 0:
|
||||||
|
return filtered_list
|
||||||
|
|
||||||
sorted_list = []
|
graph = defaultdict(list)
|
||||||
# Perform topological sorting
|
# Build the graph
|
||||||
while zero_in_degree_queue:
|
for node, deps in reverse.items():
|
||||||
current = zero_in_degree_queue.popleft()
|
for dep in deps:
|
||||||
sorted_list.append(current)
|
graph[dep].append(node)
|
||||||
|
|
||||||
# For each node that current points to, reduce its in-degree
|
base_modules = set(reverse.keys()) - set(graph.keys())
|
||||||
for neighbor in graph[current]:
|
if base_modules == reverse.keys():
|
||||||
in_degree[neighbor] -= 1
|
# we are at the end
|
||||||
if in_degree[neighbor] == 0:
|
return filtered_list + list(graph.keys())
|
||||||
zero_in_degree_queue.append(neighbor)
|
to_add = []
|
||||||
|
for k in graph.keys():
|
||||||
|
if len(graph[k]) == 1 and graph[k][0] in base_modules:
|
||||||
|
if graph[k][0] in reverse:
|
||||||
|
del reverse[graph[k][0]]
|
||||||
|
if k not in filtered_list:
|
||||||
|
to_add += [k]
|
||||||
|
for k in base_modules:
|
||||||
|
if k not in filtered_list:
|
||||||
|
to_add += [k]
|
||||||
|
filtered_list += list(to_add)
|
||||||
|
return filter_one_by_one(filtered_list, reverse)
|
||||||
|
|
||||||
# Handle nodes that have no dependencies and were not initially part of the loop
|
final_order = filter_one_by_one([], graph)
|
||||||
for node in dependencies:
|
|
||||||
if node not in sorted_list:
|
|
||||||
sorted_list.append(node)
|
|
||||||
|
|
||||||
return sorted_list
|
return [new_dependencies.get(k) for k in final_order if k in new_dependencies]
|
||||||
|
|
||||||
|
|
||||||
# Function to extract class and import info from a file
|
# Function to extract class and import info from a file
|
||||||
@ -46,7 +54,7 @@ def extract_classes_and_imports(file_path):
|
|||||||
for node in ast.walk(tree):
|
for node in ast.walk(tree):
|
||||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||||
module = node.module if isinstance(node, ast.ImportFrom) else None
|
module = node.module if isinstance(node, ast.ImportFrom) else None
|
||||||
if module and "transformers" in module:
|
if module and (".modeling_" in module):
|
||||||
imports.add(module)
|
imports.add(module)
|
||||||
return imports
|
return imports
|
||||||
|
|
||||||
@ -56,7 +64,7 @@ def map_dependencies(py_files):
|
|||||||
dependencies = defaultdict(set)
|
dependencies = defaultdict(set)
|
||||||
# First pass: Extract all classes and map to files
|
# First pass: Extract all classes and map to files
|
||||||
for file_path in py_files:
|
for file_path in py_files:
|
||||||
dependencies[file_path].add(None)
|
# dependencies[file_path].add(None)
|
||||||
class_to_file = extract_classes_and_imports(file_path)
|
class_to_file = extract_classes_and_imports(file_path)
|
||||||
for module in class_to_file:
|
for module in class_to_file:
|
||||||
dependencies[file_path].add(module)
|
dependencies[file_path].add(module)
|
||||||
@ -66,4 +74,4 @@ def map_dependencies(py_files):
|
|||||||
def find_priority_list(py_files):
|
def find_priority_list(py_files):
|
||||||
dependencies = map_dependencies(py_files)
|
dependencies = map_dependencies(py_files)
|
||||||
ordered_classes = topological_sort(dependencies)
|
ordered_classes = topological_sort(dependencies)
|
||||||
return ordered_classes[::-1]
|
return ordered_classes
|
||||||
|
@ -1678,7 +1678,7 @@ if __name__ == "__main__":
|
|||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--files_to_parse",
|
"--files_to_parse",
|
||||||
default=["src/transformers/models/aria/modular_aria.py"],
|
default=["all"],
|
||||||
nargs="+",
|
nargs="+",
|
||||||
help="A list of `modular_xxxx` files that should be converted to single model file",
|
help="A list of `modular_xxxx` files that should be converted to single model file",
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user