mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-08 07:10:06 +06:00
9fe09cec76
* add benchmark for all kinds of models * improved import * delete bogus files * make style
121 lines
4.3 KiB
Python
121 lines
4.3 KiB
Python
import csv
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional
|
|
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
from matplotlib.ticker import ScalarFormatter
|
|
|
|
from transformers import HfArgumentParser
|
|
|
|
|
|
@dataclass
|
|
class PlotArguments:
|
|
"""
|
|
Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
|
|
"""
|
|
|
|
csv_file: str = field(metadata={"help": "The csv file to plot."},)
|
|
plot_along_batch: bool = field(
|
|
default=False,
|
|
metadata={"help": "Whether to plot along batch size or sequence lengh. Defaults to sequence length."},
|
|
)
|
|
is_time: bool = field(
|
|
default=False,
|
|
metadata={"help": "Whether the csv file has time results or memory results. Defaults to memory results."},
|
|
)
|
|
no_log_scale: bool = field(
|
|
default=False, metadata={"help": "Disable logarithmic scale when plotting"},
|
|
)
|
|
is_train: bool = field(
|
|
default=False,
|
|
metadata={
|
|
"help": "Whether the csv file has training results or inference results. Defaults to inference results."
|
|
},
|
|
)
|
|
figure_png_file: Optional[str] = field(
|
|
default=None, metadata={"help": "Filename under which the plot will be saved. If unused no plot is saved."},
|
|
)
|
|
|
|
|
|
class Plot:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.result_dict = defaultdict(lambda: dict(bsz=[], seq_len=[], result={}))
|
|
|
|
with open(self.args.csv_file, newline="") as csv_file:
|
|
reader = csv.DictReader(csv_file)
|
|
for row in reader:
|
|
model_name = row["model"]
|
|
self.result_dict[model_name]["bsz"].append(int(row["batch_size"]))
|
|
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
|
|
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = row[
|
|
"result"
|
|
]
|
|
|
|
def plot(self):
|
|
fig, ax = plt.subplots()
|
|
title_str = "Time usage" if self.args.is_time else "Memory usage"
|
|
title_str = title_str + " for training" if self.args.is_train else title_str + " for inference"
|
|
|
|
if not self.args.no_log_scale:
|
|
# set logarithm scales
|
|
ax.set_xscale("log")
|
|
ax.set_yscale("log")
|
|
|
|
for axis in [ax.xaxis, ax.yaxis]:
|
|
axis.set_major_formatter(ScalarFormatter())
|
|
|
|
for model_name in self.result_dict.keys():
|
|
batch_sizes = sorted(list(set(self.result_dict[model_name]["bsz"])))
|
|
sequence_lengths = sorted(list(set(self.result_dict[model_name]["seq_len"])))
|
|
results = self.result_dict[model_name]["result"]
|
|
|
|
(x_axis_array, inner_loop_array) = (
|
|
(batch_sizes, sequence_lengths) if self.args.plot_along_batch else (sequence_lengths, batch_sizes)
|
|
)
|
|
|
|
for inner_loop_value in inner_loop_array:
|
|
if self.args.plot_along_batch:
|
|
y_axis_array = np.asarray([results[(x, inner_loop_value)] for x in x_axis_array], dtype=np.int)
|
|
else:
|
|
y_axis_array = np.asarray([results[(inner_loop_value, x)] for x in x_axis_array], dtype=np.float32)
|
|
|
|
(x_axis_label, inner_loop_label) = (
|
|
("batch_size", "sequence_length in #tokens")
|
|
if self.args.plot_along_batch
|
|
else ("sequence_length in #tokens", "batch_size")
|
|
)
|
|
|
|
x_axis_array = np.asarray(x_axis_array, np.int)
|
|
plt.scatter(x_axis_array, y_axis_array, label=f"{model_name} - {inner_loop_label}: {inner_loop_value}")
|
|
plt.plot(x_axis_array, y_axis_array, "--")
|
|
|
|
title_str += f" {model_name} vs."
|
|
|
|
title_str = title_str[:-4]
|
|
y_axis_label = "Time in s" if self.args.is_time else "Memory in MB"
|
|
|
|
# plot
|
|
plt.title(title_str)
|
|
plt.xlabel(x_axis_label)
|
|
plt.ylabel(y_axis_label)
|
|
plt.legend()
|
|
|
|
if self.args.figure_png_file is not None:
|
|
plt.savefig(self.args.figure_png_file)
|
|
else:
|
|
plt.show()
|
|
|
|
|
|
def main():
|
|
parser = HfArgumentParser(PlotArguments)
|
|
plot_args = parser.parse_args_into_dataclasses()[0]
|
|
plot = Plot(args=plot_args)
|
|
plot.plot()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|