Merges benchmarks

This script merges benchmark from Compares implementations of ReduceMax, Compares implementations of ReduceMean, Compares implementations of ReduceSum.

Reads data

One file looks like this:

import os
import pandas
import matplotlib.pyplot as plt

df = pandas.read_excel("keep_plot_reducesum_master.xlsx")
df.head(n=4).T

	0	1	2	3
average	0.00245	0.001515	0.001128	0.012992
deviation	0.000288	0.000208	0.000203	0.014079
min_exec	0.00218	0.0013	0.000893	0.00505
max_exec	0.002985	0.001908	0.001505	0.041095
repeat	5	5	5	5
number	5	5	5	5
context_size	232	232	232	232
N	8	8	8	8
fct	numpy	ort	tf	torch
axes	(3,)	(3,)	(3,)	(3,)
shape	(8, 24, 48, 8)	(8, 24, 48, 8)	(8, 24, 48, 8)	(8, 24, 48, 8)

The other files.

index = ['fct', 'axes', 'N', 'shape']
value = ['average']
files = [
    ('ReduceSum', 'keep_plot_reducesum_master.xlsx', 'keep_plot_reducesum.xlsx'),
    ('ReduceMax', 'plot_reducemax_master.xlsx', 'plot_reducemax.xlsx'),
    ('ReduceMean', 'plot_reducemean_master.xlsx', 'plot_reducemean.xlsx'),
]

merged = []
for title, ref, impl in files:
    if not os.path.exists(ref) or not os.path.exists(impl):
        continue
    df1 = pandas.read_excel(ref)
    df2 = pandas.read_excel(impl)
    df1 = df1[index + value]
    df2 = df2[index + value]
    merge = df1.merge(df2, on=index, suffixes=('_ref', '_new'))
    merge['op'] = title
    merge['SpeedUp'] = merge[value[0] + "_ref"] / merge[value[0] + "_new"]
    merged.append(merge)

all_merged = pandas.concat(merged)
all_merged = all_merged.sort_values(['op'] + index)
all_merged.head()

	fct	axes	N	shape	average_ref	average_new	op	SpeedUp
220	numpy	(0, 2)	8	(8, 24, 48, 8)	0.001230	0.001571	ReduceSum	0.783275
224	numpy	(0, 2)	16	(8, 24, 48, 16)	0.002844	0.002704	ReduceSum	1.051685
228	numpy	(0, 2)	32	(8, 24, 48, 32)	0.006076	0.005119	ReduceSum	1.186992
232	numpy	(0, 2)	64	(8, 24, 48, 64)	0.010574	0.009804	ReduceSum	1.078504
236	numpy	(0, 2)	100	(8, 24, 48, 100)	0.014287	0.014141	ReduceSum	1.010320

Markdown

piv = all_merged.pivot_table(values=['SpeedUp'], index=index, columns=['op'])
piv = piv.reset_index(drop=False).sort_values(index)
piv.columns = index + [_[1] for _ in piv.columns[4:]]
#print(piv.to_markdown(index=False, floatfmt=".2f"))

Graphs

df = all_merged
graph_col = ['op', 'axes']
set_val = set(tuple(_[1:]) for _ in df[graph_col].itertuples())

axes = list(sorted(set(df['axes'])))
op = list(sorted(set(df['op'])))

for suffix in ['_ref', '_new']:
    fig, ax = plt.subplots(len(axes), len(op), figsize=(14, 14))
    for i, a in enumerate(axes):
        for j, o in enumerate(op):
            sub = df[(df['op'] == o) & (df['axes'] == a)]
            piv = sub.pivot("N", "fct", "average" + suffix)
            ref = piv['numpy'].copy()
            for c in piv.columns:
                piv[c] = ref / piv[c]
            piv.plot(ax=ax[i, j], logx=True)
            shape = list(sub['shape'])[0]
            ax[i, j].set_title("%s - %s - %s" % (o, a, shape), fontsize=5)
            ax[i, j].legend(fontsize=5)
            plt.setp(ax[i, j].get_xticklabels(), fontsize=5)
            plt.setp(ax[i, j].get_yticklabels(), fontsize=5)
    fig.suptitle(suffix)

plt.show()

Traceback (most recent call last):
  File "/var/lib/jenkins/workspace/mlprodict/mlprodict_UT_39_std/_doc/examples/plot_op_merge_benchmark.py", line 82, in <module>
    piv.plot(ax=ax[i, j], logx=True)
IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed

previous

Grid search ONNX models

next

Speed up scikit-learn inference with ONNX