Merges benchmarks#

This script merges benchmark from Compares implementations of ReduceMax, Compares implementations of ReduceMean, Compares implementations of ReduceSum.

Reads data#

One file looks like this:

import os
import pandas
import matplotlib.pyplot as plt

df = pandas.read_excel("keep_plot_reducesum_master.xlsx")
df.head(n=4).T
0 1 2 3
average 0.00245 0.001515 0.001128 0.012992
deviation 0.000288 0.000208 0.000203 0.014079
min_exec 0.00218 0.0013 0.000893 0.00505
max_exec 0.002985 0.001908 0.001505 0.041095
repeat 5 5 5 5
number 5 5 5 5
context_size 232 232 232 232
N 8 8 8 8
fct numpy ort tf torch
axes (3,) (3,) (3,) (3,)
shape (8, 24, 48, 8) (8, 24, 48, 8) (8, 24, 48, 8) (8, 24, 48, 8)


The other files.

index = ['fct', 'axes', 'N', 'shape']
value = ['average']
files = [
    ('ReduceSum', 'keep_plot_reducesum_master.xlsx', 'keep_plot_reducesum.xlsx'),
    ('ReduceMax', 'plot_reducemax_master.xlsx', 'plot_reducemax.xlsx'),
    ('ReduceMean', 'plot_reducemean_master.xlsx', 'plot_reducemean.xlsx'),
]

merged = []
for title, ref, impl in files:
    if not os.path.exists(ref) or not os.path.exists(impl):
        continue
    df1 = pandas.read_excel(ref)
    df2 = pandas.read_excel(impl)
    df1 = df1[index + value]
    df2 = df2[index + value]
    merge = df1.merge(df2, on=index, suffixes=('_ref', '_new'))
    merge['op'] = title
    merge['SpeedUp'] = merge[value[0] + "_ref"] / merge[value[0] + "_new"]
    merged.append(merge)

all_merged = pandas.concat(merged)
all_merged = all_merged.sort_values(['op'] + index)
all_merged.head()
fct axes N shape average_ref average_new op SpeedUp
220 numpy (0, 2) 8 (8, 24, 48, 8) 0.001230 0.001571 ReduceSum 0.783275
224 numpy (0, 2) 16 (8, 24, 48, 16) 0.002844 0.002704 ReduceSum 1.051685
228 numpy (0, 2) 32 (8, 24, 48, 32) 0.006076 0.005119 ReduceSum 1.186992
232 numpy (0, 2) 64 (8, 24, 48, 64) 0.010574 0.009804 ReduceSum 1.078504
236 numpy (0, 2) 100 (8, 24, 48, 100) 0.014287 0.014141 ReduceSum 1.010320


Markdown#

piv = all_merged.pivot_table(values=['SpeedUp'], index=index, columns=['op'])
piv = piv.reset_index(drop=False).sort_values(index)
piv.columns = index + [_[1] for _ in piv.columns[4:]]
#print(piv.to_markdown(index=False, floatfmt=".2f"))

Graphs#

df = all_merged
graph_col = ['op', 'axes']
set_val = set(tuple(_[1:]) for _ in df[graph_col].itertuples())

axes = list(sorted(set(df['axes'])))
op = list(sorted(set(df['op'])))

for suffix in ['_ref', '_new']:
    fig, ax = plt.subplots(len(axes), len(op), figsize=(14, 14))
    for i, a in enumerate(axes):
        for j, o in enumerate(op):
            sub = df[(df['op'] == o) & (df['axes'] == a)]
            piv = sub.pivot("N", "fct", "average" + suffix)
            ref = piv['numpy'].copy()
            for c in piv.columns:
                piv[c] = ref / piv[c]
            piv.plot(ax=ax[i, j], logx=True)
            shape = list(sub['shape'])[0]
            ax[i, j].set_title("%s - %s - %s" % (o, a, shape), fontsize=5)
            ax[i, j].legend(fontsize=5)
            plt.setp(ax[i, j].get_xticklabels(), fontsize=5)
            plt.setp(ax[i, j].get_yticklabels(), fontsize=5)
    fig.suptitle(suffix)

plt.show()
Traceback (most recent call last):
  File "/var/lib/jenkins/workspace/mlprodict/mlprodict_UT_39_std/_doc/examples/plot_op_merge_benchmark.py", line 82, in <module>
    piv.plot(ax=ax[i, j], logx=True)
IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed

Total running time of the script: ( 0 minutes 3.784 seconds)

Gallery generated by Sphinx-Gallery