Execute ONNX graphs#

This package implements a python runtime for ONNX in class OnnxInference. It does not depend on scikit-learn, only numpy and this module. However, this module was not really developped to get the fastest python runtime but mostly to easily develop converters.

Python Runtime for ONNX
Other runtimes with OnnxInference

Python Runtime for ONNX #

Class OnnxInference implements a python runtime for a subset of ONNX operators needed to convert many scikit-learn models.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([2, 2, 2, 2, 1]), 'scores': array([[2.74 , 2.872, 0.94 ],
           [3.308, 1.995, 0.364],
           [3.996, 1.43 , 1.168],
           [3.964, 1.395, 0.632],
           [4.615, 1.005, 1.444]], dtype=float32)}

It is usually useful to get information on intermediate results in the graph itself to understand where the discrepencies begin.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=39.08943176269531 max=94.06978607177734)
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.25 max=6.860714435577393)
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=34.77000045776367 max=73.05999755859375)
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-165.67855834960938 max=-73.67118072509766)
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-92.61856079101562 max=-22.302940368652344)
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.008258819580078125 max=27.46263885498047)
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.09087804704904556 max=5.240480899810791)
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=1 max=2)
    {'label': array([1, 1, 2, 1, 1]), 'scores': array([[5.032, 0.091, 3.304],
           [5.24 , 0.434, 3.478],
           [1.205, 4.097, 0.857],
           [4.78 , 0.524, 3.025],
           [4.628, 0.626, 2.976]], dtype=float32)}

The verbosity can be increased.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=3, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=38.65431213378906 max=92.83167266845703
    [38.654 92.832 63.403]
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.2349998950958252 max=6.82424259185791
    [[4.975 3.432 1.438 0.235]
     [6.824 3.073 5.718 2.03 ]
     [5.867 2.746 4.395 1.459]]
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0
    [0.]
    -kv='X' shape=(5, 4) dtype=float32 min=0.30000001192092896 max=5.699999809265137
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=34.09000015258789 max=62.269996643066406)
    [[43.13]
     [62.27]
     [53.23]
     [52.5 ]
     [34.09]]
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    [[0.]
     [0.]
     [0.]
     [0.]
     [0.]]
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-151.74606323242188 max=-72.52100372314453)
    [[ -81.572 -113.375  -96.386]
     [ -89.485 -151.746 -125.606]
     [ -82.625 -140.225 -116.118]
     [ -85.096 -137.862 -114.842]
     [ -72.521 -101.178  -85.978]]
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-89.47606658935547 max=-27.215499877929688)
    [[-38.442 -70.245 -53.256]
     [-27.215 -89.476 -63.336]
     [-29.395 -86.995 -62.888]
     [-32.597 -85.362 -62.342]
     [-38.431 -67.088 -51.888]]
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.0670013427734375 max=25.74347686767578)
    [[ 0.213 22.587 10.147]
     [11.439  3.356  0.067]
     [ 9.259  5.836  0.515]
     [ 6.058  7.469  1.061]
     [ 0.223 25.743 11.515]]
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.2588461637496948 max=5.073802947998047)
    [[0.461 4.753 3.185]
     [3.382 1.832 0.259]
     [3.043 2.416 0.717]
     [2.461 2.733 1.03 ]
     [0.473 5.074 3.393]]
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=2)
    [0 2 2 2 0]
    {'label': array([0, 2, 2, 2, 0]), 'scores': array([[0.461, 4.753, 3.185],
           [3.382, 1.832, 0.259],
           [3.043, 2.416, 0.717],
           [2.461, 2.733, 1.03 ],
           [0.473, 5.074, 3.393]], dtype=float32)}

Other runtimes with OnnxInference #

OnnxInference can also call onnxruntime to compute the predictions by using runtime='onnxruntime1'.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='onnxruntime1')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([1, 1, 1, 0, 0], dtype=int64), 'scores': array([[3.058, 0.531, 2.286],
           [3.616, 0.26 , 1.582],
           [4.191, 0.774, 1.224],
           [0.5  , 3.546, 5.236],
           [1.301, 3.766, 5.51 ]], dtype=float32)}

Intermediate cannot be seen but the class may decompose the ONNX graph into smaller graphs, one per operator, to look into intermediate results.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='onnxruntime2')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))

>>>

    +ki='Ad_Addcst': (3,) (dtype=float32 min=39.569862365722656 max=94.56828308105469)
    +ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.2558826208114624 max=6.900000095367432)
    +ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
    -- OnnxInference: run 7 nodes
    Onnx-ReduceSumSquare(X) -> Re_reduced0    (name='Re_ReduceSumSquare')
    +kr='Re_reduced0': (5, 1) (dtype=float32 min=46.1099967956543 max=69.32999420166016)
    Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0    (name='Mu_Mul')
    +kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
    Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0    (name='Ge_Gemm')
    +kr='Ge_Y0': (5, 3) (dtype=float32 min=-161.4214324951172 max=-85.22058868408203)
    Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01    (name='Ad_Add')
    +kr='Ad_C01': (5, 3) (dtype=float32 min=-92.2057113647461 max=-22.64588165283203)
    Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0    (name='Ad_Add1')
    +kr='Ad_C0': (5, 3) (dtype=float32 min=0.13265228271484375 max=26.228988647460938)
    Onnx-ArgMin(Ad_C0) -> label    (name='Ar_ArgMin')
    +kr='label': (5,) (dtype=int64 min=0 max=1)
    Onnx-Sqrt(Ad_C0) -> scores    (name='Sq_Sqrt')
    +kr='scores': (5, 3) (dtype=float32 min=0.36421459913253784 max=5.121424674987793)
    {'label': array([1, 1, 0, 1, 1], dtype=int64), 'scores': array([[3.363, 0.44 , 1.913],
           [2.929, 0.516, 2.261],
           [0.678, 3.52 , 5.121],
           [4.114, 0.919, 1.537],
           [3.535, 0.364, 1.574]], dtype=float32)}

Finally, a last runtime ‘python_compiled’ converts some part of the class OnnxInference into python code then dynamically compiled. As a consequence, interdiate results cannot be seen anymore.

<<<

import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference

iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)

model_def = to_onnx(clr, X_train.astype(numpy.float32),
                    target_opset=12)

oinf = OnnxInference(model_def, runtime='python_compiled')
print(oinf.run({'X': X_test[:5]}))

>>>

    {'label': array([1, 1, 0, 0, 1]), 'scores': array([[3.345, 0.578, 2.043],
           [2.95 , 0.5  , 2.296],
           [1.086, 3.648, 5.148],
           [0.396, 3.319, 4.968],
           [3.488, 0.208, 1.66 ]], dtype=float32)}

Tutorial

Converters with options

Execute ONNX graphs#

Python Runtime for ONNX#

Other runtimes with OnnxInference#

Python Runtime for ONNX #

Other runtimes with OnnxInference #