Execute ONNX graphs#
This package implements a python runtime for ONNX
in class OnnxInference.
It does not depend on scikit-learn, only numpy
and this module. However, this module was not really developped to
get the fastest python runtime but mostly to easily develop converters.
Python Runtime for ONNX#
Class OnnxInference
implements a python runtime for a subset of ONNX operators needed
to convert many scikit-learn models.
<<<
import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference
iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)
model_def = to_onnx(clr, X_train.astype(numpy.float32),
target_opset=12)
oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}))
>>>
somewhere/.local/lib/python3.9/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
warnings.warn(
{'label': array([1, 0, 1, 1, 1]), 'scores': array([[4.968, 0.353, 1.842],
[0.3 , 5.03 , 3.269],
[4.493, 1.163, 1.281],
[4.997, 0.397, 1.791],
[4.814, 0.572, 1.571]], dtype=float32)}
It is usually useful to get information on intermediate results in the graph itself to understand where the discrepencies begin.
<<<
import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference
iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)
model_def = to_onnx(clr, X_train.astype(numpy.float32),
target_opset=12)
oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))
>>>
somewhere/.local/lib/python3.9/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
warnings.warn(
+ki='Ad_Addcst': (3,) (dtype=float32 min=38.988189697265625 max=93.33505249023438)
+ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.2562500238418579 max=6.828571319580078)
+ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
-- OnnxInference: run 7 nodes with 1 inputs
Onnx-ReduceSumSquare(X) -> Re_reduced0 (name='Re_ReduceSumSquare')
+kr='Re_reduced0': (5, 1) (dtype=float32 min=36.720001220703125 max=71.86000061035156)
Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0 (name='Mu_Mul')
+kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0 (name='Ge_Gemm')
+kr='Ge_Y0': (5, 3) (dtype=float32 min=-163.74400329589844 max=-75.56999969482422)
Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01 (name='Ad_Add')
+kr='Ad_C01': (5, 3) (dtype=float32 min=-91.88400268554688 max=-22.436256408691406)
Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0 (name='Ad_Add1')
+kr='Ad_C0': (5, 3) (dtype=float32 min=0.023807525634765625 max=27.476768493652344)
Onnx-ArgMin(Ad_C0) -> label (name='Ar_ArgMin')
+kr='label': (5,) (dtype=int64 min=1 max=2)
Onnx-Sqrt(Ad_C0) -> scores (name='Sq_Sqrt')
+kr='scores': (5, 3) (dtype=float32 min=0.154296875 max=5.241828918457031)
{'label': array([2, 1, 2, 2, 1]), 'scores': array([[1.205, 4.068, 0.692],
[5.242, 0.372, 3.612],
[3.432, 1.989, 1.698],
[1.977, 3.33 , 0.583],
[5.05 , 0.154, 3.463]], dtype=float32)}
The verbosity can be increased.
<<<
import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference
iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)
model_def = to_onnx(clr, X_train.astype(numpy.float32),
target_opset=12)
oinf = OnnxInference(model_def, runtime='python')
print(oinf.run({'X': X_test[:5]}, verbose=3, fLOG=print))
>>>
somewhere/.local/lib/python3.9/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
warnings.warn(
+ki='Ad_Addcst': (3,) (dtype=float32 min=39.13035583496094 max=95.00074005126953
[95.001 39.13 63.112]
+ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.25277769565582275 max=6.8689656257629395
[[6.869 3.093 5.803 2.138]
[5.017 3.442 1.433 0.253]
[5.883 2.73 4.364 1.417]]
+ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0
[0.]
-kv='X' shape=(5, 4) dtype=float32 min=0.20000000298023224 max=6.300000190734863
-- OnnxInference: run 7 nodes with 1 inputs
Onnx-ReduceSumSquare(X) -> Re_reduced0 (name='Re_ReduceSumSquare')
+kr='Re_reduced0': (5, 1) (dtype=float32 min=35.90999984741211 max=75.79000091552734)
[[68.09]
[35.91]
[75.79]
[38.25]
[45.14]]
Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0 (name='Mu_Mul')
+kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
[[0.]
[0.]
[0.]
[0.]
[0.]]
Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0 (name='Ge_Gemm')
+kr='Ge_Y0': (5, 3) (dtype=float32 min=-169.47930908203125 max=-74.90277862548828)
[[-160.486 -90.435 -130.496]
[-104.759 -74.903 -88.236]
[-169.479 -97.862 -138.174]
[-109.883 -77.111 -92.189]
[-115.339 -84.049 -97.395]]
Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01 (name='Ad_Add')
+kr='Ad_C01': (5, 3) (dtype=float32 min=-93.6893081665039 max=-22.07166290283203)
[[-92.396 -22.345 -62.406]
[-68.849 -38.993 -52.326]
[-93.689 -22.072 -62.384]
[-71.633 -38.861 -53.939]
[-70.199 -38.909 -52.255]]
Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0 (name='Ad_Add1')
+kr='Ad_C0': (5, 3) (dtype=float32 min=0.13757705688476562 max=26.152122497558594)
[[ 2.605 16.785 0.706]
[26.152 0.138 10.786]
[ 1.311 17.059 0.728]
[23.367 0.269 9.174]
[24.801 0.221 10.857]]
Onnx-ArgMin(Ad_C0) -> label (name='Ar_ArgMin')
+kr='label': (5,) (dtype=int64 min=1 max=2)
[2 1 2 1 1]
Onnx-Sqrt(Ad_C0) -> scores (name='Sq_Sqrt')
+kr='scores': (5, 3) (dtype=float32 min=0.3709138035774231 max=5.113914489746094)
[[1.614 4.097 0.84 ]
[5.114 0.371 3.284]
[1.145 4.13 0.853]
[4.834 0.519 3.029]
[4.98 0.47 3.295]]
[VALIDATE] type <class 'onnx.onnx_ml_pb2.ModelProto'>
[VALIDATE] mis={}
{'label': array([2, 1, 2, 1, 1]), 'scores': array([[1.614, 4.097, 0.84 ],
[5.114, 0.371, 3.284],
[1.145, 4.13 , 0.853],
[4.834, 0.519, 3.029],
[4.98 , 0.47 , 3.295]], dtype=float32)}
Other runtimes with OnnxInference#
OnnxInference
can also call onnxruntime to compute the predictions by using
runtime='onnxruntime1'.
<<<
import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference
iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)
model_def = to_onnx(clr, X_train.astype(numpy.float32),
target_opset=12)
oinf = OnnxInference(model_def, runtime='onnxruntime1')
print(oinf.run({'X': X_test[:5]}))
>>>
somewhere/.local/lib/python3.9/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
warnings.warn(
{'label': array([2, 2, 2, 1, 1], dtype=int64), 'scores': array([[4.858, 1.743, 0.823],
[4.61 , 1.556, 0.743],
[4.218, 1.082, 0.918],
[2.956, 0.504, 2.299],
[3.607, 0.882, 1.575]], dtype=float32)}
Intermediate cannot be seen but the class may decompose the ONNX graph into smaller graphs, one per operator, to look into intermediate results.
<<<
import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference
iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)
model_def = to_onnx(clr, X_train.astype(numpy.float32),
target_opset=12)
oinf = OnnxInference(model_def, runtime='onnxruntime2')
print(oinf.run({'X': X_test[:5]}, verbose=1, fLOG=print))
>>>
somewhere/.local/lib/python3.9/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
warnings.warn(
+ki='Ad_Addcst': (3,) (dtype=float32 min=39.464210510253906 max=93.69570922851562)
+ki='Ge_Gemmcst': (3, 4) (dtype=float32 min=0.2558823227882385 max=6.889999866485596)
+ki='Mu_Mulcst': (1,) (dtype=float32 min=0.0 max=0.0)
-- OnnxInference: run 7 nodes with 1 inputs
Onnx-ReduceSumSquare(X) -> Re_reduced0 (name='Re_ReduceSumSquare')
+kr='Re_reduced0': (5, 1) (dtype=float32 min=27.32000160217285 max=67.20999908447266)
Onnx-Mul(Re_reduced0, Mu_Mulcst) -> Mu_C0 (name='Mu_Mul')
+kr='Mu_C0': (5, 1) (dtype=float32 min=0.0 max=0.0)
Onnx-Gemm(X, Ge_Gemmcst, Mu_C0) -> Ge_Y0 (name='Ge_Gemm')
+kr='Ge_Y0': (5, 3) (dtype=float32 min=-158.5153350830078 max=-65.11764526367188)
Onnx-Add(Re_reduced0, Ge_Y0) -> Ad_C01 (name='Ad_Add')
+kr='Ad_C01': (5, 3) (dtype=float32 min=-91.30533599853516 max=-23.957061767578125)
Onnx-Add(Ad_Addcst, Ad_C01) -> Ad_C0 (name='Ad_Add1')
+kr='Ad_C0': (5, 3) (dtype=float32 min=0.26983642578125 max=28.71038055419922)
Onnx-Sqrt(Ad_C0) -> scores (name='Sq_Sqrt')
+kr='scores': (5, 3) (dtype=float32 min=0.5194578170776367 max=5.3582072257995605)
Onnx-ArgMin(Ad_C0) -> label (name='Ar_ArgMin')
+kr='label': (5,) (dtype=int64 min=1 max=2)
{'label': array([2, 2, 1, 2, 2], dtype=int64), 'scores': array([[2.422, 3.485, 1.083],
[2.298, 2.817, 0.519],
[5.358, 1.291, 3.574],
[1.735, 3.696, 0.652],
[1.546, 3.938, 0.856]], dtype=float32)}
Finally, a last runtime ‘python_compiled’ converts some
part of the class OnnxInference
into python code then dynamically compiled.
As a consequence, interdiate results cannot be seen anymore.
<<<
import numpy
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from skl2onnx import to_onnx
from mlprodict.onnxrt import OnnxInference
iris = load_iris()
X = iris.data.astype(numpy.float32)
X_train, X_test = train_test_split(X)
clr = KMeans(n_clusters=3)
clr.fit(X_train)
model_def = to_onnx(clr, X_train.astype(numpy.float32),
target_opset=12)
oinf = OnnxInference(model_def, runtime='python_compiled')
print(oinf.run({'X': X_test[:5]}))
>>>
somewhere/.local/lib/python3.9/site-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning
warnings.warn(
{'label': array([1, 1, 2, 0, 0]), 'scores': array([[5.036, 0.433, 1.732],
[5.021, 0.688, 1.674],
[3.012, 2.427, 0.717],
[0.74 , 4.964, 3.504],
[0.572, 4.832, 3.332]], dtype=float32)}