Benchmark (ONNX) for sklearn-onnx unit tests#

Overview#

The following graph plots the ratio between onnxruntime and scikit-learn. The lower, the better for onnxruntime. Each test is mapped to a unit test in sklearn-onnx in folder tests.

(Source code, png, hires.png, pdf)

../_images/onnxruntime_unittest-1.png

Ratio by model#

(Source code, png, hires.png, pdf)

../_images/onnxruntime_unittest-2.png

Configuration#

<<<

from pyquickhelper.pandashelper import df2rst
import pandas
name = os.path.join(
    __WD__, "../../onnx/results/bench_plot_skl2onnx_unittest.time.csv")
df = pandas.read_csv(name)
print(df2rst(df, number_format=4))

>>>

name

version

value

date

2019-12-11

python

3.7.2 (default, Mar 1 2019, 18:34:21) [GCC 6.3.0 20170516]

platform

linux

OS

Linux-4.9.0-8-amd64-x86_64-with-debian-9.6

machine

x86_64

processor

release

4.9.0-8-amd64

architecture

(‘64bit’, ‘’)

numpy

1.17.4

openblas, language=c

onnx

1.6.34

opset=12

onnxruntime

1.1.992

CPU-MKL-ML

pandas

0.25.3

skl2onnx

1.6.992

sklearn

0.22

Errors#

<<<

from pyquickhelper.pandashelper import df2rst
import pandas
name = os.path.join(
    __WD__, "../../onnx/results/bench_plot_skl2onnx_unittest.perf.csv")
df = pandas.read_csv(name)
err = df[~df['stderr'].isnull()]
err = err[["_model", "stderr"]]
print(df2rst(err))

>>>

_model

stderr

SklearnOVRRegressionFloat-Out0

Model ‘somewhere/workspace/_benchmarks/_benchmarks_ORT_UNIT_37_std/onnx/onnxruntime-skl2onnx/SklearnOVRRegressionFloat-Out0.model.onnx’ has discrepencies.

Raw results#

bench_plot_skl2onnx_unittest.csv

<<<

from pyquickhelper.pandashelper import df2rst
from pymlbenchmark.benchmark.bench_helper import bench_pivot
import pandas
name = os.path.join(
    __WD__, "../../onnx/results/bench_plot_skl2onnx_unittest.perf.csv")
df = pandas.read_csv(name)
df = df[df['stderr'].isnull() & ~df.ratio.isnull()].sort_values("ratio").copy()
df['model'] = df['_model'].apply(lambda s: s.replace("Sklearn", ""))
piv = df[["model", "original_time", "original_std",
          "onnxrt_time", "onnxrt_std", "ratio"]]
piv.columns = ["model", "sklearn", "skl dev",
               "onnxruntime", "ort dev", "ratio"]
print(df2rst(piv, number_format=4))

Benchmark code#

bench_plot_skl2onnx_unittest.py

# coding: utf-8
"""
Benchmark of :epkg:`onnxruntime` on all unit tests from
:epkg:`skl2onnx`.
"""
# Authors: Xavier Dupré (benchmark)
# License: MIT
import matplotlib
matplotlib.use('Agg')

import os
import unittest
import warnings
import contextlib
from time import perf_counter as time
from io import StringIO
import numpy
import pandas
import matplotlib.pyplot as plt
import sklearn
from sklearn.utils._testing import ignore_warnings
from sklearn.utils.extmath import softmax
from pyquickhelper.loghelper import run_cmd, sys_path_append
from pymlbenchmark.context import machine_information


def run_all_tests(location, folder=None, verbose=True):
    """
    Runs all unit tests or unit tests specific to one library.
    The tests produce a series of files dumped into ``folder``
    which can be later used to tests a backend (or a runtime).
    """
    if folder is None:
        raise ValueError("folder cannot be None")
    os.environ["ONNXTESTDUMP"] = folder
    os.environ["ONNXTESTDUMPERROR"] = "1"
    os.environ["ONNXTESTBENCHMARK"] = "1"

    if verbose:
        print("[benchmark] look into '{0}'".format(location))
        print("[benchmark] dump into '{0}'".format(folder))

    subs = [location]
    loader = unittest.TestLoader()
    suites = []

    for sub in subs:
        fold = os.path.join(this, sub)
        if not os.path.exists(fold):
            raise FileNotFoundError("Unable to find '{0}'".format(fold))

        with sys_path_append(fold):
            names = [_ for _ in os.listdir(fold) if _.startswith("test")]
            for name in names:
                if "test_utils" in name:
                    continue
                if "dump" in name.lower():
                    continue
                name = os.path.splitext(name)[0]
                ts = loader.loadTestsFromName(name)
                suites.append(ts)

    with warnings.catch_warnings():
        warnings.filterwarnings(category=DeprecationWarning, action="ignore")
        warnings.filterwarnings(category=FutureWarning, action="ignore")
        st = StringIO()
        runner = unittest.TextTestRunner(st, verbosity=0)
        name = ""
        for tsi, ts in enumerate(suites):
            for k in ts:
                try:
                    for t in k:
                        name = t.__class__.__name__
                        break
                except TypeError as e:
                    warnings.warn(
                        "[ERROR] Unable to run test '{}' - {}.".format(ts, str(e).replace("\n", " ")))
            if verbose:
                print("[benchmark] {}/{}: '{}'".format(tsi + 1, len(suites), name))
            with contextlib.redirect_stderr(st):
                with contextlib.redirect_stdout(st):
                    runner.run(ts)

    from test_utils.tests_helper import make_report_backend
    df = make_report_backend(folder, as_df=True)
    return df


#########################
# Clones skl2onnx
# +++++++++++++++

this = os.path.abspath(os.path.dirname(__file__))
skl = os.path.join(this, "sklearn-onnx")
if os.path.exists(skl):
    pth = skl
    cmd = "git pull"
else:
    pth = None
    cmd = "git clone https://github.com/onnx/sklearn-onnx.git " + skl
run_cmd(cmd, wait=True, change_path=pth, fLOG=print)

#########################
# Runs the benchmark
# ++++++++++++++++++

folder = os.path.join(this, 'onnxruntime-skl2onnx')
location = os.path.join(this, 'sklearn-onnx', "tests")
filename = os.path.splitext(os.path.split(__file__)[-1])[0]
full_filename = filename + ".perf.csv"
if not os.path.exists(full_filename):
    with sklearn.config_context(assume_finite=True):
        df = run_all_tests(location, folder, verbose=True)
    print("[benchmark] saves into '{}'.".format(full_filename))
    df.to_csv(full_filename, index=False)
else:
    print("[benchmark] restores from '{}'.".format(full_filename))
    df = pandas.read_csv(full_filename)
print(df.head())

#########################
# Extracts information about the machine used
# +++++++++++++++++++++++++++++++++++++++++++

pkgs = ['numpy', 'pandas', 'sklearn', 'skl2onnx', 'onnxruntime', 'onnx']
dfi = pandas.DataFrame(machine_information(pkgs))
dfi.to_csv("%s.time.csv" % filename, index=False)
print(dfi)

#########################
# Shows errors.
# +++++++++++++

if 'stderr' in df.columns:
    err = df[~df.stderr.isnull()]
    err = err[["_model", "stderr"]]
    err.to_csv(filename + ".err.csv", index=False)
    print(err)
else:
    print('no error.')
    df['stderr'] = numpy.nan

#############################
# Plot the results by time
# ++++++++++++++++++++++++

df = df[df.stderr.isnull() & ~df.ratio.isnull()].sort_values("ratio").copy()
df['model'] = df['_model'].apply(lambda s: s.replace("Sklearn", ""))

fig, ax = plt.subplots(1, 1, figsize=(10, 10))
df.plot(x="original_time", y="ratio", ax=ax,
        logx=True, logy=True, kind="scatter")
xmin, xmax = df.original_time.min(), df.original_time.max()
ax.plot([xmin, xmax], [1, 1], "--", label="1x")
ax.plot([xmin, xmax], [2, 2], "--", label="2x slower")
ax.plot([xmin, xmax], [0.5, 0.5], "--", label="2x faster")
ax.set_title("Ratio onnxruntime / scikit-learn\nLower is better")
ax.set_xlabel("execution time with scikit-learn (seconds)")
ax.set_ylabel("Ratio onnxruntime / scikit-learn\nLower is better.")
ax.legend()
fig.tight_layout()
fig.savefig("%s.xy.png" % filename)


#############################
# Plot the results by model
# +++++++++++++++++++++++++


fig, ax = plt.subplots(1, 1, figsize=(10, 25))
df.plot.barh(x="model", y="ratio", ax=ax, logx=True)
ymin, ymax = ax.get_ylim()
ax.plot([0.5, 0.5], [ymin, ymax], '--', label="2x faster")
ax.plot([1, 1], [ymin, ymax], '-', label="1x")
ax.plot([2, 2], [ymin, ymax], '--', label="2x slower")
for tick in ax.yaxis.get_major_ticks():
    tick.label.set_fontsize(8)
ax.legend(loc='upper left')
ax.grid()
ax.set_title("Ratio onnxruntime / scikit-learn\nLower is better")
fig.tight_layout()
fig.savefig("%s.model.png" % filename)

import sys
if "--quiet" not in sys.argv:
    plt.show()