Benchmark of onnxruntime on LogisticRegression

The example uses what pymlbenchmark implements, in particular class OnnxRuntimeBenchPerfTestBinaryClassification which defines a side-by-side benchmark to compare the prediction function between scikit-learn, onnxruntime and a simple numpy implementation.

Benchmark function

from time import perf_counter as time
import numpy
import pandas
import matplotlib.pyplot as plt
import sklearn
from sklearn.linear_model import LogisticRegression
try:
    from sklearn.utils._testing import ignore_warnings
except ImportError:
    from sklearn.utils.testing import ignore_warnings
from scipy.special import expit
from pymlbenchmark.context import machine_information
from pymlbenchmark.benchmark import BenchPerf
from pymlbenchmark.external import OnnxRuntimeBenchPerfTestBinaryClassification
from pymlbenchmark.plotting import plot_bench_results


class OnnxRuntimeBenchPerfTestBinaryClassification3(
        OnnxRuntimeBenchPerfTestBinaryClassification):
    """
    Overwrites the class to add a pure python implementation
    of the logistic regression.
    """

    def fcts(self, dim=None, **kwargs):

        def predict_py_predict(X, model=self.skl):
            coef = model.coef_
            intercept = model.intercept_
            pred = numpy.dot(X, coef.T) + intercept
            return (pred >= 0).astype(numpy.int32)

        def predict_py_predict_proba(X, model=self.skl):
            coef = model.coef_
            intercept = model.intercept_
            pred = numpy.dot(X, coef.T) + intercept
            decision_2d = numpy.c_[-pred, pred]
            return expit(decision_2d)

        res = OnnxRuntimeBenchPerfTestBinaryClassification.fcts(
            self, dim=dim, **kwargs)
        res.extend([
            {'method': 'predict', 'lib': 'py', 'fct': predict_py_predict},
            {'method': 'predict_proba', 'lib': 'py',
             'fct': predict_py_predict_proba},
        ])
        return res


@ignore_warnings(category=FutureWarning)
def run_bench(repeat=100, verbose=False):

    pbefore = dict(dim=[1, 5, 10, 20, 50, 100, 150],
                   fit_intercept=[True, False])
    pafter = dict(N=[1, 10, 100, 1000, 10000])
    test = lambda dim=None, **opts: (
        OnnxRuntimeBenchPerfTestBinaryClassification3(
            LogisticRegression, dim=dim, **opts))
    bp = BenchPerf(pbefore, pafter, test)

    with sklearn.config_context(assume_finite=True):
        start = time()
        results = list(bp.enumerate_run_benchs(repeat=repeat, verbose=verbose))
        end = time()

    results_df = pandas.DataFrame(results)
    print("Total time = %0.3f sec\n" % (end - start))
    return results_df

Run the benchmark

df = run_bench(verbose=True)
df.to_csv("bench_plot_onnxruntime_logistic_regression.perf.csv", index=False)
print(df.head(n=4).T)
  0%|          | 0/70 [00:00<?, ?it/s]
  1%|1         | 1/70 [00:04<05:06,  4.45s/it]
  3%|2         | 2/70 [00:05<02:54,  2.57s/it]
  4%|4         | 3/70 [00:11<04:16,  3.83s/it]
  6%|5         | 4/70 [00:56<22:29, 20.45s/it]
  7%|7         | 5/70 [00:58<14:37, 13.50s/it]
  9%|8         | 6/70 [00:58<09:48,  9.19s/it]
 10%|#         | 7/70 [01:00<06:55,  6.60s/it]
 11%|#1        | 8/70 [01:05<06:24,  6.20s/it]
 13%|#2        | 9/70 [01:51<18:48, 18.51s/it]
 14%|#4        | 10/70 [01:52<13:12, 13.20s/it]
 16%|#5        | 11/70 [01:53<09:15,  9.42s/it]
 17%|#7        | 12/70 [01:54<06:41,  6.93s/it]
 19%|#8        | 13/70 [01:59<06:06,  6.43s/it]
 20%|##        | 14/70 [02:45<17:11, 18.41s/it]
 21%|##1       | 15/70 [02:47<12:09, 13.26s/it]
 23%|##2       | 16/70 [02:48<08:33,  9.52s/it]
 24%|##4       | 17/70 [02:49<06:12,  7.03s/it]
 26%|##5       | 18/70 [02:54<05:38,  6.52s/it]
 27%|##7       | 19/70 [03:40<15:42, 18.48s/it]
 29%|##8       | 20/70 [03:42<11:07, 13.34s/it]
 30%|###       | 21/70 [03:43<07:49,  9.59s/it]
 31%|###1      | 22/70 [03:44<05:40,  7.08s/it]
 33%|###2      | 23/70 [03:49<05:08,  6.56s/it]
 34%|###4      | 24/70 [04:36<14:11, 18.51s/it]
 36%|###5      | 25/70 [04:37<10:01, 13.37s/it]
 37%|###7      | 26/70 [04:38<07:02,  9.61s/it]
 39%|###8      | 27/70 [04:39<05:05,  7.10s/it]
 40%|####      | 28/70 [04:44<04:36,  6.58s/it]
 41%|####1     | 29/70 [05:31<12:40, 18.54s/it]
 43%|####2     | 30/70 [05:33<08:58, 13.46s/it]
 44%|####4     | 31/70 [05:33<06:17,  9.67s/it]
 46%|####5     | 32/70 [05:35<04:31,  7.15s/it]
 47%|####7     | 33/70 [05:40<04:05,  6.64s/it]
 49%|####8     | 34/70 [06:27<11:15, 18.76s/it]
 50%|#####     | 35/70 [06:29<07:56, 13.63s/it]
 51%|#####1    | 36/70 [06:30<05:32,  9.79s/it]
 53%|#####2    | 37/70 [06:31<03:58,  7.24s/it]
 54%|#####4    | 38/70 [06:36<03:34,  6.70s/it]
 56%|#####5    | 39/70 [07:24<09:49, 19.01s/it]
 57%|#####7    | 40/70 [07:26<06:56, 13.89s/it]
 59%|#####8    | 41/70 [07:27<04:49,  9.97s/it]
 60%|######    | 42/70 [07:28<03:26,  7.36s/it]
 61%|######1   | 43/70 [07:34<03:05,  6.86s/it]
 63%|######2   | 44/70 [08:23<08:26, 19.49s/it]
 64%|######4   | 45/70 [08:25<05:59, 14.39s/it]
 66%|######5   | 46/70 [08:26<04:07, 10.33s/it]
 67%|######7   | 47/70 [08:27<02:55,  7.62s/it]
 69%|######8   | 48/70 [08:33<02:35,  7.05s/it]
 70%|#######   | 49/70 [09:22<06:51, 19.58s/it]
 71%|#######1  | 50/70 [09:25<04:52, 14.63s/it]
 73%|#######2  | 51/70 [09:26<03:19, 10.50s/it]
 74%|#######4  | 52/70 [09:27<02:19,  7.75s/it]
 76%|#######5  | 53/70 [09:33<02:03,  7.25s/it]
 77%|#######7  | 54/70 [10:26<05:32, 20.75s/it]
 79%|#######8  | 55/70 [10:28<03:49, 15.33s/it]
 80%|########  | 56/70 [10:29<02:33, 10.99s/it]
 81%|########1 | 57/70 [10:30<01:45,  8.09s/it]
 83%|########2 | 58/70 [10:37<01:30,  7.53s/it]
 84%|########4 | 59/70 [11:28<03:48, 20.78s/it]
 86%|########5 | 60/70 [11:32<02:36, 15.64s/it]
 87%|########7 | 61/70 [11:33<01:40, 11.21s/it]
 89%|########8 | 62/70 [11:34<01:06,  8.25s/it]
 90%|######### | 63/70 [11:41<00:54,  7.75s/it]
 91%|#########1| 64/70 [12:36<02:11, 21.92s/it]
 93%|#########2| 65/70 [12:39<01:21, 16.33s/it]
 94%|#########4| 66/70 [12:40<00:46, 11.69s/it]
 96%|#########5| 67/70 [12:41<00:25,  8.60s/it]
 97%|#########7| 68/70 [12:48<00:16,  8.06s/it]
 99%|#########8| 69/70 [13:43<00:22, 22.16s/it]
 99%|#########8| 69/70 [13:43<00:11, 11.94s/it]
Total time = 823.947 sec

                               0  ...              3
method                   predict  ...  predict_proba
lib                          skl  ...            skl
skl_nb_base_estimators       1.0  ...            1.0
N                              1  ...              1
dim                            1  ...              1
fit_intercept               True  ...           True
repeat                       100  ...            100
number                         1  ...              1
min                     0.000226  ...       0.000321
max                     0.000508  ...       0.000515
min3                    0.000232  ...       0.000321
max3                    0.000485  ...       0.000378
mean                    0.000362  ...       0.000335
lower                   0.000226  ...       0.000321
upper                   0.000506  ...       0.000382
count                        100  ...            100
median                  0.000349  ...       0.000328
error_c                        0  ...              0
onnx_nodes                   NaN  ...            NaN
onnx_opset                   NaN  ...            NaN
ort_size                     NaN  ...            NaN

[21 rows x 4 columns]

Extract information about the machine used

pkgs = ['numpy', 'pandas', 'sklearn', 'skl2onnx',
        'onnxruntime', 'onnx', 'mlprodict']
dfi = pandas.DataFrame(machine_information(pkgs))
dfi.to_csv("bench_plot_onnxruntime_logistic_regression.time.csv", index=False)
print(dfi)
                         name  ...                                              value
0                        date  ...                                                NaN
1                      python  ...  3.9.1 (default, Jan 18 2021, 16:35:58) \n[GCC ...
2                    platform  ...                                              linux
3                          OS  ...        Linux-4.19.0-13-amd64-x86_64-with-glibc2.28
4                     machine  ...                                             x86_64
5                   processor  ...
6                     release  ...                                    4.19.0-13-amd64
7                architecture  ...                                       (64bit, ELF)
8                        arch  ...                                             X86_64
9                   brand_raw  ...            Intel(R) Atom(TM) CPU  C2750  @ 2.40GHz
10                      count  ...                                                  8
11                      flags  ...  3dnowprefetch acpi aes aperfmperf apic arat ar...
12              hz_advertised  ...                                    [2400000000, 0]
13         l1_data_cache_size  ...                                              24576
14  l1_instruction_cache_size  ...                                              32768
15     l2_cache_associativity  ...                                                  8
16         l2_cache_line_size  ...                                               1024
17              l2_cache_size  ...                                            1048576
18              l3_cache_size  ...                                            1048576
19                   stepping  ...                                                  8
20                  mlprodict  ...                                                NaN
21                      numpy  ...                               openblas, language=c
22                       onnx  ...                                           opset=17
23                onnxruntime  ...                                                CPU
24                     pandas  ...                                                NaN
25                   skl2onnx  ...                                                NaN
26                    sklearn  ...                                                NaN

[27 rows x 3 columns]

Plot the results

def label_fct(la):
    la = la.replace("onxpython_compiled", "opy")
    la = la.replace("onxpython", "opy")
    la = la.replace("onxonnxruntime1", "ort")
    la = la.replace("fit_intercept", "fi")
    la = la.replace("True", "1")
    la = la.replace("False", "0")
    return la


def color_fct(la, col):
    if "onxpython_compiled" in la:
        return "red"
    if "onxpython" in la:
        return "red"
    return col


plot_bench_results(
    df, row_cols=['N', 'fit_intercept'], col_cols='method', x_value='dim',
    title="LogisticRegression\nBenchmark scikit-learn / onnxruntime",
    label_fct=label_fct, color_fct=color_fct)
plt.show()
LogisticRegression Benchmark scikit-learn / onnxruntime, method=predict, method=predict_proba
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:143: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  piv = ds.pivot(*y_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:153: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  lower_piv = ds.pivot(*lower_cols)
somewhere/workspace/pymlbenchmark/pymlbenchmark_UT_39_std/_doc/sphinxdoc/source/pymlbenchmark/plotting/plot_bench_results.py:164: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.
  upper_piv = ds.pivot(*upper_cols)

Total running time of the script: ( 14 minutes 20.025 seconds)

Gallery generated by Sphinx-Gallery