Benchmark, comparison scikit-learn - forward-backward#

The benchmark compares the processing time between scikit-learn and onnxruntime-training on a linear regression and a neural network. It replicates the benchmark implemented in Benchmark, comparison scikit-learn - onnxruntime-training but uses the forward backward approach developped in Train a linear regression with forward backward.

First comparison: neural network#

import warnings
import time
import numpy
import matplotlib.pyplot as plt
from pandas import DataFrame
from onnxruntime import get_device
from pyquickhelper.pycode.profiling import profile, profile2graph
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPRegressor
from mlprodict.onnx_conv import to_onnx
from onnxcustom.utils.onnx_helper import onnx_rename_weights
from onnxcustom.training.optimizers_partial import (
    OrtGradientForwardBackwardOptimizer)


X, y = make_regression(1000, n_features=100, bias=2)
X = X.astype(numpy.float32)
y = y.astype(numpy.float32)
X_train, X_test, y_train, y_test = train_test_split(X, y)

Benchmark function.

def benchmark(X, y, skl_model, train_session, name, verbose=True):
    """
    :param skl_model: model from scikit-learn
    :param train_session: instance of OrtGradientForwardBackwardOptimizer
    :param name: experiment name
    :param verbose: to debug
    """
    print("[benchmark] %s" % name)
    begin = time.perf_counter()
    skl_model.fit(X, y)
    duration_skl = time.perf_counter() - begin
    length_skl = len(skl_model.loss_curve_)
    print("[benchmark] skl=%r iterations - %r seconds" % (
        length_skl, duration_skl))

    begin = time.perf_counter()
    train_session.fit(X, y)
    duration_ort = time.perf_counter() - begin
    length_ort = len(train_session.train_losses_)
    print("[benchmark] ort=%r iteration - %r seconds" % (
        length_ort, duration_ort))

    return dict(skl=duration_skl, ort=duration_ort, name=name,
                iter_skl=length_skl, iter_ort=length_ort,
                losses_skl=skl_model.loss_curve_,
                losses_ort=train_session.train_losses_)

Common parameters and model

batch_size = 15
max_iter = 100

nn = MLPRegressor(hidden_layer_sizes=(50, 10), max_iter=max_iter,
                  solver='sgd', learning_rate_init=5e-4, alpha=0,
                  n_iter_no_change=max_iter * 3, batch_size=batch_size,
                  nesterovs_momentum=False, momentum=0,
                  learning_rate="invscaling")

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    nn.fit(X_train, y_train)

Conversion to ONNX and trainer initialization

onx = to_onnx(nn, X_train[:1].astype(numpy.float32), target_opset=15)
onx = onnx_rename_weights(onx)

train_session = OrtGradientForwardBackwardOptimizer(
    onx, device='cpu', learning_rate=1e-5,
    warm_start=False, max_iter=max_iter, batch_size=batch_size)


benches = [benchmark(X_train, y_train, nn, train_session, name='NN-CPU')]

Out:

[benchmark] NN-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  warnings.warn(
[benchmark] skl=100 iterations - 10.036501206457615 seconds
[benchmark] ort=100 iteration - 10.246101723983884 seconds

Profiling#

def clean_name(text):
    pos = text.find('onnxruntime')
    if pos >= 0:
        return text[pos:]
    pos = text.find('sklearn')
    if pos >= 0:
        return text[pos:]
    pos = text.find('onnxcustom')
    if pos >= 0:
        return text[pos:]
    pos = text.find('site-packages')
    if pos >= 0:
        return text[pos:]
    return text


ps = profile(lambda: benchmark(X_train, y_train,
             nn, train_session, name='NN-CPU'))[0]
root, nodes = profile2graph(ps, clean_text=clean_name)
text = root.to_text()
print(text)

Out:

[benchmark] NN-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  warnings.warn(
[benchmark] skl=100 iterations - 11.541221112012863 seconds
[benchmark] ort=100 iteration - 12.097604973241687 seconds
filter                                                       --     18     18 -- 0.00008 0.00020 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter)
    filter                                                   --     12     12 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:350:filter (filter)
    filter                                                   --      6      6 -- 0.00006 0.00008 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:483:filter (filter)
        <built-in method builtins.isinstance>                --     18     18 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.hasattr>                       --     18     18 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
acquire                                                      --     30     30 -- 0.00007 0.00012 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire)
    <method 'acquire' of '_thread.RLock' objects>            --     30     30 -- 0.00005 0.00005 -- ~:0:<method 'acquire' of '_thread.RLock' objects> (<method 'acquire' of '_thread.RLock' objects>)
release                                                      --     30     30 -- 0.00006 0.00008 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release)
    <method 'release' of '_thread.RLock' objects>            --     30     30 -- 0.00002 0.00002 -- ~:0:<method 'release' of '_thread.RLock' objects> (<method 'release' of '_thread.RLock' objects>)
emit                                                         --     12     12 -- 0.00009 0.00115 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit)
    format                                                   --     12     12 -- 0.00004 0.00060 -- /usr/local/lib/python3.9/logging/__init__.py:912:format (format)
        format                                               --     12     12 -- 0.00010 0.00056 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:528:format (format)
            format                                           --     12     12 -- 0.00008 0.00041 -- /usr/local/lib/python3.9/logging/__init__.py:646:format (format)
                usesTime                                     --     12     12 -- 0.00002 0.00009 -- /usr/local/lib/python3.9/logging/__init__.py:624:usesTime (usesTime)
                    usesTime                                 --     12     12 -- 0.00004 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:417:usesTime (usesTime)
                        <method 'find' of 'str' objects>     --     12     12 -- 0.00002 0.00002 -- ~:0:<method 'find' of 'str' objects> (<method 'find' of 'str' objects>)
                formatMessage                                --     12     12 -- 0.00002 0.00009 -- /usr/local/lib/python3.9/logging/__init__.py:630:formatMessage (formatMessage)
                    format                                   --     12     12 -- 0.00002 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:428:format (format)
                        _format                              --     12     12 -- 0.00005 0.00005 -- /usr/local/lib/python3.9/logging/__init__.py:425:_format (_format)
                getMessage                                   --     12     12 -- 0.00007 0.00014 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:88:getMessage (getMessage)
                    getMessage                               --     12     12 -- 0.00006 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:354:getMessage (getMessage)
                    <built-in method builtins.getattr>       --     12     12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
            colorize                                         --      2      2 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/console.py:81:colorize (colorize)
                escseq                                       --      4      4 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/console.py:82:escseq (escseq)
            <built-in method builtins.getattr>               --     12     12 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
    flush                                                    --     12     12 -- 0.00007 0.00039 -- /usr/local/lib/python3.9/logging/__init__.py:1056:flush (flush)
        acquire                                              --     12     12 -- 0.00002 0.00004 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
        release                                              --     12     12 -- 0.00003 0.00004 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
        flush                                                --      6      6 -- 0.00003 0.00022 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:554:flush (flush)
            <method 'flush' of '_io.TextIOWrapper' objects>  --      6      6 -- 0.00019 0.00019 -- ~:0:<method 'flush' of '_io.TextIOWrapper' objects> (<method 'flush' of '_io.TextIOWrapper' objects>)
        <built-in method builtins.hasattr>                   --     12     12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    write                                                    --      6      6 -- 0.00003 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:546:write (write)
    write                                                    --      6      6 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:564:write (write)
isEnabledFor                                                 --     12     12 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor)
inner                                                        --      6      6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/typing.py:256:inner (inner)
cast                                                         --      6      6 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/typing.py:1326:cast (cast)
__init__                                                     --      2      2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__)
<lambda>                                                     --      1      1 -- 0.00001 23.66170 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_fwbw.py:124:<lambda> (<lambda>)
    benchmark                                                --      1      1 -- 0.02055 23.66169 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_fwbw.py:46:benchmark (benchmark)
        fit                                                  --      1      1 -- 0.00571 12.09756 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:289:fit (fit)
            __init__                                         --      1      1 -- 0.00005 0.00011 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:31:__init__ (__init__)
                get_ort_device                               --      1      1 -- 0.00000 0.00000 -- onnxruntime_helper.py:63:get_ort_device (get_ort_device)
                numpy_to_ort_value                           --      2      2 -- 0.00001 0.00005 -- onnxruntime_helper.py:134:numpy_to_ort_value (numpy_to_ort_value) +++
            needs_grad                                       --      3      3 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:104:needs_grad (needs_grad)
                needs_grad                                   --      3      3 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:194:needs_grad (needs_grad)
            get_full_state                                   --    101    101 -- 0.00070 0.00222 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:155:get_full_state (get_full_state) +++
            set_state                                        --      2      2 -- 0.00013 0.00041 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:216:set_state (set_state)
                _get_att_state                               --      2      2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:147:_get_att_state (_get_att_state) +++
                numpy_to_ort_value                           --     12     12 -- 0.00004 0.00022 -- onnxruntime_helper.py:134:numpy_to_ort_value (numpy_to_ort_value) +++
                <method 'append' of 'list' objects>          --     28     28 -- 0.00001 0.00001 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                <built-in method builtins.isinstance>        --     12     12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
            <listcomp>                                       --      1      1 -- 0.00006 0.00303 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:340:<listcomp> (<listcomp>)
                get_initializer                              --      7      7 -- 0.00020 0.00297 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:280:get_initializer (get_initializer)
                    to_array                                 --      6      6 -- 0.00019 0.00278 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/numpy_helper.py:21:to_array (to_array)
                        uses_external_data                   --      6      6 -- 0.00002 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/external_data_helper.py:224:uses_external_data (uses_external_data)
                        <method 'HasField...essage' objects> --     12     12 -- 0.00004 0.00004 -- ~:0:<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects>) +++
                        <method 'astype' ...darray' objects> --      6      6 -- 0.00010 0.00010 -- ~:0:<method 'astype' of 'numpy.ndarray' objects> (<method 'astype' of 'numpy.ndarray' objects>) +++
                        <built-in method numpy.asarray>      --      6      6 -- 0.00234 0.00234 -- ~:0:<built-in method numpy.asarray> (<built-in method numpy.asarray>) +++
            _iteration                                       --    100    100 -- 1.60233 12.03529 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:426:_iteration (_iteration)
                iter_ortvalue                                --   5100   5100 -- 0.12527 0.65378 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:147:iter_ortvalue (iter_ortvalue)
                    _next_iter                               --   5000   5000 -- 0.03467 0.33117 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:98:_next_iter (_next_iter)
                        <method 'randint'...mState' objects> --   5000   5000 -- 0.28371 0.28371 -- ~:0:<method 'randint' of 'numpy.random.mtrand.RandomState' objects> (<method 'randint' of 'numpy.random.mtrand.RandomState' objects>)
                        <built-in method builtins.len>       --   5000   5000 -- 0.00758 0.01280 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                    numpy_to_ort_value                       --  10000  10000 -- 0.02982 0.16306 -- onnxruntime_helper.py:134:numpy_to_ort_value (numpy_to_ort_value) +++
                    <built-in method builtins.len>           --   5200   5200 -- 0.01680 0.03428 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                forward                                      --   5000   5000 -- 1.21651 1.60318 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:654:forward (forward)
                    input_to_ort                             --   5000   5000 -- 0.24313 0.33225 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:577:input_to_ort (input_to_ort) +++
                    save_for_backward                        --   5000   5000 -- 0.04523 0.04523 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:629:save_for_backward (save_for_backward)
                    <method 'append' of 'list' objects>      --   5000   5000 -- 0.00919 0.00919 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                backward                                     --   5000   5000 -- 1.58726 1.77357 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:732:backward (backward)
                    input_to_ort                             --   5000   5000 -- 0.13343 0.17137 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:577:input_to_ort (input_to_ort) +++
                    saved_tensors                            --   5000   5000 -- 0.00697 0.00697 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:641:saved_tensors (saved_tensors)
                    <method 'pop' of 'list' objects>         --   5000   5000 -- 0.00796 0.00796 -- ~:0:<method 'pop' of 'list' objects> (<method 'pop' of 'list' objects>)
                loss_gradient                                --   5000   5000 -- 0.26729 1.08337 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_loss.py:61:loss_gradient (loss_gradient)
                    clear_binding_inputs                     --   5000   5000 -- 0.01950 0.04610 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:146:clear_binding_inputs (clear_binding_inputs)
                        _cache_in_clear                      --   5000   5000 -- 0.02011 0.02660 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:135:_cache_in_clear (_cache_in_clear)
                            <built-in method builtins.id>    --   5000   5000 -- 0.00649 0.00649 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>) +++
                    _bind_input_ortvalue                     --  10000  10000 -- 0.07427 0.34762 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:178:_bind_input_ortvalue (_bind_input_ortvalue) +++
                    _call_iobinding                          --   5000   5000 -- 0.41261 0.41261 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_loss.py:58:_call_iobinding (_call_iobinding)
                    <built-in method builtins.hasattr>       --  10000  10000 -- 0.00975 0.00975 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                penalty_loss                                 --   5000   5000 -- 0.00396 0.00396 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:101:penalty_loss (penalty_loss)
                update_weights                               --  30000  30000 -- 0.01911 0.01911 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:115:update_weights (update_weights)
                update_weights                               --  30000  30000 -- 1.37146 5.21990 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:258:update_weights (update_weights)
                    _bind_input_ortvalue                     --  90000  90000 -- 0.44386 1.41558 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:178:_bind_input_ortvalue (_bind_input_ortvalue) +++
                    _bind_output_ortvalue                    --  30000  30000 -- 0.14137 0.45005 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:224:_bind_output_ortvalue (_bind_output_ortvalue)
                        _bio_cache                           --  30000  30000 -- 0.09486 0.11775 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:157:_bio_cache (_bio_cache) +++
                        _bio_ptr                             --  30000  30000 -- 0.17543 0.17543 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:174:_bio_ptr (_bio_ptr) +++
                        <built-in method ...tins.isinstance> --  30000  30000 -- 0.01550 0.01550 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
                    _call_iobinding                          --  30000  30000 -- 1.46079 1.46079 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:33:_call_iobinding (_call_iobinding)
                    value                                    --  30000  30000 -- 0.03511 0.03511 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:186:value (value) +++
                    <built-in method on...tvalue_from_numpy> --  30000  30000 -- 0.43931 0.43931 -- ~:0:<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy> (<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy>) +++
                    <built-in method builtins.hasattr>       --  60000  60000 -- 0.04760 0.04760 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                <method 'mean' of 'numpy.ndarray' objects>   --    100    100 -- 0.00070 0.01538 -- ~:0:<method 'mean' of 'numpy.ndarray' objects> (<method 'mean' of 'numpy.ndarray' objects>) +++
                <built-in method numpy.array>                --    100    100 -- 0.00832 0.00832 -- ~:0:<built-in method numpy.array> (<built-in method numpy.array>) +++
                <method 'append' of 'list' objects>          --   5000   5000 -- 0.00582 0.00582 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                <built-in method builtins.len>               --  30100  30100 -- 0.04656 0.04656 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
            _create_training_session                         --      1      1 -- 0.00003 0.04624 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:661:_create_training_session (_create_training_session)
                __init__                                     --      1      1 -- 0.00019 0.04617 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:59:__init__ (__init__)
                    <listcomp>                               --      1      1 -- 0.00003 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:96:<listcomp> (<listcomp>)
                    <listcomp>                               --      1      1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:99:<listcomp> (<listcomp>)
                    <listcomp>                               --      1      1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:118:<listcomp> (<listcomp>)
                    _init_next                               --      1      1 -- 0.00020 0.04591 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:171:_init_next (_init_next)
                        <listcomp>                           --      1      1 -- 0.00003 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:181:<listcomp> (<listcomp>)
                        <listcomp>                           --      1      1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:183:<listcomp> (<listcomp>)
                        <listcomp>                           --      1      1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:186:<listcomp> (<listcomp>)
                        _create_onnx_graphs                  --      1      1 -- 0.00980 0.04565 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:301:_create_onnx_graphs (_create_onnx_graphs)
                            <listcomp>                       --      1      1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:413:<listcomp> (<listcomp>)
                            <listcomp>                       --      1      1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:414:<listcomp> (<listcomp>)
                            <listcomp>                       --      1      1 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:416:<listcomp> (<listcomp>)
                                _provider_nam..._device_type --      1      1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:271:_provider_name_to_device_type (_provider_name_to_device_type) +++
                            <listcomp>                       --      1      1 -- 0.00009 0.00012 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:421:<listcomp> (<listcomp>)
                                _provider_nam..._device_type --      7      7 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:271:_provider_name_to_device_type (_provider_name_to_device_type) +++
                            <listcomp>                       --      1      1 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:427:<listcomp> (<listcomp>)
                                _provider_nam..._device_type --      1      1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:271:_provider_name_to_device_type (_provider_name_to_device_type) +++
                            <listcomp>                       --      1      1 -- 0.00003 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:495:<listcomp> (<listcomp>)
                            <listcomp>                       --      1      1 -- 0.00003 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:496:<listcomp> (<listcomp>)
                            load_model                       --      2      2 -- 0.00002 0.00120 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:107:load_model (load_model)
                                _load_bytes                  --      2      2 -- 0.00005 0.00007 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:30:_load_bytes (_load_bytes)
                                    inner                    --      4      4 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/typing.py:256:inner (inner) +++
                                    cast                     --      4      4 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/typing.py:1326:cast (cast) +++
                                _get_file_path               --      2      2 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:50:_get_file_path (_get_file_path)
                                load_model_from_string       --      2      2 -- 0.00005 0.00110 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:147:load_model_from_string (load_model_from_string)
                                    _deserialize             --      2      2 -- 0.00004 0.00105 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:81:_deserialize (_deserialize)
                                        inner                --      2      2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/typing.py:256:inner (inner) +++
                                        cast                 --      2      2 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/typing.py:1326:cast (cast) +++
                                        <method '...objects> --      2      2 -- 0.00099 0.00099 -- ~:0:<method 'ParseFromString' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'ParseFromString' of 'google.protobuf.pyext._message.CMessage' objects>)
                            get_inputs                       --      1      1 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:111:get_inputs (get_inputs)
                            get_outputs                      --      1      1 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:115:get_outputs (get_outputs)
                            __init__                         --      2      2 -- 0.00009 0.03403 -- onnxruntime/capi/onnxruntime_inference_collection.py:283:__init__ (__init__)
                                get                          --      2      2 -- 0.00002 0.00009 -- /usr/local/lib/python3.9/_collections_abc.py:675:get (get)
                                    __getitem__              --      2      2 -- 0.00003 0.00007 -- /usr/local/lib/python3.9/os.py:674:__getitem__ (__getitem__)
                                        encode               --      2      2 -- 0.00003 0.00003 -- /usr/local/lib/python3.9/os.py:754:encode (encode)
                                __init__                     --      2      2 -- 0.00001 0.00001 -- onnxruntime/capi/onnxruntime_inference_collection.py:101:__init__ (__init__)
                                _create_inference_session    --      2      2 -- 0.03362 0.03384 -- onnxruntime/capi/onnxruntime_inference_collection.py:346:_create_inference_session (_create_inference_session)
                                    check_and_n...vider_args --      2      2 -- 0.00010 0.00019 -- onnxruntime/capi/onnxruntime_inference_collection.py:25:check_and_normalize_provider_args (check_and_normalize_provider_args)
                                        set_provider_options --      2      2 -- 0.00002 0.00003 -- onnxruntime/capi/onnxruntime_inference_collection.py:53:set_provider_options (set_provider_options)
                                            <dictcomp>       --      2      2 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:62:<dictcomp> (<dictcomp>)
                                        <listcomp>           --      2      2 -- 0.00001 0.00001 -- onnxruntime/capi/onnxruntime_inference_collection.py:75:<listcomp> (<listcomp>)
                                        <listcomp>           --      2      2 -- 0.00000 0.00001 -- onnxruntime/capi/onnxruntime_inference_collection.py:78:<listcomp> (<listcomp>)
                            <method 'Serial...sage' objects> --      1      1 -- 0.00031 0.00031 -- ~:0:<method 'SerializeToString' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'SerializeToString' of 'google.protobuf.pyext._message.CMessage' objects>)
                            <built-in method builtins.len>   --     16     16 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                new_instance                                 --      1      1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:219:new_instance (new_instance)
                    __init__                                 --      1      1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:520:__init__ (__init__)
                device_to_providers                          --      1      1 -- 0.00003 0.00003 -- onnxruntime_helper.py:150:device_to_providers (device_to_providers)
            value                                            --    100    100 -- 0.00011 0.00011 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:186:value (value) +++
            init_learning_rate                               --      1      1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:205:init_learning_rate (init_learning_rate)
            update_learning_rate                             --    100    100 -- 0.00337 0.00337 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:226:update_learning_rate (update_learning_rate)
            proto_type_to_dtype                              --      6      6 -- 0.00003 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/utils/onnx_helper.py:61:proto_type_to_dtype (proto_type_to_dtype)
            <method 'randn' of 'num....RandomState' objects> --      6      6 -- 0.00076 0.00076 -- ~:0:<method 'randn' of 'numpy.random.mtrand.RandomState' objects> (<method 'randn' of 'numpy.random.mtrand.RandomState' objects>)
            <method 'append' of 'list' objects>              --    107    107 -- 0.00010 0.00010 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
            <built-in method builtins.len>                   --    108    108 -- 0.00007 0.00007 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
        fit                                                  --      1      1 -- 0.00002 11.54121 -- sklearn/neural_network/_multilayer_perceptron.py:735:fit (fit)
            _fit                                             --      1      1 -- 0.00015 11.54118 -- sklearn/neural_network/_multilayer_perceptron.py:376:_fit (_fit)
                any                                          --      1      1 -- 0.00001 0.00011 -- <__array_function__ internals>:177:any (any)
                    _any_dispatcher                          --      1      1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2300:_any_dispatcher (_any_dispatcher)
                _initialize                                  --      1      1 -- 0.00006 0.00066 -- sklearn/neural_network/_multilayer_perceptron.py:319:_initialize (_initialize)
                    is_classifier                            --      1      1 -- 0.00000 0.00000 -- sklearn/base.py:960:is_classifier (is_classifier)
                    _init_coef                               --      3      3 -- 0.00011 0.00059 -- sklearn/neural_network/_multilayer_perceptron.py:359:_init_coef (_init_coef)
                        <method 'uniform'...mState' objects> --      6      6 -- 0.00041 0.00041 -- ~:0:<method 'uniform' of 'numpy.random.mtrand.RandomState' objects> (<method 'uniform' of 'numpy.random.mtrand.RandomState' objects>)
                <listcomp>                                   --      1      1 -- 0.00001 0.00006 -- sklearn/neural_network/_multilayer_perceptron.py:416:<listcomp> (<listcomp>)
                <listcomp>                                   --      1      1 -- 0.00001 0.00003 -- sklearn/neural_network/_multilayer_perceptron.py:421:<listcomp> (<listcomp>)
                _validate_hyperparameters                    --      1      1 -- 0.00003 0.00003 -- sklearn/neural_network/_multilayer_perceptron.py:445:_validate_hyperparameters (_validate_hyperparameters)
                _fit_stochastic                              --      1      1 -- 0.28744 11.53879 -- sklearn/neural_network/_multilayer_perceptron.py:553:_fit_stochastic (_fit_stochastic)
                    clip                                     --      1      1 -- 0.00001 0.00022 -- <__array_function__ internals>:177:clip (clip)
                        _clip_dispatcher                     --      1      1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2079:_clip_dispatcher (_clip_dispatcher)
                        <built-in method ..._array_function> --      1      1 -- 0.00000 0.00021 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                    _backprop                                --   5000   5000 -- 0.63084 8.00041 -- sklearn/neural_network/_multilayer_perceptron.py:240:_backprop (_backprop)
                        dot                                  --  15000  15000 -- 0.07069 0.29887 -- <__array_function__ internals>:177:dot (dot)
                            dot                              --  15000  15000 -- 0.01281 0.01281 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:736:dot (dot)
                            <built-in metho...rray_function> --  15000  15000 -- 0.21537 0.21537 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                        inplace_relu_derivative              --  10000  10000 -- 0.43476 0.43476 -- sklearn/neural_network/_base.py:132:inplace_relu_derivative (inplace_relu_derivative)
                        squared_loss                         --   5000   5000 -- 0.25856 0.91129 -- sklearn/neural_network/_base.py:158:squared_loss (squared_loss)
                            <method 'mean' ...rray' objects> --   5000   5000 -- 0.02989 0.65273 -- ~:0:<method 'mean' of 'numpy.ndarray' objects> (<method 'mean' of 'numpy.ndarray' objects>) +++
                        _forward_pass                        --   5000   5000 -- 0.50802 1.55989 -- sklearn/neural_network/_multilayer_perceptron.py:118:_forward_pass (_forward_pass)
                            inplace_identity                 --   5000   5000 -- 0.00501 0.00501 -- sklearn/neural_network/_base.py:13:inplace_identity (inplace_identity)
                            inplace_relu                     --  10000  10000 -- 0.34700 0.34700 -- sklearn/neural_network/_base.py:47:inplace_relu (inplace_relu)
                            safe_sparse_dot                  --  15000  15000 -- 0.65480 0.69987 -- sklearn/utils/extmath.py:120:safe_sparse_dot (safe_sparse_dot) +++
                        _compute_loss_grad                   --  15000  15000 -- 1.34554 3.79741 -- sklearn/neural_network/_multilayer_perceptron.py:176:_compute_loss_grad (_compute_loss_grad)
                            mean                             --  15000  15000 -- 0.07626 1.71267 -- <__array_function__ internals>:177:mean (mean)
                                _mean_dispatcher             --  15000  15000 -- 0.01427 0.01427 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3351:_mean_dispatcher (_mean_dispatcher)
                                <built-in met...ay_function> --  15000  15000 -- 0.06958 1.62214 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                            safe_sparse_dot                  --  15000  15000 -- 0.69437 0.73920 -- sklearn/utils/extmath.py:120:safe_sparse_dot (safe_sparse_dot) +++
                        safe_sparse_dot                      --  10000  10000 -- 0.30302 0.33221 -- sklearn/utils/extmath.py:120:safe_sparse_dot (safe_sparse_dot) +++
                        <method 'ravel' o...darray' objects> --  15000  15000 -- 0.03513 0.03513 -- ~:0:<method 'ravel' of 'numpy.ndarray' objects> (<method 'ravel' of 'numpy.ndarray' objects>)
                    _update_no_improvement_count             --    100    100 -- 0.00079 0.00079 -- sklearn/neural_network/_multilayer_perceptron.py:706:_update_no_improvement_count (_update_no_improvement_count)
                    update_params                            --   5000   5000 -- 0.48247 2.10999 -- sklearn/neural_network/_stochastic_optimizers.py:29:update_params (update_params)
                        <genexpr>                            --  35000  35000 -- 0.03425 0.03425 -- sklearn/neural_network/_stochastic_optimizers.py:43:<genexpr> (<genexpr>)
                        _get_updates                         --   5000   5000 -- 0.06994 1.59328 -- sklearn/neural_network/_stochastic_optimizers.py:169:_get_updates (_get_updates)
                            <listcomp>                       --   5000   5000 -- 1.52333 1.52333 -- sklearn/neural_network/_stochastic_optimizers.py:183:<listcomp> (<listcomp>)
                    __init__                                 --      1      1 -- 0.00001 0.00042 -- sklearn/neural_network/_stochastic_optimizers.py:121:__init__ (__init__)
                        __init__                             --      1      1 -- 0.00000 0.00000 -- sklearn/neural_network/_stochastic_optimizers.py:25:__init__ (__init__)
                        <listcomp>                           --      1      1 -- 0.00003 0.00040 -- sklearn/neural_network/_stochastic_optimizers.py:136:<listcomp> (<listcomp>)
                            zeros_like                       --      6      6 -- 0.00002 0.00037 -- <__array_function__ internals>:177:zeros_like (zeros_like)
                                _zeros_like_dispatcher       --      6      6 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numeric.py:72:_zeros_like_dispatcher (_zeros_like_dispatcher)
                                <built-in met...ay_function> --      6      6 -- 0.00003 0.00034 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                    iteration_ends                           --    100    100 -- 0.00120 0.00120 -- sklearn/neural_network/_stochastic_optimizers.py:138:iteration_ends (iteration_ends)
                    _safe_indexing                           --   5000   5000 -- 0.05548 1.03604 -- sklearn/utils/__init__.py:307:_safe_indexing (_safe_indexing) +++
                    shuffle                                  --    100    100 -- 0.00129 0.05356 -- sklearn/utils/__init__.py:602:shuffle (shuffle)
                        resample                             --    100    100 -- 0.00336 0.05227 -- sklearn/utils/__init__.py:452:resample (resample)
                            <listcomp>                       --    100    100 -- 0.00042 0.00074 -- sklearn/utils/__init__.py:593:<listcomp> (<listcomp>)
                                isspmatrix                   --    100    100 -- 0.00017 0.00033 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1291:isspmatrix (isspmatrix) +++
                            <listcomp>                       --    100    100 -- 0.00038 0.01982 -- sklearn/utils/__init__.py:594:<listcomp> (<listcomp>)
                                _safe_indexing               --    100    100 -- 0.00103 0.01945 -- sklearn/utils/__init__.py:307:_safe_indexing (_safe_indexing) +++
                            check_consistent_length          --    100    100 -- 0.00113 0.01723 -- sklearn/utils/validation.py:318:check_consistent_length (check_consistent_length) +++
                            check_random_state               --    100    100 -- 0.00073 0.00218 -- sklearn/utils/validation.py:1043:check_random_state (check_random_state) +++
                            <method 'shuffl...tate' objects> --    100    100 -- 0.00676 0.00676 -- ~:0:<method 'shuffle' of 'numpy.random.mtrand.RandomState' objects> (<method 'shuffle' of 'numpy.random.mtrand.RandomState' objects>)
                            <built-in method numpy.arange>   --    100    100 -- 0.00177 0.00177 -- ~:0:<built-in method numpy.arange> (<built-in method numpy.arange>) +++
                            <built-in metho...ltins.hasattr> --    100    100 -- 0.00023 0.00023 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                            <built-in method builtins.len>   --    200    200 -- 0.00017 0.00017 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                    gen_batches                              --   5100   5100 -- 0.04658 0.04738 -- sklearn/utils/__init__.py:712:gen_batches (gen_batches)
                        <built-in method ...tins.isinstance> --    100    100 -- 0.00032 0.00079 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
                    <method 'append' of 'list' objects>      --    100    100 -- 0.00013 0.00013 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                    <built-in method _warnings.warn>         --      1      1 -- 0.00007 0.00120 -- ~:0:<built-in method _warnings.warn> (<built-in method _warnings.warn>)
                        _showwarnmsg                         --      1      1 -- 0.00002 0.00112 -- /usr/local/lib/python3.9/warnings.py:96:_showwarnmsg (_showwarnmsg)
                            _showwarning                     --      1      1 -- 0.00001 0.00111 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:473:_showwarning (_showwarning)
                                formatwarning                --      1      1 -- 0.00001 0.00006 -- /usr/local/lib/python3.9/warnings.py:15:formatwarning (formatwarning)
                                    _formatwarnmsg_impl      --      1      1 -- 0.00003 0.00005 -- /usr/local/lib/python3.9/warnings.py:35:_formatwarnmsg_impl (_formatwarnmsg_impl)
                                        getline              --      1      1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/linecache.py:26:getline (getline)
                                            getlines         --      1      1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/linecache.py:36:getlines (getlines)
                                    __init__                 --      1      1 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__) +++
                                write                        --      1      1 -- 0.00002 0.00103 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:81:write (write) +++
                        __init__                             --      1      1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__) +++
                _validate_input                              --      1      1 -- 0.00002 0.00131 -- sklearn/neural_network/_multilayer_perceptron.py:1588:_validate_input (_validate_input)
                    _validate_data                           --      1      1 -- 0.00003 0.00129 -- sklearn/base.py:495:_validate_data (_validate_data)
                        _check_n_features                    --      1      1 -- 0.00001 0.00002 -- sklearn/base.py:359:_check_n_features (_check_n_features)
                            _num_features                    --      1      1 -- 0.00001 0.00002 -- sklearn/utils/validation.py:201:_num_features (_num_features)
                        _check_feature_names                 --      1      1 -- 0.00000 0.00001 -- sklearn/base.py:405:_check_feature_names (_check_feature_names)
                            _get_feature_names               --      1      1 -- 0.00000 0.00000 -- sklearn/utils/validation.py:1653:_get_feature_names (_get_feature_names)
                        check_X_y                            --      1      1 -- 0.00001 0.00123 -- sklearn/utils/validation.py:845:check_X_y (check_X_y)
                            check_consistent_length          --      1      1 -- 0.00001 0.00021 -- sklearn/utils/validation.py:318:check_consistent_length (check_consistent_length) +++
                            check_array                      --      1      1 -- 0.00006 0.00066 -- sklearn/utils/validation.py:494:check_array (check_array) +++
                            _check_y                         --      1      1 -- 0.00001 0.00034 -- sklearn/utils/validation.py:986:_check_y (_check_y)
                                check_array                  --      1      1 -- 0.00004 0.00034 -- sklearn/utils/validation.py:494:check_array (check_array) +++
                check_random_state                           --      1      1 -- 0.00001 0.00001 -- sklearn/utils/validation.py:1043:check_random_state (check_random_state) +++
        <built-in method builtins.print>                     --      3      3 -- 0.00004 0.00237 -- ~:0:<built-in method builtins.print> (<built-in method builtins.print>)
            write                                            --      6      6 -- 0.00015 0.00233 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:81:write (write) +++
_bio_cache                                                   --  130000  130000 -- 0.43257 0.53726 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:157:_bio_cache (_bio_cache)
    <built-in method builtins.id>                            --  130000  130000 -- 0.10470 0.10470 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>) +++
_bio_ptr                                                     --  130000  130000 -- 0.78539 0.78539 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:174:_bio_ptr (_bio_ptr)
_bind_input_ortvalue                                         --  100000  100000 -- 0.51813 1.76320 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:178:_bind_input_ortvalue (_bind_input_ortvalue)
    _bio_cache                                               --  100000  100000 -- 0.33771 0.41951 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:157:_bio_cache (_bio_cache) +++
    _bio_do_bind_in                                          --  10593  10593 -- 0.16148 0.16148 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:170:_bio_do_bind_in (_bio_do_bind_in)
    _bio_ptr                                                 --  100000  100000 -- 0.60996 0.60996 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:174:_bio_ptr (_bio_ptr) +++
    <built-in method builtins.isinstance>                    --  100000  100000 -- 0.05412 0.05412 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_get_att_state                                               --    103    103 -- 0.00012 0.00012 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:147:_get_att_state (_get_att_state)
get_full_state                                               --    101    201 -- 0.00121 0.00222 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:155:get_full_state (get_full_state)
    _get_att_state                                           --    101    101 -- 0.00012 0.00012 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:147:_get_att_state (_get_att_state) +++
    <listcomp>                                               --    100    100 -- 0.00053 0.00146 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:163:<listcomp> (<listcomp>)
        get_full_state                                       --    100    100 -- 0.00051 0.00092 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:155:get_full_state (get_full_state) +++
    <built-in method builtins.getattr>                       --    101    101 -- 0.00007 0.00007 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
    <built-in method builtins.hasattr>                       --    101    101 -- 0.00009 0.00009 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    <built-in method builtins.isinstance>                    --    201    201 -- 0.00020 0.00020 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_provider_name_to_device_type                                --      9      9 -- 0.00002 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:271:_provider_name_to_device_type (_provider_name_to_device_type)
input_to_ort                                                 --  10000  10000 -- 0.37656 0.50362 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:577:input_to_ort (input_to_ort)
    <built-in method builtins.all>                           --  10000  10000 -- 0.04346 0.08831 -- ~:0:<built-in method builtins.all> (<built-in method builtins.all>) +++
    <built-in method builtins.isinstance>                    --  10000  10000 -- 0.03055 0.03055 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.len>                           --  10000  10000 -- 0.00819 0.00819 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
value                                                        --  30100  30100 -- 0.03523 0.03523 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:186:value (value)
_mean                                                        --  20100  20100 -- 1.01900 2.03772 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:162:_mean (_mean)
    _count_reduce_items                                      --  20100  20100 -- 0.38435 0.43986 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:66:_count_reduce_items (_count_reduce_items)
        <built-in method numpy.co...th.normalize_axis_index> --  25200  25200 -- 0.03685 0.03685 -- ~:0:<built-in method numpy.core._multiarray_umath.normalize_axis_index> (<built-in method numpy.core._multiarray_umath.normalize_axis_index>)
        <built-in method builtins.isinstance>                --  15000  15000 -- 0.01865 0.01865 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method numpy.asanyarray>                       --  20100  20100 -- 0.01596 0.01596 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>) +++
    <method 'reduce' of 'numpy.ufunc' objects>               --  20100  20100 -- 0.49958 0.49958 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>) +++
    <built-in method builtins.hasattr>                       --   5100   5100 -- 0.00913 0.00913 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    <built-in method builtins.isinstance>                    --  20100  20100 -- 0.01898 0.01898 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.issubclass>                    --  40200  40200 -- 0.03521 0.03521 -- ~:0:<built-in method builtins.issubclass> (<built-in method builtins.issubclass>) +++
_wrapreduction                                               --      3      3 -- 0.00005 0.00043 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:69:_wrapreduction (_wrapreduction)
    <dictcomp>                                               --      3      3 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:70:<dictcomp> (<dictcomp>)
    <method 'reduce' of 'numpy.ufunc' objects>               --      3      3 -- 0.00037 0.00037 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>) +++
isspmatrix                                                   --  45202  45202 -- 0.07324 0.13190 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1291:isspmatrix (isspmatrix)
    <built-in method builtins.isinstance>                    --  45202  45202 -- 0.05867 0.05867 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
write                                                        --      7      7 -- 0.00018 0.00336 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:81:write (write)
    verbose                                                  --      6      6 -- 0.00005 0.00316 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:127:verbose (verbose)
        log                                                  --      6      6 -- 0.00008 0.00311 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:120:log (log)
            log                                              --      6      6 -- 0.00009 0.00303 -- /usr/local/lib/python3.9/logging/__init__.py:1825:log (log)
                log                                          --      6      6 -- 0.00008 0.00284 -- /usr/local/lib/python3.9/logging/__init__.py:1485:log (log)
                    _log                                     --      6      6 -- 0.00006 0.00275 -- /usr/local/lib/python3.9/logging/__init__.py:1553:_log (_log)
                        findCaller                           --      6      6 -- 0.00010 0.00017 -- /usr/local/lib/python3.9/logging/__init__.py:1502:findCaller (findCaller)
                            <lambda>                         --      6      6 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:156:<lambda> (<lambda>)
                            normcase                         --     12     12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/posixpath.py:52:normcase (normcase)
                                <built-in met...osix.fspath> --     12     12 -- 0.00001 0.00001 -- ~:0:<built-in method posix.fspath> (<built-in method posix.fspath>) +++
                            <built-in metho...ltins.hasattr> --     12     12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                        makeRecord                           --      6      6 -- 0.00006 0.00080 -- /usr/local/lib/python3.9/logging/__init__.py:1538:makeRecord (makeRecord)
                            __init__                         --      6      6 -- 0.00032 0.00075 -- /usr/local/lib/python3.9/logging/__init__.py:278:__init__ (__init__)
                                getLevelName                 --      6      6 -- 0.00005 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:119:getLevelName (getLevelName)
                                    <method 'ge...' objects> --     12     12 -- 0.00001 0.00001 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>) +++
                                current_process              --      6      6 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/multiprocessing/process.py:37:current_process (current_process)
                                name                         --      6      6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/multiprocessing/process.py:189:name (name)
                                splitext                     --      6      6 -- 0.00003 0.00010 -- /usr/local/lib/python3.9/posixpath.py:117:splitext (splitext)
                                    _splitext                --      6      6 -- 0.00004 0.00006 -- /usr/local/lib/python3.9/genericpath.py:121:_splitext (_splitext)
                                        <method '...objects> --     12     12 -- 0.00002 0.00002 -- ~:0:<method 'rfind' of 'str' objects> (<method 'rfind' of 'str' objects>) +++
                                basename                     --      6      6 -- 0.00005 0.00009 -- /usr/local/lib/python3.9/posixpath.py:140:basename (basename)
                                    _get_sep                 --      6      6 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/posixpath.py:41:_get_sep (_get_sep)
                                name                         --      6      6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/threading.py:1053:name (name)
                                current_thread               --      6      6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/threading.py:1318:current_thread (current_thread)
                        handle                               --      6      6 -- 0.00003 0.00172 -- /usr/local/lib/python3.9/logging/__init__.py:1579:handle (handle)
                            filter                           --      6      6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter) +++
                            callHandlers                     --      6      6 -- 0.00007 0.00168 -- /usr/local/lib/python3.9/logging/__init__.py:1633:callHandlers (callHandlers)
                                handle                       --     12     12 -- 0.00008 0.00161 -- /usr/local/lib/python3.9/logging/__init__.py:935:handle (handle)
                                    filter                   --     12     12 -- 0.00007 0.00019 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter) +++
                                    acquire                  --     12     12 -- 0.00004 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
                                    release                  --     12     12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
                                    emit                     --      6      6 -- 0.00004 0.00041 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit) +++
                                    emit                     --      6      6 -- 0.00007 0.00084 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:150:emit (emit)
                                        acquire              --      6      6 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
                                        release              --      6      6 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
                                        emit                 --      6      6 -- 0.00005 0.00073 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit) +++
                    isEnabledFor                             --      6      6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor) +++
                isEnabledFor                                 --      6      6 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:1834:isEnabledFor (isEnabledFor)
                    isEnabledFor                             --      6      6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor) +++
                process                                      --      6      6 -- 0.00005 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:130:process (process)
numpy_to_ort_value                                           --  10014  10014 -- 0.02986 0.16333 -- onnxruntime_helper.py:134:numpy_to_ort_value (numpy_to_ort_value)
    <built-in method onnxruntim...state.ortvalue_from_numpy> --  10014  10014 -- 0.13347 0.13347 -- ~:0:<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy> (<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy>) +++
_safe_indexing                                               --   5100   5100 -- 0.05651 1.05549 -- sklearn/utils/__init__.py:307:_safe_indexing (_safe_indexing)
    _array_indexing                                          --   5100   5100 -- 0.22130 0.86015 -- sklearn/utils/__init__.py:193:_array_indexing (_array_indexing)
        isspmatrix                                           --   5100   5100 -- 0.00691 0.01248 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1291:isspmatrix (isspmatrix) +++
        parse                                                --   5100   5100 -- 0.02599 0.60096 -- sklearn/externals/_packaging/version.py:65:parse (parse)
            __init__                                         --   5100   5100 -- 0.24509 0.57498 -- sklearn/externals/_packaging/version.py:284:__init__ (__init__)
                <lambda>                                     --   5100   5100 -- 0.01310 0.02516 -- <string>:1:<lambda> (<lambda>)
                    <built-in method __...at 0x7f943acac960> --   5100   5100 -- 0.01205 0.01205 -- ~:0:<built-in method __new__ of type object at 0x7f943acac960> (<built-in method __new__ of type object at 0x7f943acac960>)
                <genexpr>                                    --  15300  15300 -- 0.03846 0.03846 -- sklearn/externals/_packaging/version.py:294:<genexpr> (<genexpr>)
                _parse_letter_version                        --  15300  15300 -- 0.01239 0.01239 -- sklearn/externals/_packaging/version.py:416:_parse_letter_version (_parse_letter_version)
                _parse_local_version                         --   5100   5100 -- 0.00382 0.00382 -- sklearn/externals/_packaging/version.py:455:_parse_local_version (_parse_local_version)
                _cmpkey                                      --   5100   5100 -- 0.08449 0.08973 -- sklearn/externals/_packaging/version.py:467:_cmpkey (_cmpkey)
                    <lambda>                                 --   5100   5100 -- 0.00524 0.00524 -- sklearn/externals/_packaging/version.py:482:<lambda> (<lambda>)
                <method 'split' of 'str' objects>            --   5100   5100 -- 0.01222 0.01222 -- ~:0:<method 'split' of 'str' objects> (<method 'split' of 'str' objects>)
                <method 'group' of 're.Match' objects>       --  51000  51000 -- 0.05993 0.05993 -- ~:0:<method 'group' of 're.Match' objects> (<method 'group' of 're.Match' objects>)
                <method 'search' of 're.Pattern' objects>    --   5100   5100 -- 0.08817 0.08817 -- ~:0:<method 'search' of 're.Pattern' objects> (<method 'search' of 're.Pattern' objects>)
        __lt__                                               --   5100   5100 -- 0.01831 0.02256 -- sklearn/externals/_packaging/version.py:92:__lt__ (__lt__)
            <built-in method builtins.isinstance>            --   5100   5100 -- 0.00425 0.00425 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
        <built-in method builtins.isinstance>                --   5100   5100 -- 0.00284 0.00284 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    _determine_key_type                                      --   5100   5100 -- 0.09625 0.12472 -- sklearn/utils/__init__.py:237:_determine_key_type (_determine_key_type)
        <method 'keys' of 'dict' objects>                    --   5100   5100 -- 0.00636 0.00636 -- ~:0:<method 'keys' of 'dict' objects> (<method 'keys' of 'dict' objects>)
        <built-in method builtins.hasattr>                   --   5100   5100 -- 0.00423 0.00423 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
        <built-in method builtins.isinstance>                --  15300  15300 -- 0.01787 0.01787 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.hasattr>                       --  10200  10200 -- 0.01411 0.01411 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
safe_sparse_dot                                              --  40000  40000 -- 1.65219 1.77128 -- sklearn/utils/extmath.py:120:safe_sparse_dot (safe_sparse_dot)
    isspmatrix                                               --  40000  40000 -- 0.06616 0.11909 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1291:isspmatrix (isspmatrix) +++
_num_samples                                                 --    104    104 -- 0.00232 0.00441 -- sklearn/utils/validation.py:254:_num_samples (_num_samples)
    <built-in method builtins.hasattr>                       --    312    312 -- 0.00046 0.00046 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    <built-in method builtins.isinstance>                    --    104    104 -- 0.00031 0.00158 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.len>                           --    104    104 -- 0.00006 0.00006 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
check_consistent_length                                      --    101    101 -- 0.00114 0.01744 -- sklearn/utils/validation.py:318:check_consistent_length (check_consistent_length)
    unique                                                   --    101    101 -- 0.00055 0.01151 -- <__array_function__ internals>:177:unique (unique)
        _unique_dispatcher                                   --    101    101 -- 0.00009 0.00009 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:133:_unique_dispatcher (_unique_dispatcher)
        <built-in method numpy.co...mplement_array_function> --    101    101 -- 0.00077 0.01086 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
    <listcomp>                                               --    101    101 -- 0.00041 0.00473 -- sklearn/utils/validation.py:329:<listcomp> (<listcomp>)
        _num_samples                                         --    102    102 -- 0.00228 0.00432 -- sklearn/utils/validation.py:254:_num_samples (_num_samples) +++
    <built-in method builtins.len>                           --    101    101 -- 0.00006 0.00006 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
check_array                                                  --      2      2 -- 0.00010 0.00099 -- sklearn/utils/validation.py:494:check_array (check_array)
    simplefilter                                             --      2      2 -- 0.00001 0.00007 -- /usr/local/lib/python3.9/warnings.py:165:simplefilter (simplefilter)
        _add_filter                                          --      2      2 -- 0.00003 0.00005 -- /usr/local/lib/python3.9/warnings.py:181:_add_filter (_add_filter)
    __init__                                                 --      2      2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:437:__init__ (__init__)
    __enter__                                                --      2      2 -- 0.00003 0.00003 -- /usr/local/lib/python3.9/warnings.py:458:__enter__ (__enter__)
    __exit__                                                 --      2      2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:477:__exit__ (__exit__)
    isspmatrix                                               --      2      2 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1291:isspmatrix (isspmatrix) +++
    _assert_all_finite                                       --      2      2 -- 0.00012 0.00064 -- sklearn/utils/validation.py:90:_assert_all_finite (_assert_all_finite)
        get_config                                           --      2      2 -- 0.00001 0.00002 -- sklearn/_config.py:24:get_config (get_config)
            _get_threadlocal_config                          --      2      2 -- 0.00000 0.00001 -- sklearn/_config.py:16:_get_threadlocal_config (_get_threadlocal_config)
        _safe_accumulator_op                                 --      2      2 -- 0.00003 0.00048 -- sklearn/utils/extmath.py:869:_safe_accumulator_op (_safe_accumulator_op)
            sum                                              --      2      2 -- 0.00001 0.00041 -- <__array_function__ internals>:177:sum (sum)
                _sum_dispatcher                              --      2      2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2155:_sum_dispatcher (_sum_dispatcher)
                <built-in method nump...ment_array_function> --      2      2 -- 0.00001 0.00040 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
            issubdtype                                       --      2      2 -- 0.00001 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numerictypes.py:356:issubdtype (issubdtype)
                issubclass_                                  --      4      4 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numerictypes.py:282:issubclass_ (issubclass_)
    _num_samples                                             --      2      2 -- 0.00004 0.00009 -- sklearn/utils/validation.py:254:_num_samples (_num_samples) +++
    _ensure_no_complex_data                                  --      2      2 -- 0.00001 0.00001 -- sklearn/utils/validation.py:484:_ensure_no_complex_data (_ensure_no_complex_data)
check_random_state                                           --    101    101 -- 0.00073 0.00219 -- sklearn/utils/validation.py:1043:check_random_state (check_random_state)
    <built-in method builtins.isinstance>                    --    200    200 -- 0.00061 0.00146 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
<built-in method builtins.len>                               --  50957  50957 -- 0.07959 0.10229 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>)
    __len__                                                  --  10200  10200 -- 0.02270 0.02270 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:94:__len__ (__len__)
<method 'astype' of 'numpy.ndarray' objects>                 --     18     18 -- 0.00025 0.00025 -- ~:0:<method 'astype' of 'numpy.ndarray' objects> (<method 'astype' of 'numpy.ndarray' objects>)
<method 'append' of 'list' objects>                          --  10253  10253 -- 0.01526 0.01526 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>)
<built-in method builtins.hasattr>                           --  91002  91002 -- 0.08570 0.08570 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>)
<built-in method builtins.isinstance>                        --  286510  286510 -- 0.23505 0.23770 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>)
    __instancecheck__                                        --    314    314 -- 0.00049 0.00265 -- /usr/local/lib/python3.9/abc.py:96:__instancecheck__ (__instancecheck__)
        <built-in method _abc._abc_instancecheck>            --    314    314 -- 0.00152 0.00216 -- ~:0:<built-in method _abc._abc_instancecheck> (<built-in method _abc._abc_instancecheck>)
            __subclasscheck__                                --    104    104 -- 0.00015 0.00064 -- /usr/local/lib/python3.9/abc.py:100:__subclasscheck__ (__subclasscheck__)
                <built-in method _abc._abc_subclasscheck>    --    104    104 -- 0.00049 0.00049 -- ~:0:<built-in method _abc._abc_subclasscheck> (<built-in method _abc._abc_subclasscheck>)
<built-in method builtins.getattr>                           --    148    148 -- 0.00013 0.00013 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>)
<method 'mean' of 'numpy.ndarray' objects>                   --   5100   5100 -- 0.03059 0.66811 -- ~:0:<method 'mean' of 'numpy.ndarray' objects> (<method 'mean' of 'numpy.ndarray' objects>)
    _mean                                                    --   5100   5100 -- 0.31638 0.63752 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:162:_mean (_mean) +++
<built-in method numpy.array>                                --    101    101 -- 0.00834 0.00834 -- ~:0:<built-in method numpy.array> (<built-in method numpy.array>)
<built-in method onnxruntime....1_state.ortvalue_from_numpy> --  40014  40014 -- 0.57278 0.57278 -- ~:0:<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy> (<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy>)
<built-in method builtins.id>                                --  135001  135001 -- 0.11119 0.11119 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>)
<built-in method numpy.empty>                                --    107    107 -- 0.00066 0.00066 -- ~:0:<built-in method numpy.empty> (<built-in method numpy.empty>)
<built-in method numpy.arange>                               --    101    101 -- 0.00179 0.00179 -- ~:0:<built-in method numpy.arange> (<built-in method numpy.arange>)
<built-in method numpy.core._...th.implement_array_function> --  30111  30125 -- 0.28591 1.84942 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>)
    clip                                                     --      1      1 -- 0.00001 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2083:clip (clip)
        _wrapfunc                                            --      1      1 -- 0.00001 0.00020 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:51:_wrapfunc (_wrapfunc)
            _wrapit                                          --      1      1 -- 0.00002 0.00019 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:38:_wrapit (_wrapit)
                <method 'clip' of 'numpy.ndarray' objects>   --      1      1 -- 0.00001 0.00016 -- ~:0:<method 'clip' of 'numpy.ndarray' objects> (<method 'clip' of 'numpy.ndarray' objects>)
                    _clip                                    --      1      1 -- 0.00002 0.00015 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:125:_clip (_clip)
                        _clip_dep_is_scalar_nan              --      2      2 -- 0.00006 0.00010 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:91:_clip_dep_is_scalar_nan (_clip_dep_is_scalar_nan)
                            ndim                             --      2      2 -- 0.00001 0.00004 -- <__array_function__ internals>:177:ndim (ndim)
                                _ndim_dispatcher             --      2      2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3160:_ndim_dispatcher (_ndim_dispatcher)
                        _clip_dep_is_byte_swapped            --      2      2 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:101:_clip_dep_is_byte_swapped (_clip_dep_is_byte_swapped)
                        _clip_dep_invoke_with_casting        --      1      1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:106:_clip_dep_invoke_with_casting (_clip_dep_invoke_with_casting)
    sum                                                      --      2      2 -- 0.00002 0.00038 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2160:sum (sum)
        _wrapreduction                                       --      2      2 -- 0.00002 0.00036 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:69:_wrapreduction (_wrapreduction) +++
    any                                                      --      1      1 -- 0.00001 0.00008 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2305:any (any)
        _wrapreduction                                       --      1      1 -- 0.00003 0.00007 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:69:_wrapreduction (_wrapreduction) +++
    ndim                                                     --      2      2 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3164:ndim (ndim)
    mean                                                     --  15000  15000 -- 0.15235 1.55255 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3356:mean (mean)
        _mean                                                --  15000  15000 -- 0.70261 1.40020 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:162:_mean (_mean) +++
    zeros_like                                               --      6      6 -- 0.00007 0.00032 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numeric.py:76:zeros_like (zeros_like)
        copyto                                               --      6      6 -- 0.00003 0.00010 -- <__array_function__ internals>:177:copyto (copyto)
            copyto                                           --      6      6 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:1071:copyto (copyto)
        empty_like                                           --      6      6 -- 0.00004 0.00010 -- <__array_function__ internals>:177:empty_like (empty_like)
            empty_like                                       --      6      6 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:80:empty_like (empty_like)
    unique                                                   --    101    101 -- 0.00103 0.01009 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:138:unique (unique)
        _unpack_tuple                                        --    101    101 -- 0.00026 0.00032 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:125:_unpack_tuple (_unpack_tuple)
            <built-in method builtins.len>                   --    101    101 -- 0.00007 0.00007 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
        _unique1d                                            --    101    101 -- 0.00600 0.00768 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:320:_unique1d (_unique1d)
            <method 'flatten' of 'numpy.ndarray' objects>    --    101    101 -- 0.00082 0.00082 -- ~:0:<method 'flatten' of 'numpy.ndarray' objects> (<method 'flatten' of 'numpy.ndarray' objects>)
            <method 'sort' of 'numpy.ndarray' objects>       --    101    101 -- 0.00022 0.00022 -- ~:0:<method 'sort' of 'numpy.ndarray' objects> (<method 'sort' of 'numpy.ndarray' objects>)
            <built-in method numpy.asanyarray>               --    101    101 -- 0.00005 0.00005 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>) +++
            <built-in method numpy.empty>                    --    101    101 -- 0.00059 0.00059 -- ~:0:<built-in method numpy.empty> (<built-in method numpy.empty>) +++
        <built-in method numpy.asanyarray>                   --    101    101 -- 0.00106 0.00106 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>) +++
<method 'HasField' of 'google...._message.CMessage' objects> --     18     18 -- 0.00006 0.00006 -- ~:0:<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects>)
<built-in method numpy.asarray>                              --     11     11 -- 0.00236 0.00236 -- ~:0:<built-in method numpy.asarray> (<built-in method numpy.asarray>)
<built-in method builtins.all>                               --  10004  10004 -- 0.04346 0.08832 -- ~:0:<built-in method builtins.all> (<built-in method builtins.all>)
    <lambda>                                                 --  40000  40000 -- 0.03280 0.04486 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:597:<lambda> (<lambda>)
        <built-in method builtins.isinstance>                --  40000  40000 -- 0.01206 0.01206 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
<built-in method numpy.asanyarray>                           --  20304  20304 -- 0.01707 0.01707 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>)
<method 'reduce' of 'numpy.ufunc' objects>                   --  20103  20103 -- 0.49995 0.49995 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>)
<built-in method builtins.issubclass>                        --  40206  40206 -- 0.03522 0.03522 -- ~:0:<built-in method builtins.issubclass> (<built-in method builtins.issubclass>)
<method 'get' of 'dict' objects>                             --     33     33 -- 0.00003 0.00003 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>)
<built-in method posix.fspath>                               --     24     24 -- 0.00001 0.00001 -- ~:0:<built-in method posix.fspath> (<built-in method posix.fspath>)
<built-in method _thread.get_ident>                          --     12     12 -- 0.00001 0.00001 -- ~:0:<built-in method _thread.get_ident> (<built-in method _thread.get_ident>)
<method 'rfind' of 'str' objects>                            --     18     18 -- 0.00003 0.00003 -- ~:0:<method 'rfind' of 'str' objects> (<method 'rfind' of 'str' objects>)

if GPU is available#

if get_device().upper() == 'GPU':

    train_session = OrtGradientForwardBackwardOptimizer(
        onx, device='cuda', learning_rate=1e-5,
        warm_start=False, max_iter=max_iter, batch_size=batch_size)

    benches.append(benchmark(X_train, y_train, nn,
                   train_session, name='NN-GPU'))

Linear Regression#

lr = MLPRegressor(hidden_layer_sizes=tuple(), max_iter=max_iter,
                  solver='sgd', learning_rate_init=5e-2, alpha=0,
                  n_iter_no_change=max_iter * 3, batch_size=batch_size,
                  nesterovs_momentum=False, momentum=0,
                  learning_rate="invscaling")

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    lr.fit(X, y)


onx = to_onnx(lr, X_train[:1].astype(numpy.float32), target_opset=15)

train_session = OrtGradientForwardBackwardOptimizer(
    onx, device='cpu', learning_rate=5e-4,
    warm_start=False, max_iter=max_iter, batch_size=batch_size)

benches.append(benchmark(X_train, y_train, lr, train_session, name='LR-CPU'))

if get_device().upper() == 'GPU':

    train_session = OrtGradientForwardBackwardOptimizer(
        onx, device='cuda', learning_rate=5e-4,
        warm_start=False, max_iter=max_iter, batch_size=batch_size)

    benches.append(benchmark(X_train, y_train, nn,
                   train_session, name='LR-GPU'))

Out:

[benchmark] LR-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:692: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  warnings.warn(
[benchmark] skl=100 iterations - 3.828441008925438 seconds
[benchmark] ort=100 iteration - 4.971634171903133 seconds

GPU profiling#

if get_device().upper() == 'GPU':
    ps = profile(lambda: benchmark(X_train, y_train,
                 lr, train_session, name='LR-GPU'))[0]
    root, nodes = profile2graph(ps, clean_text=clean_name)
    text = root.to_text()
    print(text)

Graphs#

Dataframe first.

df = DataFrame(benches).set_index('name')
df
skl ort iter_skl iter_ort losses_skl losses_ort
name
NN-CPU 10.036501 10.246102 100 100 [7589.4960107421875, 1386.605005493164, 918.05... [9429.534, 7956.5093, 8667.631, 8639.109, 7085...
LR-CPU 3.828441 4.971634 100 100 [2184.9586044311523, 96.86819942474365, 86.457... [7251.0786, 7841.8813, 7501.6963, 7084.753, 64...


text output

print(df)

Out:

              skl  ...                                         losses_ort
name               ...
NN-CPU  10.036501  ...  [9429.534, 7956.5093, 8667.631, 8639.109, 7085...
LR-CPU   3.828441  ...  [7251.0786, 7841.8813, 7501.6963, 7084.753, 64...

[2 rows x 6 columns]

Graphs.

fig, ax = plt.subplots(1, 2, figsize=(10, 4))
df[['skl', 'ort']].plot.bar(title="Processing time", ax=ax[0])
ax[0].tick_params(axis='x', rotation=30)
for bench in benches:
    ax[1].plot(bench['losses_skl'][1:], label='skl-' + bench['name'])
    ax[1].plot(bench['losses_ort'][1:], label='ort-' + bench['name'])
ax[1].set_yscale('log')
ax[1].set_title("Losses")
ax[1].legend()
Processing time, Losses

Out:

<matplotlib.legend.Legend object at 0x7f9360cfb370>

The gradient update are not exactly the same. It should be improved for a fair comprison.

fig.savefig(__file__ + ".png")
# plt.show()
Traceback (most recent call last):
  File "somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_fwbw.py", line 218, in <module>
    fig.savefig(__file__ + ".png")
NameError: name '__file__' is not defined

Total running time of the script: ( 1 minutes 9.506 seconds)

Gallery generated by Sphinx-Gallery