Benchmark, comparison sklearn - forward-backward - classification#

The benchmark compares the processing time between scikit-learn and onnxruntime-training on a logistic regression regression and a neural network for classification. It replicates the benchmark implemented in Benchmark, comparison scikit-learn - forward-backward.

First comparison: neural network#

import warnings
import time
import numpy
import matplotlib.pyplot as plt
from pandas import DataFrame
from onnxruntime import get_device
from pyquickhelper.pycode.profiling import profile, profile2graph
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from mlprodict.onnx_conv import to_onnx
from mlprodict.plotting.text_plot import onnx_simple_text_plot
from mlprodict.onnx_tools.onnx_manipulations import select_model_inputs_outputs
from onnxcustom.utils.onnx_helper import onnx_rename_weights
from onnxcustom.training.optimizers_partial import (
    OrtGradientForwardBackwardOptimizer)
from onnxcustom.training.sgd_learning_rate import LearningRateSGDNesterov
from onnxcustom.training.sgd_learning_loss import NegLogLearningLoss
from onnxcustom.training.sgd_learning_penalty import ElasticLearningPenalty


X, y = make_classification(1000, n_features=100, n_classes=2)
X = X.astype(numpy.float32)
y = y.astype(numpy.int64)
X_train, X_test, y_train, y_test = train_test_split(X, y)

Benchmark function.

def benchmark(X, y, skl_model, train_session, name, verbose=True):
    """
    :param skl_model: model from scikit-learn
    :param train_session: instance of OrtGradientForwardBackwardOptimizer
    :param name: experiment name
    :param verbose: to debug
    """
    print(f"[benchmark] {name}")
    begin = time.perf_counter()
    skl_model.fit(X, y)
    duration_skl = time.perf_counter() - begin
    length_skl = len(skl_model.loss_curve_)
    print(f"[benchmark] skl={length_skl!r} iterations - {duration_skl!r} seconds")

    begin = time.perf_counter()
    train_session.fit(X, y)
    duration_ort = time.perf_counter() - begin
    length_ort = len(train_session.train_losses_)
    print(f"[benchmark] ort={length_ort!r} iteration - {duration_ort!r} seconds")

    return dict(skl=duration_skl, ort=duration_ort, name=name,
                iter_skl=length_skl, iter_ort=length_ort,
                losses_skl=skl_model.loss_curve_,
                losses_ort=train_session.train_losses_)

Common parameters and model

batch_size = 15
max_iter = 100

nn = MLPClassifier(hidden_layer_sizes=(50, 10), max_iter=max_iter,
                   solver='sgd', learning_rate_init=1e-1, alpha=1e-4,
                   n_iter_no_change=max_iter * 3, batch_size=batch_size,
                   nesterovs_momentum=True, momentum=0.9,
                   learning_rate="invscaling")

with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    nn.fit(X_train, y_train)

Conversion to ONNX and trainer initialization It is slightly different from a regression model. Probabilities usually come from raw scores transformed through a function such as the sigmoid function. The gradient of the loss is computed against the raw scores because it is easier to compute than to let onnxruntime do it.

onx = to_onnx(nn, X_train[:1].astype(numpy.float32), target_opset=15,
              options={'zipmap': False})

try:
    print(onnx_simple_text_plot(onx))
except RuntimeError as e:
    print("You should upgrade mlprodict.")
    print(e)
opset: domain='' version=14
opset: domain='ai.onnx.ml' version=1
input: name='X' type=dtype('float32') shape=[None, 100]
init: name='coefficient' type=dtype('float32') shape=(5000,)
init: name='intercepts' type=dtype('float32') shape=(50,)
init: name='coefficient1' type=dtype('float32') shape=(500,)
init: name='intercepts1' type=dtype('float32') shape=(10,)
init: name='coefficient2' type=dtype('float32') shape=(10,)
init: name='intercepts2' type=dtype('float32') shape=(1,) -- array([-0.20477049], dtype=float32)
init: name='unity' type=dtype('float32') shape=(1,) -- array([1.], dtype=float32)
init: name='classes' type=dtype('int32') shape=(2,) -- array([0, 1], dtype=int32)
init: name='shape_tensor' type=dtype('int64') shape=(1,) -- array([-1])
Cast(X, to=1) -> cast_input
  MatMul(cast_input, coefficient) -> mul_result
    Add(mul_result, intercepts) -> add_result
      Relu(add_result) -> next_activations
        MatMul(next_activations, coefficient1) -> mul_result1
          Add(mul_result1, intercepts1) -> add_result1
            Relu(add_result1) -> next_activations1
              MatMul(next_activations1, coefficient2) -> mul_result2
                Add(mul_result2, intercepts2) -> add_result2
                  Sigmoid(add_result2) -> out_activations_result
                    Sub(unity, out_activations_result) -> negative_class_proba
                    Concat(negative_class_proba, out_activations_result, axis=1) -> probabilities
                      ArgMax(probabilities, axis=1) -> argmax_output
                        ArrayFeatureExtractor(classes, argmax_output) -> array_feature_extractor_result
                          Reshape(array_feature_extractor_result, shape_tensor) -> reshaped_result
                            Cast(reshaped_result, to=7) -> label
output: name='label' type=dtype('int64') shape=[None]
output: name='probabilities' type=dtype('float32') shape=[None, 2]

Raw scores are the input of operator Sigmoid.

onx = select_model_inputs_outputs(
    onx, outputs=["add_result2"], infer_shapes=True)
print(onnx_simple_text_plot(onx))
opset: domain='' version=14
opset: domain='ai.onnx.ml' version=1
input: name='X' type=dtype('float32') shape=[None, 100]
init: name='coefficient' type=dtype('float32') shape=(5000,)
init: name='intercepts' type=dtype('float32') shape=(50,)
init: name='coefficient1' type=dtype('float32') shape=(500,)
init: name='intercepts1' type=dtype('float32') shape=(10,)
init: name='coefficient2' type=dtype('float32') shape=(10,)
init: name='intercepts2' type=dtype('float32') shape=(1,) -- array([-0.20477049], dtype=float32)
Cast(X, to=1) -> cast_input
  MatMul(cast_input, coefficient) -> mul_result
    Add(mul_result, intercepts) -> add_result
      Relu(add_result) -> next_activations
        MatMul(next_activations, coefficient1) -> mul_result1
          Add(mul_result1, intercepts1) -> add_result1
            Relu(add_result1) -> next_activations1
              MatMul(next_activations1, coefficient2) -> mul_result2
                Add(mul_result2, intercepts2) -> add_result2
output: name='add_result2' type=dtype('float32') shape=['unk__0', 1]

And the names are renamed to have them follow the alphabetical order (see OrtGradientForwardBackward).

onx = onnx_rename_weights(onx)
print(onnx_simple_text_plot(onx))
opset: domain='' version=14
opset: domain='ai.onnx.ml' version=1
input: name='X' type=dtype('float32') shape=[None, 100]
init: name='I0_coefficient' type=dtype('float32') shape=(5000,)
init: name='I1_intercepts' type=dtype('float32') shape=(50,)
init: name='I2_coefficient1' type=dtype('float32') shape=(500,)
init: name='I3_intercepts1' type=dtype('float32') shape=(10,)
init: name='I4_coefficient2' type=dtype('float32') shape=(10,)
init: name='I5_intercepts2' type=dtype('float32') shape=(1,) -- array([-0.20477049], dtype=float32)
Cast(X, to=1) -> r0
  MatMul(r0, I0_coefficient) -> r1
    Add(r1, I1_intercepts) -> r2
      Relu(r2) -> r3
        MatMul(r3, I2_coefficient1) -> r4
          Add(r4, I3_intercepts1) -> r5
            Relu(r5) -> r6
              MatMul(r6, I4_coefficient2) -> r7
                Add(r7, I5_intercepts2) -> add_result2
output: name='add_result2' type=dtype('float32') shape=['unk__0', 1]

We select the log loss (see NegLogLearningLoss, a simple regularization defined with ElasticLearningPenalty, and the Nesterov algorithm to update the weights with LearningRateSGDNesterov <onnxcustom.training.sgd_learning_rate.LearningRateSGDNesterov>.

train_session = OrtGradientForwardBackwardOptimizer(
    onx, device='cpu', warm_start=False,
    max_iter=max_iter, batch_size=batch_size,
    learning_loss=NegLogLearningLoss(),
    learning_rate=LearningRateSGDNesterov(
        1e-7, nesterov=True, momentum=0.9),
    learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4))


benches = [benchmark(X_train, y_train, nn, train_session, name='NN-CPU')]
[benchmark] NN-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:702: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  warnings.warn(
[benchmark] skl=100 iterations - 8.416117368993582 seconds
[benchmark] ort=100 iteration - 13.928476610017242 seconds

Profiling#

def clean_name(text):
    pos = text.find('onnxruntime')
    if pos >= 0:
        return text[pos:]
    pos = text.find('sklearn')
    if pos >= 0:
        return text[pos:]
    pos = text.find('onnxcustom')
    if pos >= 0:
        return text[pos:]
    pos = text.find('site-packages')
    if pos >= 0:
        return text[pos:]
    return text


ps = profile(lambda: benchmark(X_train, y_train,
             nn, train_session, name='NN-CPU'))[0]
root, nodes = profile2graph(ps, clean_text=clean_name)
text = root.to_text()
print(text)
[benchmark] NN-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:702: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  warnings.warn(
[benchmark] skl=100 iterations - 9.379087769979378 seconds
[benchmark] ort=100 iteration - 16.46740368101746 seconds
filter                                                       --      18      18 -- 0.00006 0.00015 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter)
    filter                                                   --      12      12 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:351:filter (filter)
    filter                                                   --       6       6 -- 0.00004 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:484:filter (filter)
        <built-in method builtins.isinstance>                --      18      18 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.hasattr>                       --      18      18 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
acquire                                                      --      30      30 -- 0.00005 0.00009 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire)
    <method 'acquire' of '_thread.RLock' objects>            --      30      30 -- 0.00004 0.00004 -- ~:0:<method 'acquire' of '_thread.RLock' objects> (<method 'acquire' of '_thread.RLock' objects>)
release                                                      --      30      30 -- 0.00005 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release)
    <method 'release' of '_thread.RLock' objects>            --      30      30 -- 0.00001 0.00001 -- ~:0:<method 'release' of '_thread.RLock' objects> (<method 'release' of '_thread.RLock' objects>)
emit                                                         --      12      12 -- 0.00008 0.00090 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit)
    format                                                   --      12      12 -- 0.00003 0.00046 -- /usr/local/lib/python3.9/logging/__init__.py:912:format (format)
        format                                               --      12      12 -- 0.00008 0.00043 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:531:format (format)
            format                                           --      12      12 -- 0.00006 0.00032 -- /usr/local/lib/python3.9/logging/__init__.py:646:format (format)
                usesTime                                     --      12      12 -- 0.00002 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:624:usesTime (usesTime)
                    usesTime                                 --      12      12 -- 0.00003 0.00005 -- /usr/local/lib/python3.9/logging/__init__.py:417:usesTime (usesTime)
                        <method 'find' of 'str' objects>     --      12      12 -- 0.00002 0.00002 -- ~:0:<method 'find' of 'str' objects> (<method 'find' of 'str' objects>)
                formatMessage                                --      12      12 -- 0.00002 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:630:formatMessage (formatMessage)
                    format                                   --      12      12 -- 0.00002 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:428:format (format)
                        _format                              --      12      12 -- 0.00004 0.00004 -- /usr/local/lib/python3.9/logging/__init__.py:425:_format (_format)
                getMessage                                   --      12      12 -- 0.00005 0.00011 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:89:getMessage (getMessage)
                    getMessage                               --      12      12 -- 0.00005 0.00005 -- /usr/local/lib/python3.9/logging/__init__.py:354:getMessage (getMessage)
                    <built-in method builtins.getattr>       --      12      12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
            colorize                                         --       2       2 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/console.py:84:colorize (colorize)
                escseq                                       --       4       4 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/console.py:85:escseq (escseq)
            <built-in method builtins.getattr>               --      12      12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
    flush                                                    --      12      12 -- 0.00006 0.00031 -- /usr/local/lib/python3.9/logging/__init__.py:1056:flush (flush)
        acquire                                              --      12      12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
        release                                              --      12      12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
        flush                                                --       6       6 -- 0.00002 0.00019 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:557:flush (flush)
            <method 'flush' of '_io.TextIOWrapper' objects>  --       6       6 -- 0.00016 0.00016 -- ~:0:<method 'flush' of '_io.TextIOWrapper' objects> (<method 'flush' of '_io.TextIOWrapper' objects>)
        <built-in method builtins.hasattr>                   --      12      12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    write                                                    --       6       6 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:549:write (write)
    write                                                    --       6       6 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:567:write (write)
isEnabledFor                                                 --      12      12 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor)
inner                                                        --       6       6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/typing.py:256:inner (inner)
cast                                                         --       6       6 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/typing.py:1326:cast (cast)
simplefilter                                                 --      11      11 -- 0.00004 0.00020 -- /usr/local/lib/python3.9/warnings.py:165:simplefilter (simplefilter)
    _add_filter                                              --      11      11 -- 0.00008 0.00015 -- /usr/local/lib/python3.9/warnings.py:181:_add_filter (_add_filter)
        <method 'insert' of 'list' objects>                  --      11      11 -- 0.00001 0.00001 -- ~:0:<method 'insert' of 'list' objects> (<method 'insert' of 'list' objects>)
        <method 'remove' of 'list' objects>                  --      11      11 -- 0.00005 0.00005 -- ~:0:<method 'remove' of 'list' objects> (<method 'remove' of 'list' objects>)
        <built-in method _warnings._filters_mutated>         --      11      11 -- 0.00000 0.00000 -- ~:0:<built-in method _warnings._filters_mutated> (<built-in method _warnings._filters_mutated>) +++
    <built-in method builtins.isinstance>                    --      11      11 -- 0.00000 0.00000 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
__init__                                                     --       2       2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__)
__init__                                                     --      11      11 -- 0.00003 0.00003 -- /usr/local/lib/python3.9/warnings.py:437:__init__ (__init__)
__enter__                                                    --      11      11 -- 0.00007 0.00008 -- /usr/local/lib/python3.9/warnings.py:458:__enter__ (__enter__)
    <built-in method _warnings._filters_mutated>             --      11      11 -- 0.00001 0.00001 -- ~:0:<built-in method _warnings._filters_mutated> (<built-in method _warnings._filters_mutated>) +++
__exit__                                                     --      11      11 -- 0.00004 0.00005 -- /usr/local/lib/python3.9/warnings.py:477:__exit__ (__exit__)
    <built-in method _warnings._filters_mutated>             --      11      11 -- 0.00000 0.00000 -- ~:0:<built-in method _warnings._filters_mutated> (<built-in method _warnings._filters_mutated>) +++
any                                                          --       2       2 -- 0.00002 0.00014 -- <__array_function__ internals>:177:any (any)
    _any_dispatcher                                          --       2       2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2302:_any_dispatcher (_any_dispatcher)
    <built-in method numpy.core....implement_array_function> --       2       2 -- 0.00002 0.00012 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
clip                                                         --    5001    5001 -- 0.02183 0.65427 -- <__array_function__ internals>:177:clip (clip)
    _clip_dispatcher                                         --    5001    5001 -- 0.00389 0.00389 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2081:_clip_dispatcher (_clip_dispatcher)
    <built-in method numpy.core....implement_array_function> --    5001    5001 -- 0.06252 0.62855 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
empty_like                                                   --       7       7 -- 0.00003 0.00008 -- <__array_function__ internals>:177:empty_like (empty_like)
    empty_like                                               --       7       7 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:80:empty_like (empty_like)
unique                                                       --     106     106 -- 0.00051 0.00929 -- <__array_function__ internals>:177:unique (unique)
    _unique_dispatcher                                       --     106     106 -- 0.00008 0.00008 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:133:_unique_dispatcher (_unique_dispatcher)
    <built-in method numpy.core....implement_array_function> --     106     106 -- 0.00064 0.00871 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
<lambda>                                                     --       1       1 -- 0.00001 25.84844 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_fwbw_cls.py:166:<lambda> (<lambda>)
    benchmark                                                --       1       1 -- 0.00010 25.84844 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_fwbw_cls.py:50:benchmark (benchmark)
        fit                                                  --       1       1 -- 0.00471 16.46737 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:288:fit (fit)
            __init__                                         --       1       1 -- 0.00004 0.00007 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:31:__init__ (__init__)
                get_ort_device                               --       1       1 -- 0.00000 0.00000 -- onnxruntime_helper.py:63:get_ort_device (get_ort_device)
                numpy_to_ort_value                           --       2       2 -- 0.00000 0.00002 -- onnxruntime_helper.py:133:numpy_to_ort_value (numpy_to_ort_value) +++
            needs_grad                                       --       3       3 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:104:needs_grad (needs_grad)
                needs_grad                                   --       3       3 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:337:needs_grad (needs_grad)
            get_full_state                                   --     101     101 -- 0.00061 0.00250 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:155:get_full_state (get_full_state) +++
            set_state                                        --       4       4 -- 0.00022 0.00069 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:216:set_state (set_state)
                _get_att_state                               --       4       4 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:147:_get_att_state (_get_att_state) +++
                numpy_to_ort_value                           --      24      24 -- 0.00005 0.00031 -- onnxruntime_helper.py:133:numpy_to_ort_value (numpy_to_ort_value) +++
                <built-in method numpy.zeros>                --      12      12 -- 0.00005 0.00005 -- ~:0:<built-in method numpy.zeros> (<built-in method numpy.zeros>) +++
                <method 'append' of 'list' objects>          --      56      56 -- 0.00002 0.00002 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                <built-in method builtins.isinstance>        --      24      24 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
            <listcomp>                                       --       1       1 -- 0.00004 0.00209 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:339:<listcomp> (<listcomp>)
                get_initializer                              --       7       7 -- 0.00014 0.00206 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:278:get_initializer (get_initializer) +++
            <listcomp>                                       --       1       1 -- 0.00004 0.00189 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:343:<listcomp> (<listcomp>)
                get_initializer                              --       7       7 -- 0.00013 0.00185 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:278:get_initializer (get_initializer) +++
            _iteration                                       --     100     100 -- 1.40834 16.42341 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:424:_iteration (_iteration)
                iter_ortvalue                                --    5100    5100 -- 0.10212 0.57577 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:147:iter_ortvalue (iter_ortvalue)
                    _next_iter                               --    5000    5000 -- 0.03796 0.31142 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:98:_next_iter (_next_iter)
                        <method 'randint'...mState' objects> --    5000    5000 -- 0.26414 0.26414 -- ~:0:<method 'randint' of 'numpy.random.mtrand.RandomState' objects> (<method 'randint' of 'numpy.random.mtrand.RandomState' objects>)
                        <built-in method builtins.len>       --    5000    5000 -- 0.00556 0.00933 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                    numpy_to_ort_value                       --   10000   10000 -- 0.03058 0.13049 -- onnxruntime_helper.py:133:numpy_to_ort_value (numpy_to_ort_value) +++
                    <built-in method builtins.len>           --    5200    5200 -- 0.01537 0.03174 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                forward                                      --    5000    5000 -- 0.90610 1.19639 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:662:forward (forward)
                    input_to_ort                             --    5000    5000 -- 0.17499 0.24461 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:585:input_to_ort (input_to_ort) +++
                    save_for_backward                        --    5000    5000 -- 0.03651 0.03651 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:637:save_for_backward (save_for_backward)
                    <method 'append' of 'list' objects>      --    5000    5000 -- 0.00917 0.00917 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                backward                                     --    5000    5000 -- 1.26553 1.42713 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:740:backward (backward)
                    input_to_ort                             --    5000    5000 -- 0.11084 0.14589 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:585:input_to_ort (input_to_ort) +++
                    saved_tensors                            --    5000    5000 -- 0.00635 0.00635 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:649:saved_tensors (saved_tensors)
                    <method 'pop' of 'list' objects>         --    5000    5000 -- 0.00935 0.00935 -- ~:0:<method 'pop' of 'list' objects> (<method 'pop' of 'list' objects>)
                loss_gradient                                --    5000    5000 -- 1.35168 2.20651 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_loss.py:61:loss_gradient (loss_gradient)
                    clear_binding_inputs                     --    5000    5000 -- 0.01845 0.04096 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:145:clear_binding_inputs (clear_binding_inputs)
                        _cache_in_clear                      --    5000    5000 -- 0.01727 0.02251 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:134:_cache_in_clear (_cache_in_clear)
                            <built-in method builtins.id>    --    5000    5000 -- 0.00524 0.00524 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>) +++
                    _bind_input_ortvalue                     --   10000   10000 -- 0.05538 0.25027 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:177:_bind_input_ortvalue (_bind_input_ortvalue) +++
                    _call_iobinding                          --    5000    5000 -- 0.55478 0.55478 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_loss.py:58:_call_iobinding (_call_iobinding)
                    <built-in method builtins.hasattr>       --   10000   10000 -- 0.00883 0.00883 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                penalty_loss                                 --    5000    5000 -- 0.25314 1.60548 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:168:penalty_loss (penalty_loss)
                    _bind_input_ortvalue                     --   35000   35000 -- 0.13216 0.43183 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:177:_bind_input_ortvalue (_bind_input_ortvalue) +++
                    _bind_output_ortvalue                    --    5000    5000 -- 0.03020 0.11428 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:221:_bind_output_ortvalue (_bind_output_ortvalue) +++
                    _call_iobinding                          --    5000    5000 -- 0.79483 0.79483 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:27:_call_iobinding (_call_iobinding) +++
                    <built-in method builtins.hasattr>       --   10000   10000 -- 0.00824 0.00824 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                    <built-in method builtins.len>           --   10000   10000 -- 0.00315 0.00315 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                update_weights                               --   30000   30000 -- 0.65534 2.54131 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:197:update_weights (update_weights)
                    _bind_input_ortvalue                     --   30000   30000 -- 0.11362 0.36654 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:177:_bind_input_ortvalue (_bind_input_ortvalue) +++
                    _bind_output_ortvalue                    --   30000   30000 -- 0.10566 0.32269 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:221:_bind_output_ortvalue (_bind_output_ortvalue) +++
                    _call_iobinding                          --   30000   30000 -- 1.16636 1.16636 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:27:_call_iobinding (_call_iobinding) +++
                    <built-in method builtins.hasattr>       --   60000   60000 -- 0.03038 0.03038 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                update_weights                               --   30000   30000 -- 1.23019 5.40414 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:392:update_weights (update_weights)
                    _bind_input_ortvalue                     --  150000  150000 -- 0.51547 1.59463 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:177:_bind_input_ortvalue (_bind_input_ortvalue) +++
                    _bind_output_ortvalue                    --   60000   60000 -- 0.19752 0.59880 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:221:_bind_output_ortvalue (_bind_output_ortvalue) +++
                    _call_iobinding                          --   30000   30000 -- 1.29222 1.29222 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:33:_call_iobinding (_call_iobinding)
                    value                                    --   30000   30000 -- 0.02779 0.02779 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:186:value (value) +++
                    <built-in method on...tvalue_from_numpy> --   60000   60000 -- 0.58410 0.58410 -- ~:0:<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy> (<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy>) +++
                    <built-in method builtins.hasattr>       --   60000   60000 -- 0.07642 0.07642 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                <method 'mean' of 'numpy.ndarray' objects>   --     100     100 -- 0.00057 0.01122 -- ~:0:<method 'mean' of 'numpy.ndarray' objects> (<method 'mean' of 'numpy.ndarray' objects>)
                    _mean                                    --     100     100 -- 0.00411 0.01064 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:163:_mean (_mean) +++
                <built-in method numpy.array>                --     100     100 -- 0.00620 0.00620 -- ~:0:<built-in method numpy.array> (<built-in method numpy.array>) +++
                <method 'append' of 'list' objects>          --    5000    5000 -- 0.00505 0.00505 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                <built-in method builtins.len>               --   30100   30100 -- 0.03588 0.03588 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
            _create_training_session                         --       1       1 -- 0.00003 0.02782 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:659:_create_training_session (_create_training_session)
                __init__                                     --       1       1 -- 0.00014 0.02776 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:59:__init__ (__init__)
                    <listcomp>                               --       1       1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:96:<listcomp> (<listcomp>)
                    <listcomp>                               --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:99:<listcomp> (<listcomp>)
                    <listcomp>                               --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:117:<listcomp> (<listcomp>)
                    _init_next                               --       1       1 -- 0.00016 0.02757 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:169:_init_next (_init_next)
                        <listcomp>                           --       1       1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:179:<listcomp> (<listcomp>)
                        <listcomp>                           --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:181:<listcomp> (<listcomp>)
                        <listcomp>                           --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:184:<listcomp> (<listcomp>)
                        _create_onnx_graphs                  --       1       1 -- 0.00596 0.02738 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:299:_create_onnx_graphs (_create_onnx_graphs)
                            <listcomp>                       --       1       1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:411:<listcomp> (<listcomp>)
                            <listcomp>                       --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:412:<listcomp> (<listcomp>)
                            <listcomp>                       --       1       1 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:414:<listcomp> (<listcomp>)
                                _provider_nam..._device_type --       1       1 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:269:_provider_name_to_device_type (_provider_name_to_device_type) +++
                            <listcomp>                       --       1       1 -- 0.00006 0.00008 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:419:<listcomp> (<listcomp>)
                                _provider_nam..._device_type --       7       7 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:269:_provider_name_to_device_type (_provider_name_to_device_type) +++
                            <listcomp>                       --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:425:<listcomp> (<listcomp>)
                                _provider_nam..._device_type --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:269:_provider_name_to_device_type (_provider_name_to_device_type) +++
                            <listcomp>                       --       1       1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:504:<listcomp> (<listcomp>)
                            <listcomp>                       --       1       1 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:505:<listcomp> (<listcomp>)
                            load_model                       --       2       2 -- 0.00001 0.00081 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:102:load_model (load_model)
                                _load_bytes                  --       2       2 -- 0.00004 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:25:_load_bytes (_load_bytes)
                                    inner                    --       4       4 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/typing.py:256:inner (inner) +++
                                    cast                     --       4       4 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/typing.py:1326:cast (cast) +++
                                _get_file_path               --       2       2 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:45:_get_file_path (_get_file_path)
                                load_model_from_string       --       2       2 -- 0.00002 0.00073 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:142:load_model_from_string (load_model_from_string)
                                    _deserialize             --       2       2 -- 0.00003 0.00071 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/__init__.py:76:_deserialize (_deserialize)
                                        inner                --       2       2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/typing.py:256:inner (inner) +++
                                        cast                 --       2       2 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/typing.py:1326:cast (cast) +++
                                        <method '...objects> --       2       2 -- 0.00066 0.00066 -- ~:0:<method 'ParseFromString' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'ParseFromString' of 'google.protobuf.pyext._message.CMessage' objects>)
                            get_inputs                       --       1       1 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:117:get_inputs (get_inputs)
                            get_outputs                      --       1       1 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:121:get_outputs (get_outputs)
                            __init__                         --       2       2 -- 0.00007 0.02018 -- onnxruntime/capi/onnxruntime_inference_collection.py:295:__init__ (__init__)
                                get                          --       2       2 -- 0.00001 0.00007 -- /usr/local/lib/python3.9/_collections_abc.py:675:get (get)
                                    __getitem__              --       2       2 -- 0.00003 0.00006 -- /usr/local/lib/python3.9/os.py:674:__getitem__ (__getitem__)
                                        encode               --       2       2 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/os.py:754:encode (encode)
                                __init__                     --       2       2 -- 0.00001 0.00001 -- onnxruntime/capi/onnxruntime_inference_collection.py:107:__init__ (__init__)
                                _create_inference_session    --       2       2 -- 0.01984 0.02002 -- onnxruntime/capi/onnxruntime_inference_collection.py:358:_create_inference_session (_create_inference_session)
                                    check_and_n...vider_args --       2       2 -- 0.00008 0.00015 -- onnxruntime/capi/onnxruntime_inference_collection.py:24:check_and_normalize_provider_args (check_and_normalize_provider_args)
                                        set_provider_options --       2       2 -- 0.00002 0.00002 -- onnxruntime/capi/onnxruntime_inference_collection.py:52:set_provider_options (set_provider_options)
                                            <dictcomp>       --       2       2 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:63:<dictcomp> (<dictcomp>)
                                        <listcomp>           --       2       2 -- 0.00001 0.00001 -- onnxruntime/capi/onnxruntime_inference_collection.py:76:<listcomp> (<listcomp>)
                                        <listcomp>           --       2       2 -- 0.00000 0.00000 -- onnxruntime/capi/onnxruntime_inference_collection.py:79:<listcomp> (<listcomp>)
                            <method 'Serial...sage' objects> --       1       1 -- 0.00020 0.00020 -- ~:0:<method 'SerializeToString' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'SerializeToString' of 'google.protobuf.pyext._message.CMessage' objects>)
                            <built-in method builtins.len>   --      16      16 -- 0.00000 0.00000 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                new_instance                                 --       1       1 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:217:new_instance (new_instance)
                    __init__                                 --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:529:__init__ (__init__)
                device_to_providers                          --       1       1 -- 0.00002 0.00003 -- onnxruntime_helper.py:149:device_to_providers (device_to_providers)
            value                                            --     100     100 -- 0.00008 0.00008 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:186:value (value) +++
            init_learning_rate                               --       1       1 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:348:init_learning_rate (init_learning_rate)
                init_learning_rate                           --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:205:init_learning_rate (init_learning_rate)
            update_learning_rate                             --     100     100 -- 0.00033 0.00333 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:358:update_learning_rate (update_learning_rate)
                update_learning_rate                         --     100     100 -- 0.00301 0.00301 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:226:update_learning_rate (update_learning_rate)
            proto_type_to_dtype                              --       6       6 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/utils/onnx_helper.py:61:proto_type_to_dtype (proto_type_to_dtype)
            <method 'randn' of 'num....RandomState' objects> --       6       6 -- 0.00055 0.00055 -- ~:0:<method 'randn' of 'numpy.random.mtrand.RandomState' objects> (<method 'randn' of 'numpy.random.mtrand.RandomState' objects>)
            <method 'append' of 'list' objects>              --     107     107 -- 0.00007 0.00007 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
            <built-in method builtins.len>                   --     108     108 -- 0.00005 0.00005 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
        fit                                                  --       1       1 -- 0.00002 9.37908 -- sklearn/neural_network/_multilayer_perceptron.py:745:fit (fit)
            _fit                                             --       1       1 -- 0.00013 9.37906 -- sklearn/neural_network/_multilayer_perceptron.py:377:_fit (_fit)
                any                                          --       1       1 -- 0.00001 0.00009 -- <__array_function__ internals>:177:any (any) +++
                _initialize                                  --       1       1 -- 0.00005 0.00047 -- sklearn/neural_network/_multilayer_perceptron.py:320:_initialize (_initialize)
                    is_classifier                            --       1       1 -- 0.00000 0.00000 -- sklearn/base.py:1001:is_classifier (is_classifier)
                    _init_coef                               --       3       3 -- 0.00008 0.00042 -- sklearn/neural_network/_multilayer_perceptron.py:360:_init_coef (_init_coef)
                        <method 'uniform'...mState' objects> --       6       6 -- 0.00029 0.00029 -- ~:0:<method 'uniform' of 'numpy.random.mtrand.RandomState' objects> (<method 'uniform' of 'numpy.random.mtrand.RandomState' objects>)
                <listcomp>                                   --       1       1 -- 0.00001 0.00002 -- sklearn/neural_network/_multilayer_perceptron.py:417:<listcomp> (<listcomp>)
                <listcomp>                                   --       1       1 -- 0.00001 0.00001 -- sklearn/neural_network/_multilayer_perceptron.py:422:<listcomp> (<listcomp>)
                _validate_hyperparameters                    --       1       1 -- 0.00002 0.00002 -- sklearn/neural_network/_multilayer_perceptron.py:455:_validate_hyperparameters (_validate_hyperparameters)
                _fit_stochastic                              --       1       1 -- 0.22324 9.37471 -- sklearn/neural_network/_multilayer_perceptron.py:563:_fit_stochastic (_fit_stochastic)
                    clip                                     --       1       1 -- 0.00000 0.00022 -- <__array_function__ internals>:177:clip (clip) +++
                    _backprop                                --    5000    5000 -- 0.47292 6.36596 -- sklearn/neural_network/_multilayer_perceptron.py:241:_backprop (_backprop)
                        dot                                  --   15000   15000 -- 0.04981 0.21516 -- <__array_function__ internals>:177:dot (dot)
                            dot                              --   15000   15000 -- 0.00879 0.00879 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:736:dot (dot)
                            <built-in metho...rray_function> --   15000   15000 -- 0.15656 0.15656 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                        inplace_relu_derivative              --   10000   10000 -- 0.31528 0.31528 -- sklearn/neural_network/_base.py:132:inplace_relu_derivative (inplace_relu_derivative)
                        binary_log_loss                      --    5000    5000 -- 0.52977 1.45063 -- sklearn/neural_network/_base.py:205:binary_log_loss (binary_log_loss)
                            clip                             --    5000    5000 -- 0.02183 0.65406 -- <__array_function__ internals>:177:clip (clip) +++
                            __new__                          --    5000    5000 -- 0.03364 0.03983 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/getlimits.py:458:__new__ (__new__)
                                <method 'get'...ct' objects> --    5000    5000 -- 0.00619 0.00619 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>) +++
                            <method 'sum' o...rray' objects> --   10000   10000 -- 0.03379 0.22697 -- ~:0:<method 'sum' of 'numpy.ndarray' objects> (<method 'sum' of 'numpy.ndarray' objects>)
                                _sum                         --   10000   10000 -- 0.01625 0.19318 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:46:_sum (_sum)
                                    <method 're...' objects> --   10000   10000 -- 0.17693 0.17693 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>) +++
                        _forward_pass                        --    5000    5000 -- 0.36577 1.10657 -- sklearn/neural_network/_multilayer_perceptron.py:119:_forward_pass (_forward_pass)
                            inplace_logistic                 --    5000    5000 -- 0.04760 0.04760 -- sklearn/neural_network/_base.py:25:inplace_logistic (inplace_logistic)
                            inplace_relu                     --   10000   10000 -- 0.19005 0.19005 -- sklearn/neural_network/_base.py:47:inplace_relu (inplace_relu)
                            safe_sparse_dot                  --   15000   15000 -- 0.47164 0.50315 -- sklearn/utils/extmath.py:119:safe_sparse_dot (safe_sparse_dot) +++
                        _compute_loss_grad                   --   15000   15000 -- 0.86370 2.54153 -- sklearn/neural_network/_multilayer_perceptron.py:177:_compute_loss_grad (_compute_loss_grad)
                            mean                             --   15000   15000 -- 0.05330 1.14265 -- <__array_function__ internals>:177:mean (mean)
                                _mean_dispatcher             --   15000   15000 -- 0.01043 0.01043 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3308:_mean_dispatcher (_mean_dispatcher)
                                <built-in met...ay_function> --   15000   15000 -- 0.05204 1.07893 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                            safe_sparse_dot                  --   15000   15000 -- 0.50296 0.53518 -- sklearn/utils/extmath.py:119:safe_sparse_dot (safe_sparse_dot) +++
                        safe_sparse_dot                      --   10000   10000 -- 0.21768 0.23865 -- sklearn/utils/extmath.py:119:safe_sparse_dot (safe_sparse_dot) +++
                        <method 'ravel' o...darray' objects> --   15000   15000 -- 0.02522 0.02522 -- ~:0:<method 'ravel' of 'numpy.ndarray' objects> (<method 'ravel' of 'numpy.ndarray' objects>) +++
                    _update_no_improvement_count             --     100     100 -- 0.00057 0.00057 -- sklearn/neural_network/_multilayer_perceptron.py:716:_update_no_improvement_count (_update_no_improvement_count)
                    update_params                            --    5000    5000 -- 0.26343 2.44536 -- sklearn/neural_network/_stochastic_optimizers.py:29:update_params (update_params)
                        <genexpr>                            --   35000   35000 -- 0.02200 0.02200 -- sklearn/neural_network/_stochastic_optimizers.py:43:<genexpr> (<genexpr>)
                        _get_updates                         --    5000    5000 -- 0.08014 2.15993 -- sklearn/neural_network/_stochastic_optimizers.py:169:_get_updates (_get_updates)
                            <listcomp>                       --    5000    5000 -- 1.05028 1.05028 -- sklearn/neural_network/_stochastic_optimizers.py:183:<listcomp> (<listcomp>)
                            <listcomp>                       --    5000    5000 -- 1.02951 1.02951 -- sklearn/neural_network/_stochastic_optimizers.py:190:<listcomp> (<listcomp>)
                    __init__                                 --       1       1 -- 0.00001 0.00029 -- sklearn/neural_network/_stochastic_optimizers.py:121:__init__ (__init__)
                        __init__                             --       1       1 -- 0.00000 0.00000 -- sklearn/neural_network/_stochastic_optimizers.py:25:__init__ (__init__)
                        <listcomp>                           --       1       1 -- 0.00002 0.00028 -- sklearn/neural_network/_stochastic_optimizers.py:136:<listcomp> (<listcomp>)
                            zeros_like                       --       6       6 -- 0.00002 0.00025 -- <__array_function__ internals>:177:zeros_like (zeros_like)
                                _zeros_like_dispatcher       --       6       6 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numeric.py:72:_zeros_like_dispatcher (_zeros_like_dispatcher)
                                <built-in met...ay_function> --       6       6 -- 0.00002 0.00023 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
                    iteration_ends                           --     100     100 -- 0.00096 0.00096 -- sklearn/neural_network/_stochastic_optimizers.py:138:iteration_ends (iteration_ends)
                    _safe_indexing                           --    5000    5000 -- 0.04061 0.27942 -- sklearn/utils/__init__.py:290:_safe_indexing (_safe_indexing) +++
                    shuffle                                  --     100     100 -- 0.00097 0.03118 -- sklearn/utils/__init__.py:585:shuffle (shuffle)
                        resample                             --     100     100 -- 0.00258 0.03021 -- sklearn/utils/__init__.py:435:resample (resample)
                            <listcomp>                       --     100     100 -- 0.00032 0.00057 -- sklearn/utils/__init__.py:576:<listcomp> (<listcomp>)
                                isspmatrix                   --     100     100 -- 0.00014 0.00026 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
                            <listcomp>                       --     100     100 -- 0.00030 0.00471 -- sklearn/utils/__init__.py:577:<listcomp> (<listcomp>)
                                _safe_indexing               --     100     100 -- 0.00079 0.00441 -- sklearn/utils/__init__.py:290:_safe_indexing (_safe_indexing) +++
                            check_consistent_length          --     100     100 -- 0.00096 0.01368 -- sklearn/utils/validation.py:373:check_consistent_length (check_consistent_length) +++
                            check_random_state               --     100     100 -- 0.00064 0.00191 -- sklearn/utils/validation.py:1161:check_random_state (check_random_state) +++
                            <method 'shuffl...tate' objects> --     100     100 -- 0.00508 0.00508 -- ~:0:<method 'shuffle' of 'numpy.random.mtrand.RandomState' objects> (<method 'shuffle' of 'numpy.random.mtrand.RandomState' objects>)
                            <built-in method numpy.arange>   --     100     100 -- 0.00139 0.00139 -- ~:0:<built-in method numpy.arange> (<built-in method numpy.arange>) +++
                            <built-in metho...ltins.hasattr> --     100     100 -- 0.00017 0.00017 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                            <built-in method builtins.len>   --     200     200 -- 0.00011 0.00011 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                    gen_batches                              --    5100    5100 -- 0.02590 0.02647 -- sklearn/utils/__init__.py:695:gen_batches (gen_batches)
                        <built-in method ...tins.isinstance> --     100     100 -- 0.00022 0.00056 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
                    <method 'append' of 'list' objects>      --     100     100 -- 0.00010 0.00010 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                    <built-in method _warnings.warn>         --       1       1 -- 0.00006 0.00093 -- ~:0:<built-in method _warnings.warn> (<built-in method _warnings.warn>)
                        _showwarnmsg                         --       1       1 -- 0.00001 0.00086 -- /usr/local/lib/python3.9/warnings.py:96:_showwarnmsg (_showwarnmsg)
                            _showwarning                     --       1       1 -- 0.00001 0.00085 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:571:_showwarning (_showwarning)
                                formatwarning                --       1       1 -- 0.00001 0.00005 -- /usr/local/lib/python3.9/warnings.py:15:formatwarning (formatwarning)
                                    _formatwarnmsg_impl      --       1       1 -- 0.00003 0.00004 -- /usr/local/lib/python3.9/warnings.py:35:_formatwarnmsg_impl (_formatwarnmsg_impl)
                                        getline              --       1       1 -- 0.00000 0.00001 -- /usr/local/lib/python3.9/linecache.py:26:getline (getline)
                                            getlines         --       1       1 -- 0.00000 0.00001 -- /usr/local/lib/python3.9/linecache.py:36:getlines (getlines)
                                    __init__                 --       1       1 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__) +++
                                write                        --       1       1 -- 0.00002 0.00079 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:82:write (write) +++
                        __init__                             --       1       1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__) +++
                _validate_input                              --       1       1 -- 0.00004 0.00335 -- sklearn/neural_network/_multilayer_perceptron.py:1108:_validate_input (_validate_input)
                    _validate_data                           --       1       1 -- 0.00002 0.00097 -- sklearn/base.py:495:_validate_data (_validate_data)
                        _check_n_features                    --       1       1 -- 0.00000 0.00002 -- sklearn/base.py:359:_check_n_features (_check_n_features)
                            _num_features                    --       1       1 -- 0.00001 0.00002 -- sklearn/utils/validation.py:257:_num_features (_num_features)
                        _check_feature_names                 --       1       1 -- 0.00000 0.00001 -- sklearn/base.py:405:_check_feature_names (_check_feature_names)
                            _get_feature_names               --       1       1 -- 0.00000 0.00000 -- sklearn/utils/validation.py:1824:_get_feature_names (_get_feature_names)
                        check_X_y                            --       1       1 -- 0.00001 0.00092 -- sklearn/utils/validation.py:949:check_X_y (check_X_y)
                            check_consistent_length          --       1       1 -- 0.00001 0.00019 -- sklearn/utils/validation.py:373:check_consistent_length (check_consistent_length) +++
                            check_array                      --       1       1 -- 0.00006 0.00055 -- sklearn/utils/validation.py:619:check_array (check_array) +++
                            _check_y                         --       1       1 -- 0.00001 0.00017 -- sklearn/utils/validation.py:1097:_check_y (_check_y)
                                check_array                  --       1       1 -- 0.00003 0.00016 -- sklearn/utils/validation.py:619:check_array (check_array) +++
                    __init__                                 --       1       1 -- 0.00000 0.00000 -- sklearn/preprocessing/_label.py:258:__init__ (__init__)
                    fit                                      --       1       1 -- 0.00002 0.00061 -- sklearn/preprocessing/_label.py:264:fit (fit)
                        isspmatrix                           --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
                        unique_labels                        --       1       1 -- 0.00007 0.00036 -- sklearn/utils/multiclass.py:42:unique_labels (unique_labels)
                            <genexpr>                        --       2       2 -- 0.00000 0.00018 -- sklearn/utils/multiclass.py:77:<genexpr> (<genexpr>)
                                type_of_target               --       1       1 -- 0.00003 0.00018 -- sklearn/utils/multiclass.py:203:type_of_target (type_of_target) +++
                            <genexpr>                        --       2       2 -- 0.00000 0.00008 -- sklearn/utils/multiclass.py:105:<genexpr> (<genexpr>)
                                _unique_multiclass           --       1       1 -- 0.00000 0.00008 -- sklearn/utils/multiclass.py:22:_unique_multiclass (_unique_multiclass)
                                    unique                   --       1       1 -- 0.00000 0.00008 -- <__array_function__ internals>:177:unique (unique) +++
                            <genexpr>                        --       3       3 -- 0.00000 0.00001 -- sklearn/utils/multiclass.py:108:<genexpr> (<genexpr>)
                        type_of_target                       --       1       1 -- 0.00003 0.00021 -- sklearn/utils/multiclass.py:203:type_of_target (type_of_target) +++
                        _num_samples                         --       1       1 -- 0.00001 0.00003 -- sklearn/utils/validation.py:310:_num_samples (_num_samples) +++
                    transform                                --       1       1 -- 0.00002 0.00172 -- sklearn/preprocessing/_label.py:327:transform (transform)
                        label_binarize                       --       1       1 -- 0.00011 0.00147 -- sklearn/preprocessing/_label.py:416:label_binarize (label_binarize)
                            in1d                             --       1       1 -- 0.00000 0.00010 -- <__array_function__ internals>:177:in1d (in1d)
                                _in1d_dispatcher             --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:519:_in1d_dispatcher (_in1d_dispatcher)
                            hstack                           --       1       1 -- 0.00000 0.00008 -- <__array_function__ internals>:177:hstack (hstack)
                                _vhstack_dispatcher          --       1       1 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/shape_base.py:218:_vhstack_dispatcher (_vhstack_dispatcher)
                                    _arrays_for...dispatcher --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/shape_base.py:207:_arrays_for_stack_dispatcher (_arrays_for_stack_dispatcher)
                            sort                             --       1       1 -- 0.00000 0.00003 -- <__array_function__ internals>:177:sort (sort)
                                _sort_dispatcher             --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:848:_sort_dispatcher (_sort_dispatcher)
                            searchsorted                     --       1       1 -- 0.00000 0.00005 -- <__array_function__ internals>:177:searchsorted (searchsorted)
                                _searchsorted_dispatcher     --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:1315:_searchsorted_dispatcher (_searchsorted_dispatcher)
                            any                              --       1       1 -- 0.00001 0.00005 -- <__array_function__ internals>:177:any (any) +++
                            cumsum                           --       1       1 -- 0.00000 0.00006 -- <__array_function__ internals>:177:cumsum (cumsum)
                                _cumsum_dispatcher           --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2493:_cumsum_dispatcher (_cumsum_dispatcher)
                            empty_like                       --       1       1 -- 0.00000 0.00001 -- <__array_function__ internals>:177:empty_like (empty_like) +++
                            isspmatrix                       --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
                            __init__                         --       1       1 -- 0.00003 0.00056 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_compressed.py:26:__init__ (__init__)
                                isspmatrix                   --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
                                check_format                 --       1       1 -- 0.00007 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_compressed.py:136:check_format (check_format)
                                    get_shape                --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:119:get_shape (get_shape) +++
                                    prune                    --       1       1 -- 0.00002 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_compressed.py:1168:prune (prune)
                                        _prune_array         --       2       2 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/_lib/_util.py:142:_prune_array (_prune_array)
                                        get_shape            --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:119:get_shape (get_shape) +++
                                        nnz                  --       4       4 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:291:nnz (nnz)
                                            getnnz           --       4       4 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_compressed.py:108:getnnz (getnnz)
                                        _swap                --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_csr.py:231:_swap (_swap) +++
                                    _swap                    --       2       2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_csr.py:231:_swap (_swap) +++
                                    to_native                --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:90:to_native (to_native)
                                    get_index_dtype          --       1       1 -- 0.00002 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:132:get_index_dtype (get_index_dtype) +++
                                __init__                     --       1       1 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_data.py:20:__init__ (__init__)
                                    __init__                 --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:105:__init__ (__init__)
                                get_index_dtype              --       1       1 -- 0.00006 0.00023 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:132:get_index_dtype (get_index_dtype) +++
                                isshape                      --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:221:isshape (isshape)
                                check_shape                  --       1       1 -- 0.00002 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:278:check_shape (check_shape)
                                    <genexpr>                --       3       3 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:291:<genexpr> (<genexpr>)
                            toarray                          --       1       1 -- 0.00002 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_compressed.py:1048:toarray (toarray)
                                get_shape                    --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:119:get_shape (get_shape) +++
                                _process_toarray_args        --       1       1 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1287:_process_toarray_args (_process_toarray_args)
                                    get_shape                --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:119:get_shape (get_shape) +++
                                    _get_dtype               --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_data.py:23:_get_dtype (_get_dtype)
                                tocsr                        --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_csr.py:164:tocsr (tocsr)
                                _swap                        --       2       2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_csr.py:231:_swap (_swap) +++
                            type_of_target                   --       1       1 -- 0.00003 0.00019 -- sklearn/utils/multiclass.py:203:type_of_target (type_of_target) +++
                            check_array                      --       1       1 -- 0.00003 0.00013 -- sklearn/utils/validation.py:619:check_array (check_array) +++
                            column_or_1d                     --       1       1 -- 0.00001 0.00004 -- sklearn/utils/validation.py:1120:column_or_1d (column_or_1d)
                                ravel                        --       1       1 -- 0.00000 0.00002 -- <__array_function__ internals>:177:ravel (ravel)
                                    _ravel_dispatcher        --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:1751:_ravel_dispatcher (_ravel_dispatcher)
                                shape                        --       1       1 -- 0.00000 0.00001 -- <__array_function__ internals>:177:shape (shape)
                                    _shape_dispatcher        --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:1961:_shape_dispatcher (_shape_dispatcher)
                        type_of_target                       --       1       1 -- 0.00003 0.00019 -- sklearn/utils/multiclass.py:203:type_of_target (type_of_target) +++
                        check_is_fitted                      --       1       1 -- 0.00001 0.00004 -- sklearn/utils/validation.py:1276:check_is_fitted (check_is_fitted)
                            isclass                          --       1       1 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/inspect.py:73:isclass (isclass)
                            <listcomp>                       --       1       1 -- 0.00002 0.00002 -- sklearn/utils/validation.py:1340:<listcomp> (<listcomp>)
                check_random_state                           --       1       1 -- 0.00000 0.00000 -- sklearn/utils/validation.py:1161:check_random_state (check_random_state) +++
                <built-in method builtins.all>               --       1       1 -- 0.00001 0.00023 -- ~:0:<built-in method builtins.all> (<built-in method builtins.all>) +++
        <built-in method builtins.print>                     --       3       3 -- 0.00003 0.00188 -- ~:0:<built-in method builtins.print> (<built-in method builtins.print>)
            write                                            --       6       6 -- 0.00009 0.00184 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:82:write (write) +++
_bio_cache                                                   --  320000  320000 -- 0.77940 0.96223 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:156:_bio_cache (_bio_cache)
    <built-in method builtins.id>                            --  320000  320000 -- 0.18284 0.18284 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>) +++
_bio_ptr                                                     --  320000  320000 -- 1.24025 1.24025 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:173:_bio_ptr (_bio_ptr)
_bind_input_ortvalue                                         --  225000  225000 -- 0.81663 2.64327 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:177:_bind_input_ortvalue (_bind_input_ortvalue)
    _bio_cache                                               --  225000  225000 -- 0.55995 0.69075 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:156:_bio_cache (_bio_cache) +++
    _bio_do_bind_in                                          --   15613   15613 -- 0.15879 0.15879 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:169:_bio_do_bind_in (_bio_do_bind_in)
    _bio_ptr                                                 --  225000  225000 -- 0.88995 0.88995 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:173:_bio_ptr (_bio_ptr) +++
    <built-in method builtins.isinstance>                    --  225000  225000 -- 0.08715 0.08715 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_bind_output_ortvalue                                        --   95000   95000 -- 0.33337 1.03577 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:221:_bind_output_ortvalue (_bind_output_ortvalue)
    _bio_cache                                               --   95000   95000 -- 0.21945 0.27149 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:156:_bio_cache (_bio_cache) +++
    _bio_ptr                                                 --   95000   95000 -- 0.35030 0.35030 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:173:_bio_ptr (_bio_ptr) +++
    _bio_do_bind_out                                         --    5018    5018 -- 0.04511 0.04511 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/_base_onnx_function.py:217:_bio_do_bind_out (_bio_do_bind_out)
    <built-in method builtins.isinstance>                    --   95000   95000 -- 0.03550 0.03550 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_get_att_state                                               --     205     205 -- 0.00021 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:147:_get_att_state (_get_att_state)
get_full_state                                               --     101     301 -- 0.00125 0.00250 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:155:get_full_state (get_full_state)
    _get_att_state                                           --     201     201 -- 0.00021 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:147:_get_att_state (_get_att_state) +++
    <listcomp>                                               --     100     100 -- 0.00062 0.00183 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:163:<listcomp> (<listcomp>)
        get_full_state                                       --     200     200 -- 0.00063 0.00121 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/optimizers_partial.py:155:get_full_state (get_full_state) +++
    <built-in method builtins.getattr>                       --     201     201 -- 0.00009 0.00009 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
    <built-in method builtins.hasattr>                       --     201     201 -- 0.00011 0.00011 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    <built-in method builtins.isinstance>                    --     301     301 -- 0.00022 0.00022 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_provider_name_to_device_type                                --       9       9 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:269:_provider_name_to_device_type (_provider_name_to_device_type)
get_initializer                                              --      14      14 -- 0.00028 0.00391 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:278:get_initializer (get_initializer)
    to_array                                                 --      12      12 -- 0.00025 0.00363 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/numpy_helper.py:22:to_array (to_array)
        uses_external_data                                   --      12      12 -- 0.00003 0.00005 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnx/external_data_helper.py:243:uses_external_data (uses_external_data)
            <method 'HasField' of '...age.CMessage' objects> --      12      12 -- 0.00003 0.00003 -- ~:0:<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects>) +++
        <method 'HasField' of 'go...ssage.CMessage' objects> --      24      24 -- 0.00006 0.00006 -- ~:0:<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects>) +++
        <method 'astype' of 'numpy.ndarray' objects>         --      12      12 -- 0.00008 0.00008 -- ~:0:<method 'astype' of 'numpy.ndarray' objects> (<method 'astype' of 'numpy.ndarray' objects>) +++
        <method 'reshape' of 'numpy.ndarray' objects>        --      12      12 -- 0.00014 0.00014 -- ~:0:<method 'reshape' of 'numpy.ndarray' objects> (<method 'reshape' of 'numpy.ndarray' objects>) +++
        <built-in method numpy.asarray>                      --      12      12 -- 0.00304 0.00304 -- ~:0:<built-in method numpy.asarray> (<built-in method numpy.asarray>) +++
        <built-in method builtins.getattr>                   --      12      12 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
input_to_ort                                                 --   10000   10000 -- 0.28583 0.39051 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:585:input_to_ort (input_to_ort)
    <built-in method builtins.all>                           --   10000   10000 -- 0.03408 0.06881 -- ~:0:<built-in method builtins.all> (<built-in method builtins.all>) +++
    <built-in method builtins.isinstance>                    --   10000   10000 -- 0.02968 0.02968 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.len>                           --   10000   10000 -- 0.00619 0.00619 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
_call_iobinding                                              --   35000   35000 -- 1.96119 1.96119 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_penalty.py:27:_call_iobinding (_call_iobinding)
value                                                        --   30100   30100 -- 0.02787 0.02787 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/sgd_learning_rate.py:186:value (value)
_mean                                                        --   15100   15100 -- 0.42260 0.92961 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:163:_mean (_mean)
    _count_reduce_items                                      --   15100   15100 -- 0.16854 0.19850 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:66:_count_reduce_items (_count_reduce_items)
        <built-in method numpy.co...th.normalize_axis_index> --   15200   15200 -- 0.01601 0.01601 -- ~:0:<built-in method numpy.core._multiarray_umath.normalize_axis_index> (<built-in method numpy.core._multiarray_umath.normalize_axis_index>)
        <built-in method builtins.isinstance>                --   15000   15000 -- 0.01395 0.01395 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method numpy.asanyarray>                       --   15100   15100 -- 0.00882 0.00882 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>) +++
    <method 'reduce' of 'numpy.ufunc' objects>               --   15100   15100 -- 0.27209 0.27209 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>) +++
    <built-in method builtins.hasattr>                       --     100     100 -- 0.00018 0.00018 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    <built-in method builtins.isinstance>                    --   15100   15100 -- 0.00802 0.00802 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.issubclass>                    --   30200   30200 -- 0.01939 0.01939 -- ~:0:<built-in method builtins.issubclass> (<built-in method builtins.issubclass>) +++
_wrapfunc                                                    --    5003    5003 -- 0.01872 0.53602 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:51:_wrapfunc (_wrapfunc)
    _wrapit                                                  --       1       1 -- 0.00002 0.00019 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:38:_wrapit (_wrapit)
        <method 'clip' of 'numpy.ndarray' objects>           --       1       1 -- 0.00001 0.00017 -- ~:0:<method 'clip' of 'numpy.ndarray' objects> (<method 'clip' of 'numpy.ndarray' objects>) +++
    <method 'clip' of 'numpy.ndarray' objects>               --    5000    5000 -- 0.01784 0.50982 -- ~:0:<method 'clip' of 'numpy.ndarray' objects> (<method 'clip' of 'numpy.ndarray' objects>) +++
    <built-in method builtins.getattr>                       --    5003    5003 -- 0.00723 0.00723 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
_wrapreduction                                               --       3       3 -- 0.00004 0.00032 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:69:_wrapreduction (_wrapreduction)
    <dictcomp>                                               --       3       3 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:70:<dictcomp> (<dictcomp>)
    <method 'reduce' of 'numpy.ufunc' objects>               --       3       3 -- 0.00027 0.00027 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>) +++
issubdtype                                                   --       3       3 -- 0.00002 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numerictypes.py:356:issubdtype (issubdtype)
    issubclass_                                              --       6       6 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numerictypes.py:282:issubclass_ (issubclass_)
get_shape                                                    --       4       4 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:119:get_shape (get_shape)
isspmatrix                                                   --   45210   45210 -- 0.05149 0.09307 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix)
    <built-in method builtins.isinstance>                    --   45210   45210 -- 0.04158 0.04158 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_swap                                                        --       5       5 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_csr.py:231:_swap (_swap)
get_index_dtype                                              --       2       2 -- 0.00008 0.00030 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_sputils.py:132:get_index_dtype (get_index_dtype)
    can_cast                                                 --       4       4 -- 0.00001 0.00004 -- <__array_function__ internals>:177:can_cast (can_cast)
        can_cast                                             --       4       4 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:498:can_cast (can_cast)
    __init__                                                 --       4       4 -- 0.00004 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/getlimits.py:648:__init__ (__init__)
    min                                                      --       2       2 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/getlimits.py:659:min (min)
    max                                                      --       2       2 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/getlimits.py:672:max (max)
    issubdtype                                               --       2       2 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numerictypes.py:356:issubdtype (issubdtype) +++
write                                                        --       7       7 -- 0.00011 0.00264 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:82:write (write)
    verbose                                                  --       6       6 -- 0.00004 0.00251 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:128:verbose (verbose)
        log                                                  --       6       6 -- 0.00007 0.00247 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:121:log (log)
            log                                              --       6       6 -- 0.00007 0.00240 -- /usr/local/lib/python3.9/logging/__init__.py:1825:log (log)
                log                                          --       6       6 -- 0.00006 0.00224 -- /usr/local/lib/python3.9/logging/__init__.py:1485:log (log)
                    _log                                     --       6       6 -- 0.00005 0.00217 -- /usr/local/lib/python3.9/logging/__init__.py:1553:_log (_log)
                        findCaller                           --       6       6 -- 0.00008 0.00014 -- /usr/local/lib/python3.9/logging/__init__.py:1502:findCaller (findCaller)
                            <lambda>                         --       6       6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:156:<lambda> (<lambda>)
                            normcase                         --      12      12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/posixpath.py:52:normcase (normcase)
                                <built-in met...osix.fspath> --      12      12 -- 0.00001 0.00001 -- ~:0:<built-in method posix.fspath> (<built-in method posix.fspath>) +++
                            <built-in metho...ltins.hasattr> --      12      12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                        makeRecord                           --       6       6 -- 0.00005 0.00065 -- /usr/local/lib/python3.9/logging/__init__.py:1538:makeRecord (makeRecord)
                            __init__                         --       6       6 -- 0.00026 0.00060 -- /usr/local/lib/python3.9/logging/__init__.py:278:__init__ (__init__)
                                getLevelName                 --       6       6 -- 0.00004 0.00005 -- /usr/local/lib/python3.9/logging/__init__.py:119:getLevelName (getLevelName)
                                    <method 'ge...' objects> --      12      12 -- 0.00001 0.00001 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>) +++
                                current_process              --       6       6 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/multiprocessing/process.py:37:current_process (current_process)
                                name                         --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/multiprocessing/process.py:189:name (name)
                                splitext                     --       6       6 -- 0.00003 0.00008 -- /usr/local/lib/python3.9/posixpath.py:117:splitext (splitext)
                                    _splitext                --       6       6 -- 0.00003 0.00004 -- /usr/local/lib/python3.9/genericpath.py:121:_splitext (_splitext)
                                        <method '...objects> --      12      12 -- 0.00001 0.00001 -- ~:0:<method 'rfind' of 'str' objects> (<method 'rfind' of 'str' objects>) +++
                                basename                     --       6       6 -- 0.00004 0.00007 -- /usr/local/lib/python3.9/posixpath.py:140:basename (basename)
                                    _get_sep                 --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/posixpath.py:41:_get_sep (_get_sep)
                                name                         --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/threading.py:1053:name (name)
                                current_thread               --       6       6 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/threading.py:1318:current_thread (current_thread)
                        handle                               --       6       6 -- 0.00002 0.00134 -- /usr/local/lib/python3.9/logging/__init__.py:1579:handle (handle)
                            filter                           --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter) +++
                            callHandlers                     --       6       6 -- 0.00006 0.00131 -- /usr/local/lib/python3.9/logging/__init__.py:1633:callHandlers (callHandlers)
                                handle                       --      12      12 -- 0.00007 0.00125 -- /usr/local/lib/python3.9/logging/__init__.py:935:handle (handle)
                                    filter                   --      12      12 -- 0.00005 0.00014 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter) +++
                                    acquire                  --      12      12 -- 0.00003 0.00005 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
                                    release                  --      12      12 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
                                    emit                     --       6       6 -- 0.00003 0.00032 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit) +++
                                    emit                     --       6       6 -- 0.00005 0.00066 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:151:emit (emit)
                                        acquire              --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
                                        release              --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
                                        emit                 --       6       6 -- 0.00004 0.00059 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit) +++
                    isEnabledFor                             --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor) +++
                isEnabledFor                                 --       6       6 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:1834:isEnabledFor (isEnabledFor)
                    isEnabledFor                             --       6       6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor) +++
                process                                      --       6       6 -- 0.00004 0.00005 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:131:process (process)
numpy_to_ort_value                                           --   10026   10026 -- 0.03064 0.13082 -- onnxruntime_helper.py:133:numpy_to_ort_value (numpy_to_ort_value)
    <built-in method onnxruntim...state.ortvalue_from_numpy> --   10026   10026 -- 0.10018 0.10018 -- ~:0:<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy> (<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy>) +++
_safe_indexing                                               --    5100    5100 -- 0.04140 0.28383 -- sklearn/utils/__init__.py:290:_safe_indexing (_safe_indexing)
    _array_indexing                                          --    5100    5100 -- 0.13068 0.14068 -- sklearn/utils/__init__.py:179:_array_indexing (_array_indexing)
        isspmatrix                                           --    5100    5100 -- 0.00475 0.00810 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
        <built-in method builtins.isinstance>                --    5100    5100 -- 0.00190 0.00190 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    _determine_key_type                                      --    5100    5100 -- 0.07080 0.09150 -- sklearn/utils/__init__.py:220:_determine_key_type (_determine_key_type)
        <method 'keys' of 'dict' objects>                    --    5100    5100 -- 0.00489 0.00489 -- ~:0:<method 'keys' of 'dict' objects> (<method 'keys' of 'dict' objects>)
        <built-in method builtins.hasattr>                   --    5100    5100 -- 0.00297 0.00297 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
        <built-in method builtins.isinstance>                --   15300   15300 -- 0.01284 0.01284 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.hasattr>                       --   10200   10200 -- 0.01026 0.01026 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
safe_sparse_dot                                              --   40000   40000 -- 1.19227 1.27698 -- sklearn/utils/extmath.py:119:safe_sparse_dot (safe_sparse_dot)
    isspmatrix                                               --   40000   40000 -- 0.04659 0.08470 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
type_of_target                                               --       4       4 -- 0.00012 0.00077 -- sklearn/utils/multiclass.py:203:type_of_target (type_of_target)
    simplefilter                                             --       4       4 -- 0.00001 0.00006 -- /usr/local/lib/python3.9/warnings.py:165:simplefilter (simplefilter) +++
    __init__                                                 --       4       4 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:437:__init__ (__init__) +++
    __enter__                                                --       4       4 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/warnings.py:458:__enter__ (__enter__) +++
    __exit__                                                 --       4       4 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/warnings.py:477:__exit__ (__exit__) +++
    unique                                                   --       4       4 -- 0.00001 0.00032 -- <__array_function__ internals>:177:unique (unique) +++
    isspmatrix                                               --       4       4 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
    is_multilabel                                            --       4       4 -- 0.00005 0.00018 -- sklearn/utils/multiclass.py:118:is_multilabel (is_multilabel)
        simplefilter                                         --       4       4 -- 0.00001 0.00007 -- /usr/local/lib/python3.9/warnings.py:165:simplefilter (simplefilter) +++
        __init__                                             --       4       4 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:437:__init__ (__init__) +++
        __enter__                                            --       4       4 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/warnings.py:458:__enter__ (__enter__) +++
        __exit__                                             --       4       4 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/warnings.py:477:__exit__ (__exit__) +++
_num_samples                                                 --     106     106 -- 0.00217 0.00380 -- sklearn/utils/validation.py:310:_num_samples (_num_samples)
    <built-in method builtins.hasattr>                       --     318     318 -- 0.00037 0.00037 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    <built-in method builtins.isinstance>                    --     106     106 -- 0.00022 0.00122 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.len>                           --     106     106 -- 0.00004 0.00004 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
check_consistent_length                                      --     101     101 -- 0.00097 0.01388 -- sklearn/utils/validation.py:373:check_consistent_length (check_consistent_length)
    unique                                                   --     101     101 -- 0.00049 0.00890 -- <__array_function__ internals>:177:unique (unique) +++
    <listcomp>                                               --     101     101 -- 0.00028 0.00395 -- sklearn/utils/validation.py:384:<listcomp> (<listcomp>)
        _num_samples                                         --     102     102 -- 0.00211 0.00368 -- sklearn/utils/validation.py:310:_num_samples (_num_samples) +++
    <built-in method builtins.len>                           --     101     101 -- 0.00006 0.00006 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
check_array                                                  --       3       3 -- 0.00012 0.00084 -- sklearn/utils/validation.py:619:check_array (check_array)
    simplefilter                                             --       3       3 -- 0.00001 0.00007 -- /usr/local/lib/python3.9/warnings.py:165:simplefilter (simplefilter) +++
    __init__                                                 --       3       3 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:437:__init__ (__init__) +++
    __enter__                                                --       3       3 -- 0.00003 0.00003 -- /usr/local/lib/python3.9/warnings.py:458:__enter__ (__enter__) +++
    __exit__                                                 --       3       3 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:477:__exit__ (__exit__) +++
    isspmatrix                                               --       3       3 -- 0.00000 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/scipy/sparse/_base.py:1301:isspmatrix (isspmatrix) +++
    _assert_all_finite                                       --       3       3 -- 0.00011 0.00045 -- sklearn/utils/validation.py:93:_assert_all_finite (_assert_all_finite)
        get_config                                           --       3       3 -- 0.00001 0.00003 -- sklearn/_config.py:28:get_config (get_config)
            _get_threadlocal_config                          --       3       3 -- 0.00000 0.00001 -- sklearn/_config.py:20:_get_threadlocal_config (_get_threadlocal_config)
        _safe_accumulator_op                                 --       1       1 -- 0.00002 0.00029 -- sklearn/utils/extmath.py:871:_safe_accumulator_op (_safe_accumulator_op)
            sum                                              --       1       1 -- 0.00000 0.00025 -- <__array_function__ internals>:177:sum (sum)
                _sum_dispatcher                              --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2157:_sum_dispatcher (_sum_dispatcher)
                <built-in method nump...ment_array_function> --       1       1 -- 0.00000 0.00025 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
            issubdtype                                       --       1       1 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numerictypes.py:356:issubdtype (issubdtype) +++
    _num_samples                                             --       3       3 -- 0.00005 0.00010 -- sklearn/utils/validation.py:310:_num_samples (_num_samples) +++
    _ensure_no_complex_data                                  --       3       3 -- 0.00001 0.00001 -- sklearn/utils/validation.py:561:_ensure_no_complex_data (_ensure_no_complex_data)
    _check_estimator_name                                    --       3       3 -- 0.00001 0.00001 -- sklearn/utils/validation.py:571:_check_estimator_name (_check_estimator_name)
check_random_state                                           --     101     101 -- 0.00065 0.00191 -- sklearn/utils/validation.py:1161:check_random_state (check_random_state)
    <built-in method builtins.isinstance>                    --     200     200 -- 0.00049 0.00126 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
<built-in method builtins.len>                               --   60991   60991 -- 0.06649 0.08663 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>)
    __len__                                                  --   10200   10200 -- 0.02014 0.02014 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/data_loader.py:94:__len__ (__len__)
<method 'astype' of 'numpy.ndarray' objects>                 --      26      26 -- 0.00019 0.00019 -- ~:0:<method 'astype' of 'numpy.ndarray' objects> (<method 'astype' of 'numpy.ndarray' objects>)
<method 'append' of 'list' objects>                          --   10283   10283 -- 0.01442 0.01442 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>)
<built-in method builtins.hasattr>                           --  156138  156138 -- 0.13804 0.13804 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>)
<built-in method builtins.isinstance>                        --  476562  476562 -- 0.24703 0.24921 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>)
    __instancecheck__                                        --     320     320 -- 0.00039 0.00219 -- /usr/local/lib/python3.9/abc.py:96:__instancecheck__ (__instancecheck__)
        <built-in method _abc._abc_instancecheck>            --     320     320 -- 0.00130 0.00180 -- ~:0:<built-in method _abc._abc_instancecheck> (<built-in method _abc._abc_instancecheck>)
            __subclasscheck__                                --     106     106 -- 0.00013 0.00050 -- /usr/local/lib/python3.9/abc.py:100:__subclasscheck__ (__subclasscheck__)
                <built-in method _abc._abc_subclasscheck>    --     106     106 -- 0.00038 0.00038 -- ~:0:<built-in method _abc._abc_subclasscheck> (<built-in method _abc._abc_subclasscheck>)
<built-in method numpy.array>                                --     105     105 -- 0.00625 0.00625 -- ~:0:<built-in method numpy.array> (<built-in method numpy.array>)
<built-in method builtins.all>                               --   10005   10005 -- 0.03410 0.06905 -- ~:0:<built-in method builtins.all> (<built-in method builtins.all>)
    <lambda>                                                 --   40000   40000 -- 0.02570 0.03473 -- onnxcustom/onnxcustom_UT_39_std/_doc/sphinxdoc/source/onnxcustom/training/ortgradient.py:605:<lambda> (<lambda>)
        <built-in method builtins.isinstance>                --   40000   40000 -- 0.00902 0.00902 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <genexpr>                                                --       7       7 -- 0.00009 0.00023 -- sklearn/neural_network/_multilayer_perceptron.py:447:<genexpr> (<genexpr>)
        <method 'all' of 'numpy.ndarray' objects>            --       6       6 -- 0.00002 0.00013 -- ~:0:<method 'all' of 'numpy.ndarray' objects> (<method 'all' of 'numpy.ndarray' objects>)
            _all                                             --       6       6 -- 0.00001 0.00011 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:60:_all (_all)
                <method 'reduce' of 'numpy.ufunc' objects>   --       6       6 -- 0.00010 0.00010 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>) +++
<method 'reshape' of 'numpy.ndarray' objects>                --      16      16 -- 0.00016 0.00016 -- ~:0:<method 'reshape' of 'numpy.ndarray' objects> (<method 'reshape' of 'numpy.ndarray' objects>)
<built-in method numpy.zeros>                                --      20      20 -- 0.00008 0.00008 -- ~:0:<built-in method numpy.zeros> (<built-in method numpy.zeros>)
<built-in method builtins.getattr>                           --    5257    5257 -- 0.00736 0.00736 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>)
<built-in method numpy.empty>                                --     112     112 -- 0.00050 0.00050 -- ~:0:<built-in method numpy.empty> (<built-in method numpy.empty>)
<built-in method numpy.arange>                               --     101     101 -- 0.00140 0.00140 -- ~:0:<built-in method numpy.arange> (<built-in method numpy.arange>)
<built-in method numpy.core._...th.implement_array_function> --   35128   45144 -- 0.32223 1.87369 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>)
    sort                                                     --       1       1 -- 0.00001 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:852:sort (sort)
    searchsorted                                             --       1       1 -- 0.00001 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:1319:searchsorted (searchsorted)
        _wrapfunc                                            --       1       1 -- 0.00001 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:51:_wrapfunc (_wrapfunc) +++
    ravel                                                    --       1       1 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:1755:ravel (ravel)
    shape                                                    --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:1965:shape (shape)
    clip                                                     --    5001    5001 -- 0.03009 0.56603 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2085:clip (clip)
        _wrapfunc                                            --    5001    5001 -- 0.01871 0.53594 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:51:_wrapfunc (_wrapfunc) +++
    sum                                                      --       1       1 -- 0.00001 0.00024 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2162:sum (sum)
        _wrapreduction                                       --       1       1 -- 0.00001 0.00023 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:69:_wrapreduction (_wrapreduction) +++
    any                                                      --       2       2 -- 0.00001 0.00010 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2307:any (any)
        _wrapreduction                                       --       2       2 -- 0.00003 0.00009 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:69:_wrapreduction (_wrapreduction) +++
    cumsum                                                   --       1       1 -- 0.00000 0.00005 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:2497:cumsum (cumsum)
        _wrapfunc                                            --       1       1 -- 0.00001 0.00005 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:51:_wrapfunc (_wrapfunc) +++
    ndim                                                     --   10002   10002 -- 0.01171 0.01177 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3121:ndim (ndim)
    mean                                                     --   15000   15000 -- 0.10792 1.02689 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3313:mean (mean)
        _mean                                                --   15000   15000 -- 0.41849 0.91897 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:163:_mean (_mean) +++
    zeros_like                                               --       6       6 -- 0.00005 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/numeric.py:76:zeros_like (zeros_like)
        empty_like                                           --       6       6 -- 0.00003 0.00007 -- <__array_function__ internals>:177:empty_like (empty_like) +++
        copyto                                               --       6       6 -- 0.00003 0.00007 -- <__array_function__ internals>:177:copyto (copyto)
            copyto                                           --       6       6 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:1071:copyto (copyto)
    atleast_1d                                               --       1       1 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/shape_base.py:23:atleast_1d (atleast_1d)
    hstack                                                   --       1       1 -- 0.00001 0.00007 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/shape_base.py:285:hstack (hstack)
        atleast_1d                                           --       1       1 -- 0.00000 0.00004 -- <__array_function__ internals>:177:atleast_1d (atleast_1d)
            _atleast_1d_dispatcher                           --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/shape_base.py:19:_atleast_1d_dispatcher (_atleast_1d_dispatcher)
        concatenate                                          --       1       1 -- 0.00001 0.00002 -- <__array_function__ internals>:177:concatenate (concatenate)
            concatenate                                      --       1       1 -- 0.00000 0.00000 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/multiarray.py:148:concatenate (concatenate)
    unique                                                   --     106     106 -- 0.00082 0.00806 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:138:unique (unique)
        _unpack_tuple                                        --     106     106 -- 0.00025 0.00030 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:125:_unpack_tuple (_unpack_tuple)
            <built-in method builtins.len>                   --     106     106 -- 0.00005 0.00005 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
        _unique1d                                            --     106     106 -- 0.00464 0.00613 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:323:_unique1d (_unique1d)
            <method 'flatten' of 'numpy.ndarray' objects>    --     106     106 -- 0.00072 0.00072 -- ~:0:<method 'flatten' of 'numpy.ndarray' objects> (<method 'flatten' of 'numpy.ndarray' objects>)
            <method 'sort' of 'numpy.ndarray' objects>       --     106     106 -- 0.00024 0.00024 -- ~:0:<method 'sort' of 'numpy.ndarray' objects> (<method 'sort' of 'numpy.ndarray' objects>) +++
            <built-in method numpy.asanyarray>               --     106     106 -- 0.00003 0.00003 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>) +++
            <built-in method numpy.empty>                    --     106     106 -- 0.00049 0.00049 -- ~:0:<built-in method numpy.empty> (<built-in method numpy.empty>) +++
        <built-in method numpy.asanyarray>                   --     106     106 -- 0.00082 0.00082 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>) +++
    in1d                                                     --       1       1 -- 0.00008 0.00009 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/lib/arraysetops.py:523:in1d (in1d)
<built-in method onnxruntime....1_state.ortvalue_from_numpy> --   70026   70026 -- 0.68428 0.68428 -- ~:0:<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy> (<built-in method onnxruntime.capi.onnxruntime_pybind11_state.ortvalue_from_numpy>)
<built-in method builtins.id>                                --  325001  325001 -- 0.18808 0.18808 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>)
<method 'ravel' of 'numpy.ndarray' objects>                  --   15003   15003 -- 0.02523 0.02523 -- ~:0:<method 'ravel' of 'numpy.ndarray' objects> (<method 'ravel' of 'numpy.ndarray' objects>)
<method 'get' of 'dict' objects>                             --    5034    5034 -- 0.00621 0.00621 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>)
<built-in method numpy.asanyarray>                           --   15319   15319 -- 0.00968 0.00968 -- ~:0:<built-in method numpy.asanyarray> (<built-in method numpy.asanyarray>)
<method 'reduce' of 'numpy.ufunc' objects>                   --   25113   25113 -- 0.44947 0.44947 -- ~:0:<method 'reduce' of 'numpy.ufunc' objects> (<method 'reduce' of 'numpy.ufunc' objects>)
<built-in method builtins.issubclass>                        --   30209   30209 -- 0.01940 0.01940 -- ~:0:<built-in method builtins.issubclass> (<built-in method builtins.issubclass>)
<method 'HasField' of 'google...._message.CMessage' objects> --      36      36 -- 0.00009 0.00009 -- ~:0:<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects> (<method 'HasField' of 'google.protobuf.pyext._message.CMessage' objects>)
<built-in method numpy.asarray>                              --      38      38 -- 0.00311 0.00311 -- ~:0:<built-in method numpy.asarray> (<built-in method numpy.asarray>)
<method 'sort' of 'numpy.ndarray' objects>                   --     107     107 -- 0.00025 0.00025 -- ~:0:<method 'sort' of 'numpy.ndarray' objects> (<method 'sort' of 'numpy.ndarray' objects>)
<built-in method _warnings._filters_mutated>                 --      33      33 -- 0.00001 0.00001 -- ~:0:<built-in method _warnings._filters_mutated> (<built-in method _warnings._filters_mutated>)
<method 'clip' of 'numpy.ndarray' objects>                   --    5001    5001 -- 0.01785 0.50999 -- ~:0:<method 'clip' of 'numpy.ndarray' objects> (<method 'clip' of 'numpy.ndarray' objects>)
    _clip                                                    --    5001    5001 -- 0.06007 0.49214 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:126:_clip (_clip)
        _clip_dep_is_scalar_nan                              --   10002   10002 -- 0.21314 0.31021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:92:_clip_dep_is_scalar_nan (_clip_dep_is_scalar_nan)
            ndim                                             --   10002   10002 -- 0.02890 0.09708 -- <__array_function__ internals>:177:ndim (ndim)
                _ndim_dispatcher                             --   10002   10002 -- 0.00614 0.00614 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3117:_ndim_dispatcher (_ndim_dispatcher)
                <built-in method nump...ment_array_function> --   10002   10002 -- 0.05027 0.06204 -- ~:0:<built-in method numpy.core._multiarray_umath.implement_array_function> (<built-in method numpy.core._multiarray_umath.implement_array_function>) +++
        _clip_dep_is_byte_swapped                            --   10002   10002 -- 0.01978 0.02588 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:102:_clip_dep_is_byte_swapped (_clip_dep_is_byte_swapped)
            <built-in method builtins.isinstance>            --   10002   10002 -- 0.00611 0.00611 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
        _clip_dep_invoke_with_casting                        --    5001    5001 -- 0.09598 0.09598 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/numpy/core/_methods.py:107:_clip_dep_invoke_with_casting (_clip_dep_invoke_with_casting)
<built-in method posix.fspath>                               --      24      24 -- 0.00001 0.00001 -- ~:0:<built-in method posix.fspath> (<built-in method posix.fspath>)
<built-in method _thread.get_ident>                          --      12      12 -- 0.00001 0.00001 -- ~:0:<built-in method _thread.get_ident> (<built-in method _thread.get_ident>)
<method 'rfind' of 'str' objects>                            --      18      18 -- 0.00003 0.00003 -- ~:0:<method 'rfind' of 'str' objects> (<method 'rfind' of 'str' objects>)

if GPU is available#

if get_device().upper() == 'GPU':

    train_session = OrtGradientForwardBackwardOptimizer(
        onx, device='cuda', warm_start=False,
        max_iter=max_iter, batch_size=batch_size,
        learning_loss=NegLogLearningLoss(),
        learning_rate=LearningRateSGDNesterov(
            1e-7, nesterov=False, momentum=0.9),
        learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4))

    benches.append(benchmark(X_train, y_train, nn,
                   train_session, name='NN-GPU'))

A simple linear layer#

nn = MLPClassifier(hidden_layer_sizes=tuple(), max_iter=max_iter,
                   solver='sgd', learning_rate_init=1e-1, alpha=1e-4,
                   n_iter_no_change=max_iter * 3, batch_size=batch_size,
                   nesterovs_momentum=True, momentum=0.9,
                   learning_rate="invscaling", activation='identity')


with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    nn.fit(X_train, y_train)

onx = to_onnx(nn, X_train[:1].astype(numpy.float32), target_opset=15,
              options={'zipmap': False, 'nocl': True})
onx = select_model_inputs_outputs(
    onx, outputs=["add_result"], infer_shapes=True)
onx = onnx_rename_weights(onx)

try:
    print(onnx_simple_text_plot(onx))
except RuntimeError as e:
    print("You should upgrade mlprodict.")
    print(e)

train_session = OrtGradientForwardBackwardOptimizer(
    onx, device='cpu', warm_start=False,
    max_iter=max_iter, batch_size=batch_size,
    learning_loss=NegLogLearningLoss(),
    learning_rate=LearningRateSGDNesterov(
        1e-5, nesterov=True, momentum=0.9),
    learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4))


benches.append(benchmark(X_train, y_train, nn, train_session, name='LR-CPU'))

if get_device().upper() == 'GPU':

    train_session = OrtGradientForwardBackwardOptimizer(
        onx, device='cuda', warm_start=False,
        max_iter=max_iter, batch_size=batch_size,
        learning_loss=NegLogLearningLoss(),
        learning_rate=LearningRateSGDNesterov(
            1e-5, nesterov=False, momentum=0.9),
        learning_penalty=ElasticLearningPenalty(l1=0, l2=1e-4))

    benches.append(benchmark(X_train, y_train, nn,
                   train_session, name='LR-GPU'))
opset: domain='' version=13
opset: domain='ai.onnx.ml' version=1
input: name='X' type=dtype('float32') shape=[None, 100]
init: name='I0_coefficient' type=dtype('float32') shape=(100,)
init: name='I1_intercepts' type=dtype('float32') shape=(1,) -- array([0.9456602], dtype=float32)
Cast(X, to=1) -> r0
  MatMul(r0, I0_coefficient) -> r1
    Add(r1, I1_intercepts) -> add_result
output: name='add_result' type=dtype('float32') shape=['unk__0', 1]
[benchmark] LR-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sklearn/neural_network/_multilayer_perceptron.py:702: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  warnings.warn(
[benchmark] skl=100 iterations - 3.4480608190060593 seconds
[benchmark] ort=100 iteration - 7.3609729669988155 seconds

Graphs#

Dataframe first.

df = DataFrame(benches).set_index('name')
df
skl ort iter_skl iter_ort losses_skl losses_ort
name
NN-CPU 8.416117 13.928477 100 100 [0.36517389504750175, 0.16688455392452783, 0.1... [5.7631927, 5.611171, 6.007095, 5.8362517, 5.8...
LR-CPU 3.448061 7.360973 100 100 [0.293954438083113, 0.12958447142209117, 0.132... [3.5408068, 3.5908391, 3.6169176, 3.2174234, 3...


text output

print(df)
             skl  ...                                         losses_ort
name              ...
NN-CPU  8.416117  ...  [5.7631927, 5.611171, 6.007095, 5.8362517, 5.8...
LR-CPU  3.448061  ...  [3.5408068, 3.5908391, 3.6169176, 3.2174234, 3...

[2 rows x 6 columns]

Graphs.

fig, ax = plt.subplots(1, 2, figsize=(10, 4))
df[['skl', 'ort']].plot.bar(title="Processing time", ax=ax[0])
ax[0].tick_params(axis='x', rotation=30)
for bench in benches:
    ax[1].plot(bench['losses_skl'][1:], label='skl-' + bench['name'])
    ax[1].plot(bench['losses_ort'][1:], label='ort-' + bench['name'])
ax[1].set_yscale('log')
ax[1].set_title("Losses")
ax[1].legend()
Processing time, Losses
<matplotlib.legend.Legend object at 0x7f6aaf1b5d00>

The gradient update are not exactly the same. It should be improved for a fair comprison.

fig.savefig("plot_orttraining_benchmark_fwbw_cls.png")
# plt.show()

Total running time of the script: ( 1 minutes 12.916 seconds)

Gallery generated by Sphinx-Gallery