Benchmark, comparison torch - forward-backward#

The benchmark compares the processing time between pytorch and onnxruntime-training on a linear regression and a neural network. This example starts from Train a linear regression with forward backward but uses pytorch to replace the parts updating the gradients and computing the error gradient. The training algorithm becomes:

../_images/onnxfwbwtorch.png

Class TrainingAgent (from onnxruntime-training) is still used and wrapped into ORTModule. This script then follows the same instructions as Benchmark, comparison scikit-learn - forward-backward to compare pytorch only against pytorch and onnxruntime-training.

First comparison: neural network#

import time
import numpy
from pandas import DataFrame
import matplotlib.pyplot as plt
import torch
from onnxruntime import get_device
from onnxruntime.training.ortmodule import ORTModule
from pyquickhelper.pycode.profiling import profile, profile2graph
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split


X, y = make_regression(1000, n_features=100, bias=2)
X = X.astype(numpy.float32)
y = y.astype(numpy.float32)
X_train, X_test, y_train, y_test = train_test_split(X, y)

Common parameters and training algorithm#

def from_numpy(v, device=None, requires_grad=False):
    """
    Convers a numpy array into a torch array and
    sets *device* and *requires_grad*.
    """
    v = torch.from_numpy(v)
    if device is not None:
        v = v.to(device)
    v.requires_grad_(requires_grad)
    return v

Training, two functions with same code but it is easier to distinguish between in the profiling.

def train_model_torch(model, device, x, y, n_iter=100, learning_rate=1e-5,
                      profiler=None):

    def forward_torch(model, x):
        return model(x)

    model = model.to(device)
    x = from_numpy(x, requires_grad=True, device=device)
    y = from_numpy(y, requires_grad=True, device=device)

    criterion = torch.nn.MSELoss(reduction='sum')
    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
    losses = []
    for t in range(n_iter):

        def step_train_torch():
            y_pred = forward_torch(model, x)
            loss = criterion(y_pred, y)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            return loss

        loss = step_train_torch()
        losses.append(loss)
        if profiler is not None:
            profiler.step()

    return losses


def train_model_ort(model, device, x, y, n_iter=100, learning_rate=1e-5,
                    profiler=None):
    def forward_ort(model, x):
        return model(x)

    model = model.to(device)
    x = from_numpy(x, requires_grad=True, device=device)
    y = from_numpy(y, requires_grad=True, device=device)

    criterion = torch.nn.MSELoss(reduction='sum')
    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
    losses = []
    for t in range(n_iter):

        def step_train_ort():
            y_pred = forward_ort(model, x)
            loss = criterion(y_pred, y)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            return loss

        loss = step_train_ort()
        losses.append(loss)
        if profiler is not None:
            profiler.step()

    return losses

Benchmark function

def benchmark(model_torch, model_ort, device, name, verbose=True, max_iter=100):

    print("[benchmark] %s" % name)
    begin = time.perf_counter()
    losses = train_model_torch(
        model_torch, device, X_train, y_train, n_iter=200)
    duration_torch = time.perf_counter() - begin
    length_torch = len(losses)
    print("[benchmark] torch=%r iterations - %r seconds" % (
        length_torch, duration_torch))

    begin = time.perf_counter()
    losses = train_model_ort(model_ort, device, X_train,
                             y_train, n_iter=max_iter)
    duration_ort = time.perf_counter() - begin
    length_ort = len(losses)
    print("[benchmark] onxrt=%r iteration - %r seconds" % (
        length_ort, duration_ort))

    return dict(torch=duration_torch, ort=duration_ort, name=name,
                iter_torch=length_torch, iter_ort=length_ort)


class MLPNet(torch.nn.Module):
    def __init__(self, D_in, D_out):
        super(MLPNet, self).__init__()
        self.linear1 = torch.nn.Linear(D_in, 50)
        self.linear2 = torch.nn.Linear(50, 10)
        self.linear3 = torch.nn.Linear(10, D_out)

    def forward(self, x):
        o1 = torch.sigmoid(self.linear1(x))
        o2 = torch.sigmoid(self.linear2(o1))
        return self.linear3(o2)


d_in, d_out, N = X.shape[1], 1, X.shape[0]
model_torch = MLPNet(d_in, d_out)
model_ort = ORTModule(MLPNet(d_in, d_out))
max_iter = 100

device = torch.device('cpu')
benches = [benchmark(model_torch, model_ort, device, name='NN-CPU',
                     max_iter=max_iter)]

Out:

[benchmark] NN-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:529: UserWarning: Using a target size (torch.Size([750])) that is different to the input size (torch.Size([750, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
[benchmark] torch=200 iterations - 2.77465951628983 seconds
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnxruntime/training/ortmodule/_training_manager.py:216: UserWarning: Fast path enabled - skipping checks. Rebuild graph: True, Execution agent: True, Device check: True
  warnings.warn(f"Fast path enabled - skipping checks."
[benchmark] onxrt=100 iteration - 9.256914421916008 seconds

Profiling#

def clean_name(text):
    pos = text.find('onnxruntime')
    if pos >= 0:
        return text[pos:]
    pos = text.find('onnxcustom')
    if pos >= 0:
        return text[pos:]
    pos = text.find('torch')
    if pos >= 0:
        return text[pos:]
    pos = text.find('site-packages')
    if pos >= 0:
        return text[pos:]
    return text


ps = profile(lambda: benchmark(
    model_torch, model_ort, device, name='NN-CPU', max_iter=max_iter))[0]
root, nodes = profile2graph(ps, clean_text=clean_name)
text = root.to_text()
print(text)

Out:

[benchmark] NN-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:529: UserWarning: Using a target size (torch.Size([750])) that is different to the input size (torch.Size([750, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
[benchmark] torch=200 iterations - 21.010495772585273 seconds
[benchmark] onxrt=100 iteration - 9.620220424607396 seconds
__contains__                                                 --   800   800 -- 0.00309 0.00364 -- /usr/local/lib/python3.9/enum.py:748:__contains__ (__contains__)
    <built-in method builtins.isinstance>                    --   800   800 -- 0.00055 0.00055 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
filter                                                       --    18    18 -- 0.00008 0.00019 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter)
    filter                                                   --    12    12 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:350:filter (filter)
    filter                                                   --     6     6 -- 0.00005 0.00007 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:483:filter (filter)
        <built-in method builtins.isinstance>                --    18    18 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
    <built-in method builtins.hasattr>                       --    18    18 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
acquire                                                      --    30    30 -- 0.00007 0.00012 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire)
    <method 'acquire' of '_thread.RLock' objects>            --    30    30 -- 0.00005 0.00005 -- ~:0:<method 'acquire' of '_thread.RLock' objects> (<method 'acquire' of '_thread.RLock' objects>)
release                                                      --    30    30 -- 0.00006 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release)
    <method 'release' of '_thread.RLock' objects>            --    30    30 -- 0.00001 0.00001 -- ~:0:<method 'release' of '_thread.RLock' objects> (<method 'release' of '_thread.RLock' objects>)
emit                                                         --    12    12 -- 0.00010 0.00116 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit)
    format                                                   --    12    12 -- 0.00004 0.00061 -- /usr/local/lib/python3.9/logging/__init__.py:912:format (format)
        format                                               --    12    12 -- 0.00010 0.00057 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:528:format (format)
            format                                           --    12    12 -- 0.00009 0.00042 -- /usr/local/lib/python3.9/logging/__init__.py:646:format (format)
                usesTime                                     --    12    12 -- 0.00003 0.00009 -- /usr/local/lib/python3.9/logging/__init__.py:624:usesTime (usesTime)
                    usesTime                                 --    12    12 -- 0.00004 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:417:usesTime (usesTime)
                        <method 'find' of 'str' objects>     --    12    12 -- 0.00002 0.00002 -- ~:0:<method 'find' of 'str' objects> (<method 'find' of 'str' objects>)
                formatMessage                                --    12    12 -- 0.00002 0.00010 -- /usr/local/lib/python3.9/logging/__init__.py:630:formatMessage (formatMessage)
                    format                                   --    12    12 -- 0.00002 0.00007 -- /usr/local/lib/python3.9/logging/__init__.py:428:format (format)
                        _format                              --    12    12 -- 0.00005 0.00005 -- /usr/local/lib/python3.9/logging/__init__.py:425:_format (_format)
                getMessage                                   --    12    12 -- 0.00007 0.00015 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:88:getMessage (getMessage)
                    getMessage                               --    12    12 -- 0.00006 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:354:getMessage (getMessage)
                    <built-in method builtins.getattr>       --    12    12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
            colorize                                         --     2     2 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/console.py:81:colorize (colorize)
                escseq                                       --     4     4 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/console.py:82:escseq (escseq)
            <built-in method builtins.getattr>               --    12    12 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
    flush                                                    --    12    12 -- 0.00007 0.00037 -- /usr/local/lib/python3.9/logging/__init__.py:1056:flush (flush)
        acquire                                              --    12    12 -- 0.00002 0.00004 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
        release                                              --    12    12 -- 0.00003 0.00004 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
        flush                                                --     6     6 -- 0.00003 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:554:flush (flush)
            <method 'flush' of '_io.TextIOWrapper' objects>  --     6     6 -- 0.00018 0.00018 -- ~:0:<method 'flush' of '_io.TextIOWrapper' objects> (<method 'flush' of '_io.TextIOWrapper' objects>)
        <built-in method builtins.hasattr>                   --    12    12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
    write                                                    --     6     6 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:546:write (write)
    write                                                    --     6     6 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:564:write (write)
isEnabledFor                                                 --    12    12 -- 0.00003 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor)
__init__                                                     --     2     2 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__)
from_numpy                                                   --     4     4 -- 0.00005 0.00015 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:52:from_numpy (from_numpy)
<lambda>                                                     --     1     1 -- 0.00142 30.63475 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:198:<lambda> (<lambda>)
    benchmark                                                --     1     1 -- 0.00962 30.63333 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:132:benchmark (benchmark)
        train_model_torch                                    --     1     1 -- 0.00351 21.01046 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:68:train_model_torch (train_model_torch)
            from_numpy                                       --     2     2 -- 0.00003 0.00009 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:52:from_numpy (from_numpy) +++
            step_train_torch                                 --   200   200 -- 0.00852 21.00457 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:83:step_train_torch (step_train_torch)
                forward_torch                                --   200   200 -- 0.00129 3.53889 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:71:forward_torch (forward_torch)
                    _call_impl                               --   200   200 -- 0.00277 3.53759 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1104:_call_impl (_call_impl) +++
                backward                                     --   200   200 -- 0.00348 15.37920 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:307:backward (backward) +++
                _call_impl                                   --   200   200 -- 0.00306 1.85074 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1104:_call_impl (_call_impl) +++
                wrapper                                      --   200   200 -- 0.00986 0.13706 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:83:wrapper (wrapper) +++
                zero_grad                                    --   200   200 -- 0.03179 0.09017 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:189:zero_grad (zero_grad) +++
            __init__                                         --     1     1 -- 0.00001 0.00029 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:525:__init__ (__init__) +++
            to                                               --     1     1 -- 0.00005 0.00136 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:803:to (to) +++
            __init__                                         --     1     1 -- 0.00002 0.00044 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/sgd.py:88:__init__ (__init__) +++
            <method 'append' of 'list' objects>              --   200   200 -- 0.00020 0.00020 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
        train_model_ort                                      --     1     1 -- 0.00214 9.61072 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:99:train_model_ort (train_model_ort)
            from_numpy                                       --     2     2 -- 0.00002 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:52:from_numpy (from_numpy) +++
            step_train_ort                                   --   100   100 -- 0.00464 9.60641 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:113:step_train_ort (step_train_ort)
                forward_ort                                  --   100   100 -- 0.00066 0.29955 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:101:forward_ort (forward_ort)
                    _call_impl                               --   100   100 -- 0.00157 0.29889 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1104:_call_impl (_call_impl) +++
                backward                                     --   100   100 -- 0.00156 7.21782 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:307:backward (backward) +++
                _call_impl                                   --   100   100 -- 0.00192 1.97119 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1104:_call_impl (_call_impl) +++
                wrapper                                      --   100   100 -- 0.00471 0.07118 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:83:wrapper (wrapper) +++
                zero_grad                                    --   100   100 -- 0.01201 0.04202 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:189:zero_grad (zero_grad) +++
            __init__                                         --     1     1 -- 0.00001 0.00027 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:525:__init__ (__init__) +++
            to                                               --     1     1 -- 0.00002 0.00134 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:803:to (to) +++
            __init__                                         --     1     1 -- 0.00001 0.00041 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/sgd.py:88:__init__ (__init__) +++
            <method 'append' of 'list' objects>              --   100   100 -- 0.00010 0.00010 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
        <built-in method builtins.print>                     --     3     3 -- 0.00005 0.00251 -- ~:0:<built-in method builtins.print> (<built-in method builtins.print>)
            write                                            --     6     6 -- 0.00012 0.00247 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:81:write (write) +++
write                                                        --     7     7 -- 0.00014 0.00346 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:81:write (write)
    verbose                                                  --     6     6 -- 0.00006 0.00329 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:127:verbose (verbose)
        log                                                  --     6     6 -- 0.00008 0.00324 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:120:log (log)
            log                                              --     6     6 -- 0.00010 0.00315 -- /usr/local/lib/python3.9/logging/__init__.py:1825:log (log)
                log                                          --     6     6 -- 0.00008 0.00295 -- /usr/local/lib/python3.9/logging/__init__.py:1485:log (log)
                    _log                                     --     6     6 -- 0.00006 0.00286 -- /usr/local/lib/python3.9/logging/__init__.py:1553:_log (_log)
                        findCaller                           --     6     6 -- 0.00010 0.00018 -- /usr/local/lib/python3.9/logging/__init__.py:1502:findCaller (findCaller)
                            <lambda>                         --     6     6 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:156:<lambda> (<lambda>)
                            normcase                         --    12    12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/posixpath.py:52:normcase (normcase)
                                <built-in met...osix.fspath> --    12    12 -- 0.00001 0.00001 -- ~:0:<built-in method posix.fspath> (<built-in method posix.fspath>) +++
                            <built-in metho...ltins.hasattr> --    12    12 -- 0.00001 0.00001 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
                        makeRecord                           --     6     6 -- 0.00006 0.00087 -- /usr/local/lib/python3.9/logging/__init__.py:1538:makeRecord (makeRecord)
                            __init__                         --     6     6 -- 0.00034 0.00081 -- /usr/local/lib/python3.9/logging/__init__.py:278:__init__ (__init__)
                                getLevelName                 --     6     6 -- 0.00005 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:119:getLevelName (getLevelName)
                                    <method 'ge...' objects> --    12    12 -- 0.00001 0.00001 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>) +++
                                current_process              --     6     6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/multiprocessing/process.py:37:current_process (current_process)
                                name                         --     6     6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/multiprocessing/process.py:189:name (name)
                                splitext                     --     6     6 -- 0.00003 0.00010 -- /usr/local/lib/python3.9/posixpath.py:117:splitext (splitext)
                                    _splitext                --     6     6 -- 0.00005 0.00006 -- /usr/local/lib/python3.9/genericpath.py:121:_splitext (_splitext)
                                        <method '...objects> --    12    12 -- 0.00002 0.00002 -- ~:0:<method 'rfind' of 'str' objects> (<method 'rfind' of 'str' objects>) +++
                                basename                     --     6     6 -- 0.00005 0.00009 -- /usr/local/lib/python3.9/posixpath.py:140:basename (basename)
                                    _get_sep                 --     6     6 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/posixpath.py:41:_get_sep (_get_sep)
                                name                         --     6     6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/threading.py:1053:name (name)
                                current_thread               --     6     6 -- 0.00004 0.00005 -- /usr/local/lib/python3.9/threading.py:1318:current_thread (current_thread)
                        handle                               --     6     6 -- 0.00003 0.00174 -- /usr/local/lib/python3.9/logging/__init__.py:1579:handle (handle)
                            filter                           --     6     6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter) +++
                            callHandlers                     --     6     6 -- 0.00008 0.00170 -- /usr/local/lib/python3.9/logging/__init__.py:1633:callHandlers (callHandlers)
                                handle                       --    12    12 -- 0.00008 0.00162 -- /usr/local/lib/python3.9/logging/__init__.py:935:handle (handle)
                                    filter                   --    12    12 -- 0.00007 0.00018 -- /usr/local/lib/python3.9/logging/__init__.py:787:filter (filter) +++
                                    acquire                  --    12    12 -- 0.00004 0.00006 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
                                    release                  --    12    12 -- 0.00002 0.00003 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
                                    emit                     --     6     6 -- 0.00004 0.00041 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit) +++
                                    emit                     --     6     6 -- 0.00007 0.00086 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:150:emit (emit)
                                        acquire              --     6     6 -- 0.00001 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:892:acquire (acquire) +++
                                        release              --     6     6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:899:release (release) +++
                                        emit                 --     6     6 -- 0.00006 0.00075 -- /usr/local/lib/python3.9/logging/__init__.py:1067:emit (emit) +++
                    isEnabledFor                             --     6     6 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor) +++
                isEnabledFor                                 --     6     6 -- 0.00002 0.00004 -- /usr/local/lib/python3.9/logging/__init__.py:1834:isEnabledFor (isEnabledFor)
                    isEnabledFor                             --     6     6 -- 0.00002 0.00002 -- /usr/local/lib/python3.9/logging/__init__.py:1677:isEnabledFor (isEnabledFor) +++
                process                                      --     6     6 -- 0.00005 0.00006 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx/util/logging.py:130:process (process)
backward                                                     --   300   300 -- 0.00504 22.59702 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:307:backward (backward)
    backward                                                 --   300   300 -- 0.00664 22.59177 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/__init__.py:85:backward (backward)
        _make_grads                                          --   300   300 -- 0.00674 0.01805 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/__init__.py:30:_make_grads (_make_grads)
            <built-in method ones_like>                      --   300   300 -- 0.00979 0.00979 -- ~:0:<built-in method ones_like> (<built-in method ones_like>)
            <method 'numel' of 'tor...._TensorBase' objects> --   300   300 -- 0.00048 0.00048 -- ~:0:<method 'numel' of 'torch._C._TensorBase' objects> (<method 'numel' of 'torch._C._TensorBase' objects>)
            <method 'append' of 'list' objects>              --   300   300 -- 0.00043 0.00043 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
            <built-in method builtins.isinstance>            --   300   300 -- 0.00061 0.00061 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
        _tensor_or_tensors_to_tuple                          --   300   300 -- 0.00103 0.00103 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/__init__.py:77:_tensor_or_tensors_to_tuple (_tensor_or_tensors_to_tuple)
        <method 'run_backward' of..._C._EngineBase' objects> --   300   300 -- 22.07434 22.56436 -- ~:0:<method 'run_backward' of 'torch._C._EngineBase' objects> (<method 'run_backward' of 'torch._C._EngineBase' objects>)
            apply                                            --   100   100 -- 0.00324 0.49002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/function.py:243:apply (apply)
                backward                                     --   100   100 -- 0.03767 0.48678 -- onnxruntime/training/ortmodule/_training_manager.py:121:backward (backward)
                    run_backward                             --   100   100 -- 0.40808 0.40808 -- onnxruntime/training/ortmodule/_execution_agent.py:129:run_backward (run_backward)
                    is_set                                   --   100   100 -- 0.00062 0.00257 -- onnxruntime/training/ortmodule/_graph_execution_manager.py:59:is_set (is_set) +++
                    _ortvalue_to_torch_tensor                --   700   700 -- 0.01543 0.03528 -- onnxruntime/training/ortmodule/_utils.py:64:_ortvalue_to_torch_tensor (_ortvalue_to_torch_tensor) +++
                    _torch_tensor_to_dlpack                  --   100   100 -- 0.00090 0.00164 -- onnxruntime/training/ortmodule/_utils.py:74:_torch_tensor_to_dlpack (_torch_tensor_to_dlpack) +++
                    <method 'is_contigu...nsorBase' objects> --   100   100 -- 0.00051 0.00051 -- ~:0:<method 'is_contiguous' of 'torch._C._TensorBase' objects> (<method 'is_contiguous' of 'torch._C._TensorBase' objects>) +++
                    <method 'append' of 'list' objects>      --   700   700 -- 0.00061 0.00061 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                    <built-in method builtins.len>           --   200   200 -- 0.00042 0.00042 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
        <built-in method builtins.isinstance>                --   600   600 -- 0.00128 0.00128 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
        <built-in method builtins.len>                       --   300   300 -- 0.00041 0.00041 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
    <built-in method torch._C._has_torch_function_unary>     --   300   300 -- 0.00021 0.00021 -- ~:0:<built-in method torch._C._has_torch_function_unary> (<built-in method torch._C._has_torch_function_unary>) +++
__hash__                                                     --  3660  3660 -- 0.00511 0.00987 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:706:__hash__ (__hash__)
    <built-in method torch._C._has_torch_function_unary>     --  3660  3660 -- 0.00202 0.00202 -- ~:0:<built-in method torch._C._has_torch_function_unary> (<built-in method torch._C._has_torch_function_unary>) +++
    <built-in method builtins.id>                            --  3660  3660 -- 0.00275 0.00275 -- ~:0:<built-in method builtins.id> (<built-in method builtins.id>)
grad                                                         --  10848  10848 -- 0.03244 0.03886 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:1092:grad (grad)
    <built-in method torch._C._has_torch_function_unary>     --  10848  10848 -- 0.00642 0.00642 -- ~:0:<built-in method torch._C._has_torch_function_unary> (<built-in method torch._C._has_torch_function_unary>) +++
__init__                                                     --   324   324 -- 0.00499 0.00532 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:121:__init__ (__init__)
    is_scripting                                             --   324   324 -- 0.00033 0.00033 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_jit_internal.py:957:is_scripting (is_scripting)
__enter__                                                    --   324   324 -- 0.00323 0.00693 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:126:__enter__ (__enter__)
    __init__                                                 --   324   324 -- 0.00232 0.00305 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:215:__init__ (__init__) +++
    <built-in method torch._C.is_grad_enabled>               --   324   324 -- 0.00066 0.00066 -- ~:0:<built-in method torch._C.is_grad_enabled> (<built-in method torch._C.is_grad_enabled>) +++
__exit__                                                     --   324   324 -- 0.00201 0.00430 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:130:__exit__ (__exit__)
    __init__                                                 --   324   324 -- 0.00150 0.00230 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:215:__init__ (__init__) +++
__init__                                                     --   648   648 -- 0.00382 0.00534 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:215:__init__ (__init__)
    <built-in method torch._C._set_grad_enabled>             --   648   648 -- 0.00066 0.00066 -- ~:0:<built-in method torch._C._set_grad_enabled> (<built-in method torch._C._set_grad_enabled>)
    <built-in method torch._C.is_grad_enabled>               --   648   648 -- 0.00087 0.00087 -- ~:0:<built-in method torch._C.is_grad_enabled> (<built-in method torch._C.is_grad_enabled>) +++
__init__                                                     --   600   600 -- 0.00650 0.03245 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:426:__init__ (__init__)
    <built-in method zeros>                                  --   600   600 -- 0.02595 0.02595 -- ~:0:<built-in method zeros> (<built-in method zeros>)
__enter__                                                    --   600   600 -- 0.01008 0.04464 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:435:__enter__ (__enter__)
    <built-in method torch._ops...er._record_function_enter> --   600   600 -- 0.03456 0.03456 -- ~:0:<built-in method torch._ops.profiler._record_function_enter> (<built-in method torch._ops.profiler._record_function_enter>)
__exit__                                                     --   600   600 -- 0.00430 0.01401 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:439:__exit__ (__exit__)
    <built-in method torch._ops...ler._record_function_exit> --   600   600 -- 0.00971 0.00971 -- ~:0:<built-in method torch._ops.profiler._record_function_exit> (<built-in method torch._ops.profiler._record_function_exit>)
__init__                                                     --     2     2 -- 0.00001 0.00055 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:525:__init__ (__init__)
    __init__                                                 --     2     2 -- 0.00002 0.00054 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:15:__init__ (__init__)
        __init__                                             --     2     2 -- 0.00011 0.00049 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:254:__init__ (__init__)
            __setattr__                                      --    22    22 -- 0.00026 0.00035 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1188:__setattr__ (__setattr__) +++
        __setattr__                                          --     2     2 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1188:__setattr__ (__setattr__) +++
_apply                                                       --     2     9 -- 0.00080 0.00260 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:576:_apply (_apply)
    grad                                                     --    48    48 -- 0.00018 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:1092:grad (grad) +++
    __init__                                                 --    24    24 -- 0.00018 0.00019 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:121:__init__ (__init__) +++
    __enter__                                                --    24    24 -- 0.00016 0.00034 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:126:__enter__ (__enter__) +++
    __exit__                                                 --    24    24 -- 0.00012 0.00028 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:130:__exit__ (__exit__) +++
    _apply                                                   --     7     4 -- 0.00075 0.00247 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:576:_apply (_apply) +++
    compute_should_use_set_data                              --    24    24 -- 0.00013 0.00027 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:580:compute_should_use_set_data (compute_should_use_set_data)
        get_overwrite_module_params_on_conversion            --    24    24 -- 0.00001 0.00001 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/__future__.py:18:get_overwrite_module_params_on_conversion (get_overwrite_module_params_on_conversion)
        <built-in method _has_compatible_shallow_copy_type>  --    24    24 -- 0.00012 0.00012 -- ~:0:<built-in method _has_compatible_shallow_copy_type> (<built-in method _has_compatible_shallow_copy_type>)
    convert                                                  --    24    24 -- 0.00012 0.00033 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:901:convert (convert)
        <method 'to' of 'torch._C._TensorBase' objects>      --    24    24 -- 0.00015 0.00015 -- ~:0:<method 'to' of 'torch._C._TensorBase' objects> (<method 'to' of 'torch._C._TensorBase' objects>) +++
        <method 'is_floating_poin..._C._TensorBase' objects> --    24    24 -- 0.00006 0.00006 -- ~:0:<method 'is_floating_point' of 'torch._C._TensorBase' objects> (<method 'is_floating_point' of 'torch._C._TensorBase' objects>)
    children                                                 --    16    16 -- 0.00005 0.00016 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1612:children (children)
        named_children                                       --    16    16 -- 0.00009 0.00011 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1621:named_children (named_children)
    <method 'items' of 'collections.OrderedDict' objects>    --    18    18 -- 0.00001 0.00001 -- ~:0:<method 'items' of 'collections.OrderedDict' objects> (<method 'items' of 'collections.OrderedDict' objects>) +++
to                                                           --     2     2 -- 0.00006 0.00270 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:803:to (to)
    _apply                                                   --     1     1 -- 0.00003 0.00130 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:576:_apply (_apply) +++
    _apply                                                   --     1     1 -- 0.00001 0.00131 -- onnxruntime/training/ortmodule/ortmodule.py:173:_apply (_apply)
        _apply                                               --     1     1 -- 0.00001 0.00131 -- onnxruntime/training/ortmodule/_torch_module_ort.py:27:_apply (_apply)
            _apply                                           --     1     1 -- 0.00002 0.00130 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:576:_apply (_apply) +++
_call_impl                                                   --   600  1200 -- 0.01661 7.65842 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1104:_call_impl (_call_impl)
    forward                                                  --   200   200 -- 0.01481 3.53353 -- onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py:162:forward (forward)
        _call_impl                                           --   600   600 -- 0.00730 2.55249 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1104:_call_impl (_call_impl) +++
        __getattr__                                          --   600   600 -- 0.00404 0.00404 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1172:__getattr__ (__getattr__) +++
        <built-in method sigmoid>                            --   400   400 -- 0.96219 0.96219 -- ~:0:<built-in method sigmoid> (<built-in method sigmoid>)
    forward                                                  --   600   600 -- 0.00831 2.54213 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/linear.py:102:forward (forward)
        __getattr__                                          --  1200  1200 -- 0.00277 0.00277 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1172:__getattr__ (__getattr__) +++
        <built-in method torch._C._nn.linear>                --   600   600 -- 2.53105 2.53105 -- ~:0:<built-in method torch._C._nn.linear> (<built-in method torch._C._nn.linear>)
    forward                                                  --   300   300 -- 0.00327 3.81544 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:528:forward (forward)
        mse_loss                                             --   300   300 -- 0.01592 3.81217 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/functional.py:3234:mse_loss (mse_loss)
            broadcast_tensors                                --   300   300 -- 0.00617 0.02908 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/functional.py:46:broadcast_tensors (broadcast_tensors)
                __getattr__                                  --   300   300 -- 0.00129 0.00180 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_VF.py:25:__getattr__ (__getattr__)
                    <built-in method builtins.getattr>       --   300   300 -- 0.00050 0.00050 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>) +++
                <built-in method torc..._has_torch_function> --   300   300 -- 0.00035 0.00035 -- ~:0:<built-in method torch._C._has_torch_function> (<built-in method torch._C._has_torch_function>)
                <built-in method broadcast_tensors>          --   300   300 -- 0.02077 0.02077 -- ~:0:<built-in method broadcast_tensors> (<built-in method broadcast_tensors>)
            get_enum                                         --   300   300 -- 0.00075 0.00075 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/_reduction.py:7:get_enum (get_enum)
            <built-in method torch....rch_function_variadic> --   300   300 -- 0.00039 0.00039 -- ~:0:<built-in method torch._C._has_torch_function_variadic> (<built-in method torch._C._has_torch_function_variadic>)
            <method 'size' of 'torc...._TensorBase' objects> --  1200  1200 -- 0.00415 0.00415 -- ~:0:<method 'size' of 'torch._C._TensorBase' objects> (<method 'size' of 'torch._C._TensorBase' objects>)
            <built-in method torch._C._nn.mse_loss>          --   300   300 -- 3.73429 3.73429 -- ~:0:<built-in method torch._C._nn.mse_loss> (<built-in method torch._C._nn.mse_loss>)
            <method 'format' of 'str' objects>               --   300   300 -- 0.01553 0.01553 -- ~:0:<method 'format' of 'str' objects> (<method 'format' of 'str' objects>) +++
            <built-in method _warnings.warn>                 --   300   300 -- 0.01097 0.01206 -- ~:0:<built-in method _warnings.warn> (<built-in method _warnings.warn>)
                _showwarnmsg                                 --     1     1 -- 0.00002 0.00108 -- /usr/local/lib/python3.9/warnings.py:96:_showwarnmsg (_showwarnmsg)
                    _showwarning                             --     1     1 -- 0.00001 0.00106 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:473:_showwarning (_showwarning)
                        formatwarning                        --     1     1 -- 0.00001 0.00006 -- /usr/local/lib/python3.9/warnings.py:15:formatwarning (formatwarning)
                            _formatwarnmsg_impl              --     1     1 -- 0.00003 0.00005 -- /usr/local/lib/python3.9/warnings.py:35:_formatwarnmsg_impl (_formatwarnmsg_impl)
                                getline                      --     1     1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/linecache.py:26:getline (getline)
                                    getlines                 --     1     1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/linecache.py:36:getlines (getlines)
                            __init__                         --     1     1 -- 0.00000 0.00000 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__) +++
                        write                                --     1     1 -- 0.00002 0.00099 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/sphinx_gallery/gen_rst.py:81:write (write) +++
                __init__                                     --     1     1 -- 0.00001 0.00001 -- /usr/local/lib/python3.9/warnings.py:403:__init__ (__init__) +++
    _forward                                                 --   100   100 -- 0.00063 0.29666 -- onnxruntime/training/ortmodule/_utils.py:319:_forward (_forward)
        _forward                                             --   100   100 -- 0.00146 0.29603 -- onnxruntime/training/ortmodule/_utils.py:298:_forward (_forward)
            __call__                                         --   100   100 -- 0.00016 0.00016 -- onnxruntime/training/ortmodule/_graph_execution_manager_factory.py:17:__call__ (__call__)
            is_training                                      --   100   100 -- 0.00043 0.00059 -- onnxruntime/training/ortmodule/_torch_module_ort.py:43:is_training (is_training)
                <built-in method torch._C.is_grad_enabled>   --   100   100 -- 0.00017 0.00017 -- ~:0:<built-in method torch._C.is_grad_enabled> (<built-in method torch._C.is_grad_enabled>) +++
            forward                                          --   100   100 -- 0.00698 0.29381 -- onnxruntime/training/ortmodule/_training_manager.py:198:forward (forward)
                is_pending                                   --   100   100 -- 0.00016 0.00016 -- onnxruntime/training/ortmodule/_fallback.py:153:is_pending (is_pending)
                maybe_update_cache_before_run                --   100   100 -- 0.00030 0.00041 -- onnxruntime/training/ortmodule/_gradient_accumulation_manager.py:66:maybe_update_cache_before_run (maybe_update_cache_before_run)
                    enabled                                  --   100   100 -- 0.00011 0.00011 -- onnxruntime/training/ortmodule/_gradient_accumulation_manager.py:45:enabled (enabled) +++
                is_set                                       --   200   200 -- 0.00084 0.00308 -- onnxruntime/training/ortmodule/_graph_execution_manager.py:59:is_set (is_set) +++
                _combine_input_buffers_initializers          --   100   100 -- 0.00220 0.02179 -- onnxruntime/training/ortmodule/_io.py:132:_combine_input_buffers_initializers (_combine_input_buffers_initializers)
                    is_primitive_type                        --   100   100 -- 0.00031 0.00031 -- onnxruntime/training/ortmodule/_io.py:65:is_primitive_type (is_primitive_type)
                    _expand_inputs                           --   100   100 -- 0.00082 0.00528 -- onnxruntime/training/ortmodule/_io.py:140:_expand_inputs (_expand_inputs) +++
                    <dictcomp>                               --   100   100 -- 0.00025 0.01374 -- onnxruntime/training/ortmodule/_io.py:165:<dictcomp> (<dictcomp>)
                        named_buffers                        --   100   100 -- 0.00085 0.01348 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1586:named_buffers (named_buffers)
                            _named_members                   --   100   100 -- 0.00239 0.01263 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501:_named_members (_named_members) +++
                    <method 'append' of 'list' objects>      --   100   100 -- 0.00007 0.00007 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
                    <method 'extend' of 'list' objects>      --   100   100 -- 0.00019 0.00019 -- ~:0:<method 'extend' of 'list' objects> (<method 'extend' of 'list' objects>)
                unflatten_user_output                        --   100   100 -- 0.00059 0.00120 -- onnxruntime/training/ortmodule/_io.py:275:unflatten_user_output (unflatten_user_output)
                    _replace_stub_with_tensor_value          --   100   100 -- 0.00052 0.00061 -- onnxruntime/training/ortmodule/_io.py:278:_replace_stub_with_tensor_value (_replace_stub_with_tensor_value)
                        <built-in method ...tins.isinstance> --   100   100 -- 0.00009 0.00009 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
                <built-in method apply>                      --   100   100 -- 0.00932 0.26020 -- ~:0:<built-in method apply> (<built-in method apply>)
                    forward                                  --   100   100 -- 0.00488 0.25089 -- onnxruntime/training/ortmodule/_training_manager.py:76:forward (forward)
                        set_materialize_grads                --   100   100 -- 0.00029 0.00029 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/function.py:184:set_materialize_grads (set_materialize_grads)
                        is_set                               --   100   100 -- 0.00043 0.00144 -- onnxruntime/training/ortmodule/_graph_execution_manager.py:59:is_set (is_set) +++
                        execution_session_run_forward        --   100   100 -- 0.02782 0.24427 -- onnxruntime/training/ortmodule/_training_manager.py:38:execution_session_run_forward (execution_session_run_forward)
                            run_forward                      --   100   100 -- 0.19012 0.19012 -- onnxruntime/training/ortmodule/_execution_agent.py:119:run_forward (run_forward)
                            extract_outputs...e_update_cache --   100   100 -- 0.00268 0.01429 -- onnxruntime/training/ortmodule/_gradient_accumulation_manager.py:51:extract_outputs_and_maybe_update_cache (extract_outputs_and_maybe_update_cache)
                                enabled                      --   100   100 -- 0.00012 0.00012 -- onnxruntime/training/ortmodule/_gradient_accumulation_manager.py:45:enabled (enabled) +++
                                <genexpr>                    --   200   200 -- 0.00272 0.01056 -- onnxruntime/training/ortmodule/_gradient_accumulation_manager.py:58:<genexpr> (<genexpr>)
                                    _ortvalue_t...rch_tensor --   100   100 -- 0.00319 0.00784 -- onnxruntime/training/ortmodule/_utils.py:64:_ortvalue_to_torch_tensor (_ortvalue_to_torch_tensor) +++
                                <built-in met...uiltins.len> --   100   100 -- 0.00093 0.00093 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
                            __init__                         --   100   100 -- 0.00040 0.00040 -- onnxruntime/training/ortmodule/_graph_execution_manager.py:42:__init__ (__init__)
                            <listcomp>                       --   100   100 -- 0.00123 0.00123 -- onnxruntime/training/ortmodule/_training_manager.py:65:<listcomp> (<listcomp>)
                            _torch_tensor_to_dlpack          --   700   700 -- 0.00398 0.00844 -- onnxruntime/training/ortmodule/_utils.py:74:_torch_tensor_to_dlpack (_torch_tensor_to_dlpack) +++
                            <method 'is_con...Base' objects> --   700   700 -- 0.00185 0.00185 -- ~:0:<method 'is_contiguous' of 'torch._C._TensorBase' objects> (<method 'is_contiguous' of 'torch._C._TensorBase' objects>) +++
                            <built-in method builtins.len>   --   100   100 -- 0.00012 0.00012 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>) +++
    <built-in method torch._C._get_tracing_state>            --  1200  1200 -- 0.00653 0.00653 -- ~:0:<built-in method torch._C._get_tracing_state> (<built-in method torch._C._get_tracing_state>)
__getattr__                                                  --  1800  1800 -- 0.00681 0.00681 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1172:__getattr__ (__getattr__)
__setattr__                                                  --    24    24 -- 0.00028 0.00038 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1188:__setattr__ (__setattr__)
    <method 'get' of 'dict' objects>                         --    72    72 -- 0.00004 0.00004 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>) +++
    <built-in method builtins.isinstance>                    --    48    48 -- 0.00006 0.00006 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_named_members                                               --   114   114 -- 0.00251 0.01298 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501:_named_members (_named_members)
    __hash__                                                 --    12    12 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:706:__hash__ (__hash__) +++
    <lambda>                                                 --     8     8 -- 0.00002 0.00002 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1559:<lambda> (<lambda>)
    <lambda>                                                 --   500   500 -- 0.00137 0.00238 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1607:<lambda> (<lambda>)
        <method 'items' of 'colle...ns.OrderedDict' objects> --   500   500 -- 0.00101 0.00101 -- ~:0:<method 'items' of 'collections.OrderedDict' objects> (<method 'items' of 'collections.OrderedDict' objects>) +++
    named_modules                                            --   610   610 -- 0.00228 0.00797 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1668:named_modules (named_modules) +++
    <method 'add' of 'set' objects>                          --    12    12 -- 0.00003 0.00005 -- ~:0:<method 'add' of 'set' objects> (<method 'add' of 'set' objects>) +++
parameters                                                   --    14    14 -- 0.00002 0.00040 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1514:parameters (parameters)
    named_parameters                                         --    14    14 -- 0.00003 0.00038 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1538:named_parameters (named_parameters)
        _named_members                                       --    14    14 -- 0.00012 0.00035 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501:_named_members (_named_members) +++
named_modules                                                --   610  1722 -- 0.00723 0.00797 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1668:named_modules (named_modules)
    named_modules                                            --  1112   512 -- 0.00495 0.00553 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1668:named_modules (named_modules) +++
    <method 'items' of 'collections.OrderedDict' objects>    --   508   508 -- 0.00025 0.00025 -- ~:0:<method 'items' of 'collections.OrderedDict' objects> (<method 'items' of 'collections.OrderedDict' objects>) +++
    <method 'add' of 'set' objects>                          --   508   508 -- 0.00050 0.00050 -- ~:0:<method 'add' of 'set' objects> (<method 'add' of 'set' objects>) +++
wrapper                                                      --   300   300 -- 0.01457 0.20824 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:83:wrapper (wrapper)
    decorate_context                                         --   300   300 -- 0.00484 0.14451 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:24:decorate_context (decorate_context)
        clone                                                --   300   300 -- 0.00158 0.00671 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:82:clone (clone)
            __init__                                         --   300   300 -- 0.00480 0.00512 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:121:__init__ (__init__) +++
        __enter__                                            --   300   300 -- 0.00307 0.00659 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:126:__enter__ (__enter__) +++
        __exit__                                             --   300   300 -- 0.00188 0.00402 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/grad_mode.py:130:__exit__ (__exit__) +++
        step                                                 --   300   300 -- 0.03184 0.12235 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/sgd.py:109:step (step)
            __hash__                                         --  3612  3612 -- 0.00503 0.00975 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:706:__hash__ (__hash__) +++
            grad                                             --  3600  3600 -- 0.01192 0.01403 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:1092:grad (grad) +++
            sgd                                              --   300   300 -- 0.01165 0.06381 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/_functional.py:158:sgd (sgd)
                <method 'add_' of 'to...TensorBase' objects> --  1800  1800 -- 0.05217 0.05217 -- ~:0:<method 'add_' of 'torch._C._TensorBase' objects> (<method 'add_' of 'torch._C._TensorBase' objects>)
            <method 'append' of 'list' objects>              --  5400  5400 -- 0.00292 0.00292 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
    __init__                                                 --   300   300 -- 0.00299 0.01704 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:426:__init__ (__init__) +++
    __enter__                                                --   300   300 -- 0.00486 0.02267 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:435:__enter__ (__enter__) +++
    __exit__                                                 --   300   300 -- 0.00210 0.00713 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:439:__exit__ (__exit__) +++
    <method 'format' of 'str' objects>                       --   300   300 -- 0.00232 0.00232 -- ~:0:<method 'format' of 'str' objects> (<method 'format' of 'str' objects>) +++
zero_grad                                                    --   300   300 -- 0.04380 0.13219 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:189:zero_grad (zero_grad)
    grad                                                     --  7200  7200 -- 0.02034 0.02462 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:1092:grad (grad) +++
    __init__                                                 --   300   300 -- 0.00351 0.01542 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:426:__init__ (__init__) +++
    __enter__                                                --   300   300 -- 0.00521 0.02196 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:435:__enter__ (__enter__) +++
    __exit__                                                 --   300   300 -- 0.00219 0.00688 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/autograd/profiler.py:439:__exit__ (__exit__) +++
    <method 'requires_grad_' of...h._C._TensorBase' objects> --  1800  1800 -- 0.00596 0.00596 -- ~:0:<method 'requires_grad_' of 'torch._C._TensorBase' objects> (<method 'requires_grad_' of 'torch._C._TensorBase' objects>) +++
    <method 'zero_' of 'torch._C._TensorBase' objects>       --  1800  1800 -- 0.01274 0.01274 -- ~:0:<method 'zero_' of 'torch._C._TensorBase' objects> (<method 'zero_' of 'torch._C._TensorBase' objects>)
    <method 'get' of 'dict' objects>                         --   300   300 -- 0.00037 0.00037 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>) +++
    <built-in method builtins.hasattr>                       --   300   300 -- 0.00045 0.00045 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
__init__                                                     --     2     2 -- 0.00003 0.00085 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/sgd.py:88:__init__ (__init__)
    __init__                                                 --     2     2 -- 0.00008 0.00082 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:33:__init__ (__init__)
        parameters                                           --     7     7 -- 0.00001 0.00021 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1514:parameters (parameters) +++
        _hook_for_profile                                    --     2     2 -- 0.00003 0.00004 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:78:_hook_for_profile (_hook_for_profile)
        add_param_group                                      --     2     2 -- 0.00016 0.00025 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/optim/optimizer.py:243:add_param_group (add_param_group)
            __hash__                                         --    24    24 -- 0.00003 0.00005 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:706:__hash__ (__hash__) +++
            <method 'setdefault' of 'dict' objects>          --    12    12 -- 0.00001 0.00001 -- ~:0:<method 'setdefault' of 'dict' objects> (<method 'setdefault' of 'dict' objects>) +++
            <built-in method builtins.isinstance>            --    18    18 -- 0.00002 0.00002 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
        parameters                                           --     7     7 -- 0.00001 0.00021 -- onnxruntime/training/ortmodule/ortmodule.py:233:parameters (parameters)
            parameters                                       --     7     7 -- 0.00001 0.00020 -- onnxruntime/training/ortmodule/_torch_module_ort.py:91:parameters (parameters)
                parameters                                   --     7     7 -- 0.00001 0.00019 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1514:parameters (parameters) +++
enabled                                                      --   200   200 -- 0.00023 0.00023 -- onnxruntime/training/ortmodule/_gradient_accumulation_manager.py:45:enabled (enabled)
is_set                                                       --   400   400 -- 0.00188 0.00708 -- onnxruntime/training/ortmodule/_graph_execution_manager.py:59:is_set (is_set)
    __contains__                                             --   400   400 -- 0.00104 0.00121 -- /usr/local/lib/python3.9/enum.py:748:__contains__ (__contains__) +++
    is_disabled                                              --   400   400 -- 0.00156 0.00399 -- onnxruntime/training/ortmodule/_graph_execution_manager.py:67:is_disabled (is_disabled)
        __contains__                                         --   400   400 -- 0.00205 0.00243 -- /usr/local/lib/python3.9/enum.py:748:__contains__ (__contains__) +++
_expand_inputs                                               --   100   200 -- 0.00138 0.00528 -- onnxruntime/training/ortmodule/_io.py:140:_expand_inputs (_expand_inputs)
    _expand_inputs                                           --   100   100 -- 0.00056 0.00196 -- onnxruntime/training/ortmodule/_io.py:140:_expand_inputs (_expand_inputs) +++
    <method 'append' of 'list' objects>                      --   100   100 -- 0.00007 0.00007 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>) +++
    <built-in method builtins.isinstance>                    --   500   500 -- 0.00111 0.00382 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>) +++
_ortvalue_to_torch_tensor                                    --   800   800 -- 0.01862 0.04312 -- onnxruntime/training/ortmodule/_utils.py:64:_ortvalue_to_torch_tensor (_ortvalue_to_torch_tensor)
    _torch_tensor_from_dl_pack                               --   800   800 -- 0.01171 0.02450 -- onnxruntime/training/ortmodule/_utils.py:59:_torch_tensor_from_dl_pack (_torch_tensor_from_dl_pack)
        from_dlpack                                          --   800   800 -- 0.00220 0.01278 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/utils/dlpack.py:47:from_dlpack (from_dlpack)
            <built-in method torch._C._from_dlpack>          --   800   800 -- 0.00934 0.00934 -- ~:0:<built-in method torch._C._from_dlpack> (<built-in method torch._C._from_dlpack>)
            <built-in method builtins.hasattr>               --   800   800 -- 0.00124 0.00124 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>) +++
_torch_tensor_to_dlpack                                      --   800   800 -- 0.00488 0.01009 -- onnxruntime/training/ortmodule/_utils.py:74:_torch_tensor_to_dlpack (_torch_tensor_to_dlpack)
    <built-in method torch._C._to_dlpack>                    --   800   800 -- 0.00349 0.00349 -- ~:0:<built-in method torch._C._to_dlpack> (<built-in method torch._C._to_dlpack>)
    <method 'is_contiguous' of ...h._C._TensorBase' objects> --   800   800 -- 0.00171 0.00171 -- ~:0:<method 'is_contiguous' of 'torch._C._TensorBase' objects> (<method 'is_contiguous' of 'torch._C._TensorBase' objects>) +++
<built-in method builtins.len>                               --   716   716 -- 0.00188 0.00188 -- ~:0:<built-in method builtins.len> (<built-in method builtins.len>)
<method 'append' of 'list' objects>                          --  6902  6902 -- 0.00441 0.00441 -- ~:0:<method 'append' of 'list' objects> (<method 'append' of 'list' objects>)
<method 'requires_grad_' of 'torch._C._TensorBase' objects>  --  1804  1804 -- 0.00598 0.00598 -- ~:0:<method 'requires_grad_' of 'torch._C._TensorBase' objects> (<method 'requires_grad_' of 'torch._C._TensorBase' objects>)
<method 'to' of 'torch._C._TensorBase' objects>              --    28    28 -- 0.00017 0.00017 -- ~:0:<method 'to' of 'torch._C._TensorBase' objects> (<method 'to' of 'torch._C._TensorBase' objects>)
<method 'format' of 'str' objects>                           --   602   602 -- 0.01786 0.01786 -- ~:0:<method 'format' of 'str' objects> (<method 'format' of 'str' objects>)
<method 'get' of 'dict' objects>                             --   404   404 -- 0.00044 0.00044 -- ~:0:<method 'get' of 'dict' objects> (<method 'get' of 'dict' objects>)
<built-in method builtins.hasattr>                           --  1154  1154 -- 0.00175 0.00175 -- ~:0:<built-in method builtins.hasattr> (<built-in method builtins.hasattr>)
<built-in method torch._C._has_torch_function_unary>         --  14808  14808 -- 0.00864 0.00864 -- ~:0:<built-in method torch._C._has_torch_function_unary> (<built-in method torch._C._has_torch_function_unary>)
<built-in method builtins.isinstance>                        --  2418  2418 -- 0.00378 0.00654 -- ~:0:<built-in method builtins.isinstance> (<built-in method builtins.isinstance>)
    __instancecheck__                                        --   306   306 -- 0.00053 0.00275 -- /usr/local/lib/python3.9/abc.py:96:__instancecheck__ (__instancecheck__)
        <built-in method _abc._abc_instancecheck>            --   306   306 -- 0.00125 0.00222 -- ~:0:<built-in method _abc._abc_instancecheck> (<built-in method _abc._abc_instancecheck>)
            __subclasscheck__                                --   100   100 -- 0.00015 0.00098 -- /usr/local/lib/python3.9/abc.py:100:__subclasscheck__ (__subclasscheck__)
                <built-in method _abc._abc_subclasscheck>    --   100   100 -- 0.00067 0.00083 -- ~:0:<built-in method _abc._abc_subclasscheck> (<built-in method _abc._abc_subclasscheck>)
                    __subclasshook__                         --   100   100 -- 0.00017 0.00017 -- /usr/local/lib/python3.9/_collections_abc.py:311:__subclasshook__ (__subclasshook__)
<method 'items' of 'collections.OrderedDict' objects>        --  1043  1043 -- 0.00128 0.00128 -- ~:0:<method 'items' of 'collections.OrderedDict' objects> (<method 'items' of 'collections.OrderedDict' objects>)
<built-in method builtins.getattr>                           --   338   338 -- 0.00055 0.00055 -- ~:0:<built-in method builtins.getattr> (<built-in method builtins.getattr>)
<method 'setdefault' of 'dict' objects>                      --    18    18 -- 0.00002 0.00002 -- ~:0:<method 'setdefault' of 'dict' objects> (<method 'setdefault' of 'dict' objects>)
<built-in method torch._C.is_grad_enabled>                   --  1072  1072 -- 0.00169 0.00169 -- ~:0:<built-in method torch._C.is_grad_enabled> (<built-in method torch._C.is_grad_enabled>)
<method 'add' of 'set' objects>                              --   527   527 -- 0.00053 0.00056 -- ~:0:<method 'add' of 'set' objects> (<method 'add' of 'set' objects>)
    __hash__                                                 --    12    12 -- 0.00002 0.00003 -- onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/_tensor.py:706:__hash__ (__hash__) +++
<method 'is_contiguous' of 'torch._C._TensorBase' objects>   --  1600  1600 -- 0.00407 0.00407 -- ~:0:<method 'is_contiguous' of 'torch._C._TensorBase' objects> (<method 'is_contiguous' of 'torch._C._TensorBase' objects>)
<built-in method posix.fspath>                               --    24    24 -- 0.00001 0.00001 -- ~:0:<built-in method posix.fspath> (<built-in method posix.fspath>)
<built-in method _thread.get_ident>                          --    12    12 -- 0.00001 0.00001 -- ~:0:<built-in method _thread.get_ident> (<built-in method _thread.get_ident>)
<method 'rfind' of 'str' objects>                            --    18    18 -- 0.00003 0.00003 -- ~:0:<method 'rfind' of 'str' objects> (<method 'rfind' of 'str' objects>)

if GPU is available#

if get_device().upper() == 'GPU':

    device = torch.device('cuda:0')
    benches.append(benchmark(model_torch, model_ort, device, name='NN-GPU',
                             max_iter=max_iter))

Linear Regression#

class LinearRegressionNet(torch.nn.Module):
    def __init__(self, D_in, D_out):
        super(LinearRegressionNet, self).__init__()
        self.linear = torch.nn.Linear(D_in, D_out)

    def forward(self, x):
        return self.linear(x)


d_in, d_out, N = X.shape[1], 1, X.shape[0]
model_torch = LinearRegressionNet(d_in, d_out)
model_ort = ORTModule(LinearRegressionNet(d_in, d_out))

device = torch.device('cpu')
benches.append(benchmark(model_torch, model_ort, device, name='LR-CPU',
                         max_iter=max_iter))


if get_device().upper() == 'GPU':

    device = torch.device('cuda:0')
    benches.append(benchmark(model_torch, model_ort, device, name='LR-GPU',
                             max_iter=max_iter))

    ######################################
    # GPU profiling
    # +++++++++++++

    if get_device().upper() == 'GPU':
        ps = profile(lambda: benchmark(
            model_torch, model_ort, device, name='LR-GPU',
            max_iter=max_iter))[0]
        root, nodes = profile2graph(ps, clean_text=clean_name)
        text = root.to_text()
        print(text)

Out:

[benchmark] LR-CPU
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/torch/nn/modules/loss.py:529: UserWarning: Using a target size (torch.Size([750])) that is different to the input size (torch.Size([750, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
[benchmark] torch=200 iterations - 12.983033951371908 seconds
somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_venv/lib/python3.9/site-packages/onnxruntime/training/ortmodule/_training_manager.py:216: UserWarning: Fast path enabled - skipping checks. Rebuild graph: True, Execution agent: True, Device check: True
  warnings.warn(f"Fast path enabled - skipping checks."
[benchmark] onxrt=100 iteration - 9.299985624849796 seconds

Graphs#

Dataframe first.

df = DataFrame(benches).set_index('name')
df
torch ort iter_torch iter_ort
name
NN-CPU 2.774660 9.256914 200 100
LR-CPU 12.983034 9.299986 200 100


text output

print(df)

Out:

            torch       ort  iter_torch  iter_ort
name
NN-CPU   2.774660  9.256914         200       100
LR-CPU  12.983034  9.299986         200       100

Graphs.

print(df.columns)
fig, ax = plt.subplots(1, 1, figsize=(4, 4))
df[['torch', 'ort']].plot.bar(title="Processing time", ax=ax)
ax.tick_params(axis='x', rotation=30)
fig.savefig(__file__ + ".png")

# plt.show()
Traceback (most recent call last):
  File "somewhere/workspace/onnxcustom/onnxcustom_UT_39_std/_doc/examples/plot_orttraining_benchmark_torch.py", line 276, in <module>
    fig.savefig(__file__ + ".png")
NameError: name '__file__' is not defined

Total running time of the script: ( 1 minutes 5.576 seconds)

Gallery generated by Sphinx-Gallery