Benchmark of PolynomialFeatures + partialfit of SGDClassifier (standalone)

This benchmark looks into a new implementation of PolynomialFeatures proposed in PR13290. It tests the following configurations:

This script is standalone and does not require pymlbenchmark as opposed to Benchmark of PolynomialFeatures + partialfit of SGDClassifier which reuse functions implemented in pymlbenchmark.

from time import perf_counter as time
import numpy
import numpy as np
from numpy.random import rand
import matplotlib.pyplot as plt
import pandas
import sklearn
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import SGDClassifier
try:
    from sklearn.utils._testing import ignore_warnings
except ImportError:
    from sklearn.utils.testing import ignore_warnings
from mlinsights.mlmodel import ExtendedFeatures

Implementations to benchmark

def fcts_model(X, y):

    model1 = SGDClassifier()
    model2 = make_pipeline(PolynomialFeatures(), SGDClassifier())
    model3 = make_pipeline(ExtendedFeatures(kind='poly'), SGDClassifier())
    model4 = make_pipeline(ExtendedFeatures(kind='poly-slow'), SGDClassifier())

    model1.fit(PolynomialFeatures().fit_transform(X), y)
    model2.fit(X, y)
    model3.fit(X, y)
    model4.fit(X, y)

    def partial_fit_model1(X, y, model=model1):
        return model.partial_fit(X, y)

    def partial_fit_model2(X, y, model=model2):
        X2 = model.steps[0][1].transform(X)
        return model.steps[1][1].partial_fit(X2, y)

    def partial_fit_model3(X, y, model=model3):
        X2 = model.steps[0][1].transform(X)
        return model.steps[1][1].partial_fit(X2, y)

    def partial_fit_model4(X, y, model=model4):
        X2 = model.steps[0][1].transform(X)
        return model.steps[1][1].partial_fit(X2, y)

    return (partial_fit_model1, partial_fit_model2,
            partial_fit_model3, partial_fit_model4)

Benchmarks

def build_x_y(ntrain, nfeat):
    X_train = np.empty((ntrain, nfeat))
    X_train[:, :] = rand(ntrain, nfeat)[:, :]
    X_trainsum = X_train.sum(axis=1)
    eps = rand(ntrain) - 0.5
    X_trainsum_ = X_trainsum + eps
    y_train = (X_trainsum_ >= X_trainsum).ravel().astype(int)
    return X_train, y_train


@ignore_warnings(category=(FutureWarning, DeprecationWarning))
def bench(n_obs, n_features, repeat=1000, verbose=False):
    res = []
    for n in n_obs:
        for nfeat in n_features:

            X_train, y_train = build_x_y(1000, nfeat)

            obs = dict(n_obs=n, nfeat=nfeat)

            fct1, fct2, fct3, fct4 = fcts_model(X_train, y_train)

            # creates different inputs to avoid caching in any ways
            Xs = []
            Xpolys = []
            for r in range(repeat):
                X, y = build_x_y(n, nfeat)
                Xs.append((X, y))
                Xpolys.append((PolynomialFeatures().fit_transform(X), y))

            # measure fct1
            r = len(Xs)
            st = time()
            for X, y in Xpolys:
                fct1(X, y)
            end = time()
            obs["time_sgd"] = (end - st) / r
            res.append(obs)

            # measures fct2
            st = time()
            for X, y in Xs:
                fct2(X, y)
            end = time()
            obs["time_pipe_skl"] = (end - st) / r
            res.append(obs)

            # measures fct3
            st = time()
            for X, y in Xs:
                fct3(X, y)
            end = time()
            obs["time_pipe_fast"] = (end - st) / r
            res.append(obs)

            # measures fct4
            st = time()
            for X, y in Xs:
                fct4(X, y)
            end = time()
            obs["time_pipe_slow"] = (end - st) / r
            res.append(obs)

            if verbose and (len(res) % 1 == 0 or n >= 10000):
                print("bench", len(res), ":", obs)

    return res

Plots

def plot_results(df, verbose=False):
    nrows = max(len(set(df.nfeat)), 2)
    ncols = max(1, 2)
    fig, ax = plt.subplots(nrows, ncols,
                           figsize=(nrows * 4, ncols * 4))
    colors = "gbry"
    row = 0
    for nfeat in sorted(set(df.nfeat)):
        pos = 0
        for _ in range(1):
            a = ax[row, pos]
            if row == ax.shape[0] - 1:
                a.set_xlabel("N observations", fontsize='x-small')
            if pos == 0:
                a.set_ylabel("Time (s) nfeat={}".format(nfeat),
                             fontsize='x-small')

            subset = df[df.nfeat == nfeat]
            if subset.shape[0] == 0:
                continue
            subset = subset.sort_values("n_obs")
            if verbose:
                print(subset)

            label = "SGD"
            subset.plot(x="n_obs", y="time_sgd", label=label, ax=a,
                        logx=True, logy=True, c=colors[0], style='--')
            label = "SGD-SKL"
            subset.plot(x="n_obs", y="time_pipe_skl", label=label, ax=a,
                        logx=True, logy=True, c=colors[1], style='--')
            label = "SGD-FAST"
            subset.plot(x="n_obs", y="time_pipe_fast", label=label, ax=a,
                        logx=True, logy=True, c=colors[2])
            label = "SGD-SLOW"
            subset.plot(x="n_obs", y="time_pipe_slow", label=label, ax=a,
                        logx=True, logy=True, c=colors[3])

            a.legend(loc=0, fontsize='x-small')
            if row == 0:
                a.set_title("--", fontsize='x-small')
            pos += 1
        row += 1

    plt.suptitle("Benchmark for Polynomial with SGDClassifier", fontsize=16)

Final function for the benchmark

def run_bench(repeat=100, verbose=False):
    n_obs = [10, 100, 1000]
    n_features = [5, 10, 50]

    with sklearn.config_context(assume_finite=True):
        start = time()
        results = bench(n_obs, n_features, repeat=repeat, verbose=verbose)
        end = time()

    results_df = pandas.DataFrame(results)
    print("Total time = %0.3f sec\n" % (end - start))

    # plot the results
    plot_results(results_df, verbose=verbose)
    return results_df

Run the benchmark

print("numpy:", numpy.__version__)
print("scikit-learn:", sklearn.__version__)
df = run_bench(verbose=True)
print(df)

plt.show()
Benchmark for Polynomial with SGDClassifier, --

Out:

numpy: 1.22.4
scikit-learn: 1.1.1
bench 4 : {'n_obs': 10, 'nfeat': 5, 'time_sgd': 0.0007252640900333063, 'time_pipe_skl': 0.001093377219985996, 'time_pipe_fast': 0.0009373169100217638, 'time_pipe_slow': 0.0017718485799923655}
bench 8 : {'n_obs': 10, 'nfeat': 10, 'time_sgd': 0.0007297577400095179, 'time_pipe_skl': 0.00122914702998969, 'time_pipe_fast': 0.0010638399200252025, 'time_pipe_slow': 0.00372598060999735}
bench 12 : {'n_obs': 10, 'nfeat': 50, 'time_sgd': 0.0008929046699995524, 'time_pipe_skl': 0.0025937436800086287, 'time_pipe_fast': 0.0024046017100044993, 'time_pipe_slow': 0.05859663781997369}
bench 16 : {'n_obs': 100, 'nfeat': 5, 'time_sgd': 0.0007852243700108375, 'time_pipe_skl': 0.0012213449099726858, 'time_pipe_fast': 0.0010509258499951102, 'time_pipe_slow': 0.0018742052400193643}
bench 20 : {'n_obs': 100, 'nfeat': 10, 'time_sgd': 0.0008440911700017751, 'time_pipe_skl': 0.0015123411300010047, 'time_pipe_fast': 0.001332182180012751, 'time_pipe_slow': 0.003990463199988881}
bench 24 : {'n_obs': 100, 'nfeat': 50, 'time_sgd': 0.001950420950015541, 'time_pipe_skl': 0.004869367709979997, 'time_pipe_fast': 0.004639451559996815, 'time_pipe_slow': 0.0649360409899964}
bench 28 : {'n_obs': 1000, 'nfeat': 5, 'time_sgd': 0.0014554550100001507, 'time_pipe_skl': 0.0022745840499919724, 'time_pipe_fast': 0.002048752049995528, 'time_pipe_slow': 0.002798120689985808}
bench 32 : {'n_obs': 1000, 'nfeat': 10, 'time_sgd': 0.001964197810011683, 'time_pipe_skl': 0.0036576432999936514, 'time_pipe_fast': 0.003444180079968646, 'time_pipe_slow': 0.006088137340011599}
bench 36 : {'n_obs': 1000, 'nfeat': 50, 'time_sgd': 0.010638297920013428, 'time_pipe_skl': 0.03148720231998595, 'time_pipe_fast': 0.0351094048899904, 'time_pipe_slow': 0.09803675962000852}
Total time = 47.249 sec

    n_obs  nfeat  time_sgd  time_pipe_skl  time_pipe_fast  time_pipe_slow
0      10      5  0.000725       0.001093        0.000937        0.001772
1      10      5  0.000725       0.001093        0.000937        0.001772
2      10      5  0.000725       0.001093        0.000937        0.001772
3      10      5  0.000725       0.001093        0.000937        0.001772
12    100      5  0.000785       0.001221        0.001051        0.001874
13    100      5  0.000785       0.001221        0.001051        0.001874
14    100      5  0.000785       0.001221        0.001051        0.001874
15    100      5  0.000785       0.001221        0.001051        0.001874
24   1000      5  0.001455       0.002275        0.002049        0.002798
25   1000      5  0.001455       0.002275        0.002049        0.002798
26   1000      5  0.001455       0.002275        0.002049        0.002798
27   1000      5  0.001455       0.002275        0.002049        0.002798
    n_obs  nfeat  time_sgd  time_pipe_skl  time_pipe_fast  time_pipe_slow
4      10     10  0.000730       0.001229        0.001064        0.003726
5      10     10  0.000730       0.001229        0.001064        0.003726
6      10     10  0.000730       0.001229        0.001064        0.003726
7      10     10  0.000730       0.001229        0.001064        0.003726
16    100     10  0.000844       0.001512        0.001332        0.003990
17    100     10  0.000844       0.001512        0.001332        0.003990
18    100     10  0.000844       0.001512        0.001332        0.003990
19    100     10  0.000844       0.001512        0.001332        0.003990
28   1000     10  0.001964       0.003658        0.003444        0.006088
29   1000     10  0.001964       0.003658        0.003444        0.006088
30   1000     10  0.001964       0.003658        0.003444        0.006088
31   1000     10  0.001964       0.003658        0.003444        0.006088
    n_obs  nfeat  time_sgd  time_pipe_skl  time_pipe_fast  time_pipe_slow
8      10     50  0.000893       0.002594        0.002405        0.058597
9      10     50  0.000893       0.002594        0.002405        0.058597
10     10     50  0.000893       0.002594        0.002405        0.058597
11     10     50  0.000893       0.002594        0.002405        0.058597
20    100     50  0.001950       0.004869        0.004639        0.064936
21    100     50  0.001950       0.004869        0.004639        0.064936
22    100     50  0.001950       0.004869        0.004639        0.064936
23    100     50  0.001950       0.004869        0.004639        0.064936
32   1000     50  0.010638       0.031487        0.035109        0.098037
33   1000     50  0.010638       0.031487        0.035109        0.098037
34   1000     50  0.010638       0.031487        0.035109        0.098037
35   1000     50  0.010638       0.031487        0.035109        0.098037
    n_obs  nfeat  time_sgd  time_pipe_skl  time_pipe_fast  time_pipe_slow
0      10      5  0.000725       0.001093        0.000937        0.001772
1      10      5  0.000725       0.001093        0.000937        0.001772
2      10      5  0.000725       0.001093        0.000937        0.001772
3      10      5  0.000725       0.001093        0.000937        0.001772
4      10     10  0.000730       0.001229        0.001064        0.003726
5      10     10  0.000730       0.001229        0.001064        0.003726
6      10     10  0.000730       0.001229        0.001064        0.003726
7      10     10  0.000730       0.001229        0.001064        0.003726
8      10     50  0.000893       0.002594        0.002405        0.058597
9      10     50  0.000893       0.002594        0.002405        0.058597
10     10     50  0.000893       0.002594        0.002405        0.058597
11     10     50  0.000893       0.002594        0.002405        0.058597
12    100      5  0.000785       0.001221        0.001051        0.001874
13    100      5  0.000785       0.001221        0.001051        0.001874
14    100      5  0.000785       0.001221        0.001051        0.001874
15    100      5  0.000785       0.001221        0.001051        0.001874
16    100     10  0.000844       0.001512        0.001332        0.003990
17    100     10  0.000844       0.001512        0.001332        0.003990
18    100     10  0.000844       0.001512        0.001332        0.003990
19    100     10  0.000844       0.001512        0.001332        0.003990
20    100     50  0.001950       0.004869        0.004639        0.064936
21    100     50  0.001950       0.004869        0.004639        0.064936
22    100     50  0.001950       0.004869        0.004639        0.064936
23    100     50  0.001950       0.004869        0.004639        0.064936
24   1000      5  0.001455       0.002275        0.002049        0.002798
25   1000      5  0.001455       0.002275        0.002049        0.002798
26   1000      5  0.001455       0.002275        0.002049        0.002798
27   1000      5  0.001455       0.002275        0.002049        0.002798
28   1000     10  0.001964       0.003658        0.003444        0.006088
29   1000     10  0.001964       0.003658        0.003444        0.006088
30   1000     10  0.001964       0.003658        0.003444        0.006088
31   1000     10  0.001964       0.003658        0.003444        0.006088
32   1000     50  0.010638       0.031487        0.035109        0.098037
33   1000     50  0.010638       0.031487        0.035109        0.098037
34   1000     50  0.010638       0.031487        0.035109        0.098037
35   1000     50  0.010638       0.031487        0.035109        0.098037

Total running time of the script: ( 0 minutes 50.693 seconds)

Gallery generated by Sphinx-Gallery