# LogisticRegression and Clustering#

A logistic regression implements a convex partition of the features spaces. A clustering algorithm applied before the trainer modifies the feature space in way the partition is not necessarily convex in the initial features. Letâ€™s see how.

```from jyquickhelper import add_notebook_menu
```
```%matplotlib inline
```

## A dummy datasets and not convex#

```import numpy
import numpy.random
Xs = []
Ys = []
n = 20
for i in range(0, 5):
for j in range(0, 4):
x1 = numpy.random.rand(n) + i*1.1
x2 = numpy.random.rand(n) + j*1.1
Xs.append(numpy.vstack([x1,x2]).T)
cl = numpy.random.randint(0, 4)
Ys.extend([cl for i in range(n)])
X = numpy.vstack(Xs)
Y = numpy.array(Ys)
X.shape, Y.shape, set(Y)
```
```((400, 2), (400,), {0, 1, 2, 3})
```
```import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 1, figsize=(6,4))
for i in set(Y):
ax.plot(X[Y==i,0], X[Y==i,1], 'o', label="cl%d"%i, color=plt.cm.tab20.colors[i])
ax.legend()
ax.set_title("Classification not convex");
```

## One function to plot classification in 2D#

```import matplotlib.pyplot as plt

def draw_border(clr, X, y, fct=None, incx=1, incy=1, figsize=None, border=True, clusters=None, ax=None):

# see https://sashat.me/2017/01/11/list-of-20-simple-distinct-colors/
# https://matplotlib.org/examples/color/colormaps_reference.html
_unused_ = ["Red", "Green", "Yellow", "Blue", "Orange", "Purple", "Cyan",
"Magenta", "Lime", "Pink", "Teal", "Lavender", "Brown", "Beige",
"Maroon", "Mint", "Olive", "Coral", "Navy", "Grey", "White", "Black"]

h = .02  # step size in the mesh
# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = X[:, 0].min() - incx, X[:, 0].max() + incx
y_min, y_max = X[:, 1].min() - incy, X[:, 1].max() + incy
xx, yy = numpy.meshgrid(numpy.arange(x_min, x_max, h), numpy.arange(y_min, y_max, h))
if fct is None:
Z = clr.predict(numpy.c_[xx.ravel(), yy.ravel()])
else:
Z = fct(clr, numpy.c_[xx.ravel(), yy.ravel()])

# Put the result into a color plot
cmap = plt.cm.tab20
Z = Z.reshape(xx.shape)
if ax is None:
fig, ax = plt.subplots(1, 1, figsize=figsize or (4, 3))
ax.pcolormesh(xx, yy, Z, cmap=cmap)

# Plot also the training points
ax.scatter(X[:, 0], X[:, 1], c=y, edgecolors='k', cmap=cmap)
ax.set_xlabel('Sepal length')
ax.set_ylabel('Sepal width')

ax.set_xlim(xx.min(), xx.max())
ax.set_ylim(yy.min(), yy.max())

# Plot clusters
if clusters is not None:
mat = []
ym = []
for k, v in clusters.items():
mat.append(v.cluster_centers_)
ym.extend(k for i in range(v.cluster_centers_.shape[0]))
cx = numpy.vstack(mat)
ym = numpy.array(ym)
ax.scatter(cx[:, 0], cx[:, 1], c=ym, edgecolors='y', cmap=cmap, s=300)
return ax
```

## Logistic Regression#

```from sklearn.linear_model import LogisticRegression
clr = LogisticRegression(solver='lbfgs', multi_class='multinomial')
clr.fit(X, Y)
```
```LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,
intercept_scaling=1, max_iter=100, multi_class='multinomial',
n_jobs=None, penalty='l2', random_state=None, solver='lbfgs',
tol=0.0001, verbose=0, warm_start=False)
```
```ax = draw_border(clr, X, Y, incx=1, incy=1, figsize=(6,4), border=False)
ax.set_title("Logistic Regression");
```

Not quite close!

## Logistic Regression and k-means#

```from mlinsights.mlmodel import ClassifierAfterKMeans
clk = ClassifierAfterKMeans(e_solver='lbfgs', e_multi_class='multinomial')
clk.fit(X, Y)
```
```c:python370_x64libsite-packagessklearnlinear_modellogistic.py:757: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.
"of iterations.", ConvergenceWarning)```
```ClassifierAfterKMeans(c_algorithm='auto', c_copy_x=True, c_init='k-means++',
c_max_iter=300, c_n_clusters=2, c_n_init=10, c_n_jobs=None,
c_precompute_distances='auto', c_random_state=None,
c_tol=0.0001, c_verbose=0, e_C=1.0, e_class_weight=None,
e_dual=False, e_fit_intercept=True, e_intercept_scaling=1,
e_max_iter=100, e_multi_class='multinomial', e_n_jobs=None,
e_penalty='l2', e_random_state=None, e_solver='lbfgs',
e_tol=0.0001, e_verbose=0, e_warm_start=False)
```

The centers of the first k-means:

```clk.clus_[0].cluster_centers_
```
```array([[3.26205371, 1.08211905],
[1.06113799, 3.78383125]])
```
```ax = draw_border(clk, X, Y, incx=1, incy=1, figsize=(6,4), border=False, clusters=clk.clus_)
ax.set_title("Logistic Regression and K-Means - 2 clusters per class");
```

The big cricles are the centers of the k-means fitted for each class. It look better!

## Variation#

```dt = []
for cl in range(1, 6):
clk = ClassifierAfterKMeans(c_n_clusters=cl, e_solver='lbfgs',
e_multi_class='multinomial', e_max_iter=700)
clk.fit(X, Y)
sc = clk.score(X,Y)
dt.append(dict(score=sc, nb_clusters=cl))
import pandas
pandas.DataFrame(dt)
```
nb_clusters score
0 1 0.4475
1 2 0.6600
2 3 0.7475
3 4 0.8400
4 5 0.9200
```ax = draw_border(clk, X, Y, incx=1, incy=1, figsize=(6,4), border=False, clusters=clk.clus_)
ax.set_title("Logistic Regression and K-Means - 8 clusters per class");
```

## Random Forest#

The random forest works without any clustering as expected.

```from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators=20)
rf.fit(X, Y)
```
```RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
max_depth=None, max_features='auto', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=20, n_jobs=None,
oob_score=False, random_state=None, verbose=0,
warm_start=False)
```
```ax = draw_border(rf, X, Y, incx=1, incy=1, figsize=(6,4), border=False)
ax.set_title("Random Forest");
```