{"cells": [{"cell_type": "markdown", "metadata": {}, "source": ["# example with xgboost\n", "\n", "Test XGBoost after it was compiled, pickle, unpickle."]}, {"cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [{"data": {"text/html": ["
run previous cell, wait for 2 seconds
\n", ""], "text/plain": [""]}, "execution_count": 2, "metadata": {}, "output_type": "execute_result"}], "source": ["from jyquickhelper import add_notebook_menu\n", "add_notebook_menu()"]}, {"cell_type": "markdown", "metadata": {}, "source": ["This is an example taken from [xgboost website](https://github.com/dmlc/xgboost)."]}, {"cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": ["import pickle\n", "\n", "import numpy as np\n", "from sklearn.model_selection import KFold, train_test_split\n", "from sklearn.metrics import confusion_matrix, mean_squared_error\n", "from sklearn.model_selection import GridSearchCV\n", "from sklearn.datasets import load_iris, load_digits, load_diabetes"]}, {"cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": ["import xgboost as xgb"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Zeros and Ones from the Digits dataset: binary classification"]}, {"cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [{"data": {"text/plain": ["[array([[87, 0],\n", " [ 1, 92]], dtype=int64), array([[91, 0],\n", " [ 3, 86]], dtype=int64)]"]}, "execution_count": 5, "metadata": {}, "output_type": "execute_result"}], "source": ["rng = np.random.RandomState(31337)\n", "\n", "digits = load_digits(2)\n", "y = digits['target']\n", "X = digits['data']\n", "conf = []\n", "kf = KFold(n_splits=2, shuffle=True, random_state=rng)\n", "for train_index, test_index in kf.split(X, y):\n", " xgb_model = xgb.XGBClassifier().fit(X[train_index],y[train_index])\n", " predictions = xgb_model.predict(X[test_index])\n", " actuals = y[test_index]\n", " conf.append(confusion_matrix(actuals, predictions))\n", "conf"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Iris: multiclass classification"]}, {"cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [{"data": {"text/plain": ["[array([[19, 0, 0],\n", " [ 0, 31, 3],\n", " [ 0, 1, 21]], dtype=int64), array([[31, 0, 0],\n", " [ 0, 16, 0],\n", " [ 0, 3, 25]], dtype=int64)]"]}, "execution_count": 6, "metadata": {}, "output_type": "execute_result"}], "source": ["iris = load_iris()\n", "y = iris['target']\n", "X = iris['data']\n", "kf = KFold(n_splits=2, shuffle=True, random_state=rng)\n", "conf = []\n", "for train_index, test_index in kf.split(X, y):\n", " xgb_model = xgb.XGBClassifier().fit(X[train_index],y[train_index])\n", " predictions = xgb_model.predict(X[test_index])\n", " actuals = y[test_index]\n", " conf.append(confusion_matrix(actuals, predictions))\n", "conf"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Diabetes: regression"]}, {"cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [{"data": {"text/plain": ["[9.860776812557337, 15.942418468446029]"]}, "execution_count": 7, "metadata": {}, "output_type": "execute_result"}], "source": ["data = load_diabetes()\n", "y = data['target']\n", "X = data['data']\n", "err = []\n", "kf = KFold(n_splits=2, shuffle=True, random_state=rng)\n", "for train_index, test_index in kf.split(X, y):\n", " xgb_model = xgb.XGBRegressor().fit(X[train_index],y[train_index])\n", " predictions = xgb_model.predict(X[test_index])\n", " actuals = y[test_index]\n", " err.append(mean_squared_error(actuals, predictions))\n", "err"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Parameter optimization"]}, {"cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["Fitting 5 folds for each of 9 candidates, totalling 45 fits\n"]}, {"name": "stderr", "output_type": "stream", "text": ["[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n", "[Parallel(n_jobs=1)]: Done 45 out of 45 | elapsed: 2.3s finished\n", "c:\\python370_x64\\lib\\site-packages\\sklearn\\model_selection\\_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n", " DeprecationWarning)\n"]}, {"data": {"text/plain": ["(0.6699572097100618, {'max_depth': 2, 'n_estimators': 100})"]}, "execution_count": 8, "metadata": {}, "output_type": "execute_result"}], "source": ["import joblib # to check you can parallelize GridSearchCV\n", "y = boston['target']\n", "X = boston['data']\n", "xgb_model = xgb.XGBRegressor()\n", "clf = GridSearchCV(xgb_model,\n", " {'max_depth': [2,4,6],\n", " 'n_estimators': [50,100,200]}, verbose=1, n_jobs=1, pre_dispatch=1, cv=5)\n", "clf.fit(X,y)\n", "clf.best_score_, clf.best_params_"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Pickling sklearn API models"]}, {"cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [{"data": {"text/plain": ["True"]}, "execution_count": 9, "metadata": {}, "output_type": "execute_result"}], "source": ["# The sklearn API models are picklable\n", "# must open in binary format to pickle\n", "pickle.dump(clf, open(\"best_boston.pkl\", \"wb\"))\n", "clf2 = pickle.load(open(\"best_boston.pkl\", \"rb\"))\n", "np.allclose(clf.predict(X), clf2.predict(X))"]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Early stopping"]}, {"cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["[0]\tvalidation_0-auc:0.999497\n", "Will train until validation_0-auc hasn't improved in 10 rounds.\n", "[1]\tvalidation_0-auc:0.999497\n", "[2]\tvalidation_0-auc:0.999497\n", "[3]\tvalidation_0-auc:0.999749\n", "[4]\tvalidation_0-auc:0.999749\n", "[5]\tvalidation_0-auc:0.999749\n", "[6]\tvalidation_0-auc:0.999749\n", "[7]\tvalidation_0-auc:0.999749\n", "[8]\tvalidation_0-auc:0.999749\n", "[9]\tvalidation_0-auc:0.999749\n", "[10]\tvalidation_0-auc:1\n", "[11]\tvalidation_0-auc:1\n", "[12]\tvalidation_0-auc:1\n", "[13]\tvalidation_0-auc:1\n", "[14]\tvalidation_0-auc:1\n", "[15]\tvalidation_0-auc:1\n", "[16]\tvalidation_0-auc:1\n", "[17]\tvalidation_0-auc:1\n", "[18]\tvalidation_0-auc:1\n", "[19]\tvalidation_0-auc:1\n", "[20]\tvalidation_0-auc:1\n", "Stopping. Best iteration:\n", "[10]\tvalidation_0-auc:1\n", "\n"]}, {"data": {"text/plain": ["XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n", " colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n", " max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n", " n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n", " reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n", " silent=True, subsample=1)"]}, "execution_count": 10, "metadata": {}, "output_type": "execute_result"}], "source": ["X = digits['data']\n", "y = digits['target']\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n", "clf = xgb.XGBClassifier()\n", "clf.fit(X_train, y_train, early_stopping_rounds=10, eval_metric=\"auc\",\n", " eval_set=[(X_test, y_test)])"]}, {"cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": []}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}, "nbformat": 4, "nbformat_minor": 2}