credit-part3-modeling-1-pipe2-tune-xgbr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "14f976d0",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:23.558342Z",
     "iopub.status.busy": "2021-12-25T12:04:23.556829Z",
     "iopub.status.idle": "2021-12-25T12:04:26.972084Z",
     "shell.execute_reply": "2021-12-25T12:04:26.971119Z",
     "shell.execute_reply.started": "2021-12-25T11:16:25.942907Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.446902,
     "end_time": "2021-12-25T12:04:26.972253",
     "exception": false,
     "start_time": "2021-12-25T12:04:23.525351",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "#from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b70f9754",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.027341,
     "end_time": "2021-12-25T12:04:27.029326",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.001985",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57b70d19",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:27.088123Z",
     "iopub.status.busy": "2021-12-25T12:04:27.087618Z",
     "iopub.status.idle": "2021-12-25T12:04:27.517062Z",
     "shell.execute_reply": "2021-12-25T12:04:27.517673Z",
     "shell.execute_reply.started": "2021-12-25T11:16:29.538586Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.461523,
     "end_time": "2021-12-25T12:04:27.517874",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.056351",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 198 ms, sys: 57.9 ms, total: 256 ms\n",
      "Wall time: 411 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "012d34be",
   "metadata": {
    "papermill": {
     "duration": 0.027483,
     "end_time": "2021-12-25T12:04:27.575328",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.547845",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ff423ea9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:27.654302Z",
     "iopub.status.busy": "2021-12-25T12:04:27.636977Z",
     "iopub.status.idle": "2021-12-25T12:04:27.706241Z",
     "shell.execute_reply": "2021-12-25T12:04:27.705742Z",
     "shell.execute_reply.started": "2021-12-25T11:16:30.016304Z"
    },
    "papermill": {
     "duration": 0.103714,
     "end_time": "2021-12-25T12:04:27.706396",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.602682",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "785455b1",
   "metadata": {
    "papermill": {
     "duration": 0.027448,
     "end_time": "2021-12-25T12:04:27.761885",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.734437",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d935556",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:27.835642Z",
     "iopub.status.busy": "2021-12-25T12:04:27.834323Z",
     "iopub.status.idle": "2021-12-25T12:04:27.873242Z",
     "shell.execute_reply": "2021-12-25T12:04:27.873829Z",
     "shell.execute_reply.started": "2021-12-25T11:16:30.094477Z"
    },
    "papermill": {
     "duration": 0.083534,
     "end_time": "2021-12-25T12:04:27.874019",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.790485",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2994a28",
   "metadata": {
    "papermill": {
     "duration": 0.027669,
     "end_time": "2021-12-25T12:04:27.930692",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.903023",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# X and Y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a0c107ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:27.991389Z",
     "iopub.status.busy": "2021-12-25T12:04:27.990837Z",
     "iopub.status.idle": "2021-12-25T12:04:27.993876Z",
     "shell.execute_reply": "2021-12-25T12:04:27.994233Z",
     "shell.execute_reply.started": "2021-12-25T11:16:30.153121Z"
    },
    "papermill": {
     "duration": 0.035896,
     "end_time": "2021-12-25T12:04:27.994380",
     "exception": false,
     "start_time": "2021-12-25T12:04:27.958484",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eba35144",
   "metadata": {
    "papermill": {
     "duration": 0.0278,
     "end_time": "2021-12-25T12:04:28.049994",
     "exception": false,
     "start_time": "2021-12-25T12:04:28.022194",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d804dfd6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:28.110517Z",
     "iopub.status.busy": "2021-12-25T12:04:28.109027Z",
     "iopub.status.idle": "2021-12-25T12:04:28.111137Z",
     "shell.execute_reply": "2021-12-25T12:04:28.111577Z",
     "shell.execute_reply.started": "2021-12-25T11:16:30.160800Z"
    },
    "papermill": {
     "duration": 0.033594,
     "end_time": "2021-12-25T12:04:28.111696",
     "exception": false,
     "start_time": "2021-12-25T12:04:28.078102",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71afeda4",
   "metadata": {
    "papermill": {
     "duration": 0.027496,
     "end_time": "2021-12-25T12:04:28.167498",
     "exception": false,
     "start_time": "2021-12-25T12:04:28.140002",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Impute NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8fbc5747",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:28.233018Z",
     "iopub.status.busy": "2021-12-25T12:04:28.231455Z",
     "iopub.status.idle": "2021-12-25T12:04:28.235384Z",
     "shell.execute_reply": "2021-12-25T12:04:28.234785Z",
     "shell.execute_reply.started": "2021-12-25T11:16:30.172294Z"
    },
    "papermill": {
     "duration": 0.040359,
     "end_time": "2021-12-25T12:04:28.235522",
     "exception": false,
     "start_time": "2021-12-25T12:04:28.195163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 296 µs, sys: 0 ns, total: 296 µs\n",
      "Wall time: 300 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            #early_stopping_rounds=100 ,\n",
    "            'gpu_id':0, \n",
    "            'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "             'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3cda17b",
   "metadata": {
    "papermill": {
     "duration": 0.027953,
     "end_time": "2021-12-25T12:04:28.292729",
     "exception": false,
     "start_time": "2021-12-25T12:04:28.264776",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# XGBR regression imputer \n",
    "## Predict income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "38775207",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:04:28.356023Z",
     "iopub.status.busy": "2021-12-25T12:04:28.355125Z",
     "iopub.status.idle": "2021-12-25T12:06:38.690632Z",
     "shell.execute_reply": "2021-12-25T12:06:38.691341Z"
    },
    "papermill": {
     "duration": 130.370793,
     "end_time": "2021-12-25T12:06:38.691593",
     "exception": false,
     "start_time": "2021-12-25T12:04:28.320800",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 6s, sys: 2.47 s, total: 2min 9s\n",
      "Wall time: 2min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1dc1be21",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:06:38.757521Z",
     "iopub.status.busy": "2021-12-25T12:06:38.756502Z",
     "iopub.status.idle": "2021-12-25T12:06:38.770868Z",
     "shell.execute_reply": "2021-12-25T12:06:38.771583Z"
    },
    "papermill": {
     "duration": 0.049147,
     "end_time": "2021-12-25T12:06:38.771775",
     "exception": false,
     "start_time": "2021-12-25T12:06:38.722628",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    146076 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "11aa4ac1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:06:38.838979Z",
     "iopub.status.busy": "2021-12-25T12:06:38.838060Z",
     "iopub.status.idle": "2021-12-25T12:06:38.849023Z",
     "shell.execute_reply": "2021-12-25T12:06:38.849441Z"
    },
    "papermill": {
     "duration": 0.045043,
     "end_time": "2021-12-25T12:06:38.849579",
     "exception": false,
     "start_time": "2021-12-25T12:06:38.804536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    98877 non-null   float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22148714",
   "metadata": {
    "papermill": {
     "duration": 0.02977,
     "end_time": "2021-12-25T12:06:38.911502",
     "exception": false,
     "start_time": "2021-12-25T12:06:38.881732",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict number of dependents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "153070d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:06:38.977084Z",
     "iopub.status.busy": "2021-12-25T12:06:38.974111Z",
     "iopub.status.idle": "2021-12-25T12:09:12.266479Z",
     "shell.execute_reply": "2021-12-25T12:09:12.267093Z",
     "shell.execute_reply.started": "2021-12-25T11:18:47.980517Z"
    },
    "papermill": {
     "duration": 153.326209,
     "end_time": "2021-12-25T12:09:12.267324",
     "exception": false,
     "start_time": "2021-12-25T12:06:38.941115",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 32s, sys: 945 ms, total: 2min 33s\n",
      "Wall time: 2min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d667a960",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:12.333374Z",
     "iopub.status.busy": "2021-12-25T12:09:12.332528Z",
     "iopub.status.idle": "2021-12-25T12:09:12.344059Z",
     "shell.execute_reply": "2021-12-25T12:09:12.345154Z",
     "shell.execute_reply.started": "2021-12-25T11:21:30.889115Z"
    },
    "papermill": {
     "duration": 0.046264,
     "end_time": "2021-12-25T12:09:12.345350",
     "exception": false,
     "start_time": "2021-12-25T12:09:12.299086",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    150000 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2a89c032",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:12.411772Z",
     "iopub.status.busy": "2021-12-25T12:09:12.410983Z",
     "iopub.status.idle": "2021-12-25T12:09:12.421061Z",
     "shell.execute_reply": "2021-12-25T12:09:12.420657Z",
     "shell.execute_reply.started": "2021-12-25T11:21:30.910000Z"
    },
    "papermill": {
     "duration": 0.043686,
     "end_time": "2021-12-25T12:09:12.421165",
     "exception": false,
     "start_time": "2021-12-25T12:09:12.377479",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    101503 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18a3a177",
   "metadata": {
    "papermill": {
     "duration": 0.030607,
     "end_time": "2021-12-25T12:09:12.482182",
     "exception": false,
     "start_time": "2021-12-25T12:09:12.451575",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Add bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f80821c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:12.557682Z",
     "iopub.status.busy": "2021-12-25T12:09:12.555435Z",
     "iopub.status.idle": "2021-12-25T12:09:13.359754Z",
     "shell.execute_reply": "2021-12-25T12:09:13.360541Z",
     "shell.execute_reply.started": "2021-12-25T11:21:30.933213Z"
    },
    "papermill": {
     "duration": 0.848267,
     "end_time": "2021-12-25T12:09:13.360891",
     "exception": false,
     "start_time": "2021-12-25T12:09:12.512624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) \n",
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5f67687",
   "metadata": {
    "papermill": {
     "duration": 0.065588,
     "end_time": "2021-12-25T12:09:13.494325",
     "exception": false,
     "start_time": "2021-12-25T12:09:13.428737",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7a9c6439",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:13.630278Z",
     "iopub.status.busy": "2021-12-25T12:09:13.629251Z",
     "iopub.status.idle": "2021-12-25T12:09:13.631931Z",
     "shell.execute_reply": "2021-12-25T12:09:13.631098Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.661807Z"
    },
    "papermill": {
     "duration": 0.074319,
     "end_time": "2021-12-25T12:09:13.632087",
     "exception": false,
     "start_time": "2021-12-25T12:09:13.557768",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c4dd8b82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:13.767129Z",
     "iopub.status.busy": "2021-12-25T12:09:13.765044Z",
     "iopub.status.idle": "2021-12-25T12:09:13.793469Z",
     "shell.execute_reply": "2021-12-25T12:09:13.794680Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.667663Z"
    },
    "papermill": {
     "duration": 0.101539,
     "end_time": "2021-12-25T12:09:13.794860",
     "exception": false,
     "start_time": "2021-12-25T12:09:13.693321",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "eeff78a2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:13.880497Z",
     "iopub.status.busy": "2021-12-25T12:09:13.879515Z",
     "iopub.status.idle": "2021-12-25T12:09:13.898724Z",
     "shell.execute_reply": "2021-12-25T12:09:13.898214Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.693077Z"
    },
    "papermill": {
     "duration": 0.057491,
     "end_time": "2021-12-25T12:09:13.898836",
     "exception": false,
     "start_time": "2021-12-25T12:09:13.841345",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "abe3b260",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:13.973039Z",
     "iopub.status.busy": "2021-12-25T12:09:13.971422Z",
     "iopub.status.idle": "2021-12-25T12:09:13.973730Z",
     "shell.execute_reply": "2021-12-25T12:09:13.974179Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.716060Z"
    },
    "papermill": {
     "duration": 0.041146,
     "end_time": "2021-12-25T12:09:13.974322",
     "exception": false,
     "start_time": "2021-12-25T12:09:13.933176",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20c8ef5f",
   "metadata": {
    "papermill": {
     "duration": 0.034222,
     "end_time": "2021-12-25T12:09:14.042648",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.008426",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extract final cat  and num Features \n",
    "## Cat Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a88a8553",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.119452Z",
     "iopub.status.busy": "2021-12-25T12:09:14.113474Z",
     "iopub.status.idle": "2021-12-25T12:09:14.125186Z",
     "shell.execute_reply": "2021-12-25T12:09:14.124650Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.723417Z"
    },
    "papermill": {
     "duration": 0.048284,
     "end_time": "2021-12-25T12:09:14.125320",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.077036",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac845a18",
   "metadata": {
    "papermill": {
     "duration": 0.034556,
     "end_time": "2021-12-25T12:09:14.195063",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.160507",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fe58e643",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.274184Z",
     "iopub.status.busy": "2021-12-25T12:09:14.270890Z",
     "iopub.status.idle": "2021-12-25T12:09:14.278068Z",
     "shell.execute_reply": "2021-12-25T12:09:14.277661Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.737696Z"
    },
    "papermill": {
     "duration": 0.048375,
     "end_time": "2021-12-25T12:09:14.278196",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.229821",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f3ec0ef",
   "metadata": {
    "papermill": {
     "duration": 0.031108,
     "end_time": "2021-12-25T12:09:14.341462",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.310354",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7d1b9b6a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.416186Z",
     "iopub.status.busy": "2021-12-25T12:09:14.414944Z",
     "iopub.status.idle": "2021-12-25T12:09:14.417030Z",
     "shell.execute_reply": "2021-12-25T12:09:14.417456Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.750990Z"
    },
    "papermill": {
     "duration": 0.044797,
     "end_time": "2021-12-25T12:09:14.417574",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.372777",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeaturesEngineer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X=X.copy()\n",
    "        # Calculate some metrics across rows\n",
    "        X[\"num_missing\"]  = X.isnull().sum(axis=1)\n",
    "        X[\"std_row\"] = X.std(axis=1)\n",
    "        X[\"sem_row\"] = X.sem(axis=1)\n",
    "        X[\"abs_sum_row\"] = X.abs().sum(axis=1)\n",
    "        X[\"mean_row\"] = X.mean(axis=1)\n",
    "        X[\"max_row\"]= X.max(axis=1)\n",
    "        X[\"min_row\"]= X.min(axis=1)\n",
    "        X['Weighted_Sum_PastDue'] = 2 * X['NumberOfTime30-59DaysPastDueNotWorse'] + 3 * X['NumberOfTime60-89DaysPastDueNotWorse'] + 6 * X['NumberOfTimes90DaysLate']\n",
    "        X['90days_out_of_TotalPastDue'] = X['NumberOfTimes90DaysLate'] / (X['Weighted_Sum_PastDue'])\n",
    "        X.loc[X['Weighted_Sum_PastDue']==0, '90days_out_of_TotalPastDue'] = 0\n",
    "        X['RemainingLines'] = X['NumberOfOpenCreditLinesAndLoans'] - X['NumberRealEstateLoansOrLines']\n",
    "        X['Loans_vs_Other_Lines'] = X['RemainingLines'] / (1 + X['NumberRealEstateLoansOrLines'])\n",
    "        X['Debt_per_Real_Estate_Loan'] = X['DebtRatio'] / X['NumberRealEstateLoansOrLines']\n",
    "        X.loc[X['NumberRealEstateLoansOrLines']==0, 'Debt_per_Real_Estate_Loan'] = 0\n",
    "        X['Disposable_Income_per_person'] = (X['MonthlyIncome'] - X['DebtRatio']) / (X['NumberOfDependents'] + 1)\n",
    "        X.loc[X['NumberOfDependents']==0, 'Disposable_Income_per_person'] = 0\n",
    "        X['RemainingLines_per_person'] = X['RemainingLines'] / (X['NumberOfDependents'] + 1)\n",
    "        X['NumberRE_X_DebtRatio_X_age'] = X['NumberRealEstateLoansOrLines'] * X['DebtRatio'] / X['age']\n",
    "        X['RevolvingUtilization_per_age'] = X['RevolvingUtilizationOfUnsecuredLines'] / X['age']\n",
    "        X[X==np.inf]=np.nan\n",
    "        X=reduce_mem_usage(X)\n",
    "        return X  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0e6739f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.487493Z",
     "iopub.status.busy": "2021-12-25T12:09:14.486869Z",
     "iopub.status.idle": "2021-12-25T12:09:14.489515Z",
     "shell.execute_reply": "2021-12-25T12:09:14.489072Z",
     "shell.execute_reply.started": "2021-12-25T11:21:31.765504Z"
    },
    "papermill": {
     "duration": 0.040852,
     "end_time": "2021-12-25T12:09:14.489616",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.448764",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "75599a28",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.560204Z",
     "iopub.status.busy": "2021-12-25T12:09:14.558655Z",
     "iopub.status.idle": "2021-12-25T12:09:14.560789Z",
     "shell.execute_reply": "2021-12-25T12:09:14.561176Z",
     "shell.execute_reply.started": "2021-12-25T11:49:55.108747Z"
    },
    "papermill": {
     "duration": 0.040269,
     "end_time": "2021-12-25T12:09:14.561309",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.521040",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "                        #('sparse_features', SparseInteractions(degree=2)),\n",
    "                        #('reducedim',  SelectPercentile( mutual_info_classif, percentile=90))\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            #('general_features',FeaturesEngineer()),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            #('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "                ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac20cd79",
   "metadata": {
    "papermill": {
     "duration": 0.031055,
     "end_time": "2021-12-25T12:09:14.623415",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.592360",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Second pipe to tune : \n",
    "imputer + transformer+ ploy fe + kmeans fe\n",
    "\n",
    "imputer + encoder =cat feat "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7d6fb901",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.694215Z",
     "iopub.status.busy": "2021-12-25T12:09:14.693541Z",
     "iopub.status.idle": "2021-12-25T12:09:14.795080Z",
     "shell.execute_reply": "2021-12-25T12:09:14.795498Z",
     "shell.execute_reply.started": "2021-12-25T11:49:58.343630Z"
    },
    "papermill": {
     "duration": 0.138594,
     "end_time": "2021-12-25T12:09:14.795613",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.657019",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4b52353b-6d86-4e8c-9c7f-a03605a185ab\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4b52353b-6d86-4e8c-9c7f-a03605a185ab\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfT...\n",
       "                                                 ('encoder', WOEEncoder())]),\n",
       "                                 Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"8aa9c67c-dd42-4e9e-9004-d54996755ae3\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"8aa9c67c-dd42-4e9e-9004-d54996755ae3\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"71370e19-945f-46a7-98d0-0a0cf619393d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"71370e19-945f-46a7-98d0-0a0cf619393d\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"8e5abf48-303b-4c05-91a4-daebbbec91ac\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"8e5abf48-303b-4c05-91a4-daebbbec91ac\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"391a3430-82ee-40e3-a9c0-5a9d295e2c46\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"391a3430-82ee-40e3-a9c0-5a9d295e2c46\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f8fabf54-a728-43d6-b314-773621c9091a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f8fabf54-a728-43d6-b314-773621c9091a\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4176a70e-691d-44f0-8f0c-e2961c512751\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4176a70e-691d-44f0-8f0c-e2961c512751\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c1cd2d7b-045d-4e6a-8b42-40359c8cccc4\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c1cd2d7b-045d-4e6a-8b42-40359c8cccc4\">categorical</label><div class=\"sk-toggleable__content\"><pre>Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"982e46ce-fcca-4fde-bfc2-d743c8c83086\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"982e46ce-fcca-4fde-bfc2-d743c8c83086\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, fill_value='missing',\n",
       "              strategy='most_frequent')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"07281ecb-9991-484b-8090-fef02663938e\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"07281ecb-9991-484b-8090-fef02663938e\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfT...\n",
       "                                                 ('encoder', WOEEncoder())]),\n",
       "                                 Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11dbd2d9",
   "metadata": {
    "papermill": {
     "duration": 0.032181,
     "end_time": "2021-12-25T12:09:14.860412",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.828231",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "00f5a681",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:14.931364Z",
     "iopub.status.busy": "2021-12-25T12:09:14.930704Z",
     "iopub.status.idle": "2021-12-25T12:09:14.933678Z",
     "shell.execute_reply": "2021-12-25T12:09:14.934095Z",
     "shell.execute_reply.started": "2021-12-25T11:50:03.131634Z"
    },
    "papermill": {
     "duration": 0.041487,
     "end_time": "2021-12-25T12:09:14.934211",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.892724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StratifiedKFold(n_splits=5, random_state=1, shuffle=True)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N_FOLD = 5\n",
    "cross_validation_design = StratifiedKFold( n_splits=N_FOLD,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9843d6fd",
   "metadata": {
    "papermill": {
     "duration": 0.032086,
     "end_time": "2021-12-25T12:09:14.998960",
     "exception": false,
     "start_time": "2021-12-25T12:09:14.966874",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e97368b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:15.068544Z",
     "iopub.status.busy": "2021-12-25T12:09:15.067829Z",
     "iopub.status.idle": "2021-12-25T12:09:19.380741Z",
     "shell.execute_reply": "2021-12-25T12:09:19.379992Z",
     "shell.execute_reply.started": "2021-12-25T12:00:07.309492Z"
    },
    "papermill": {
     "duration": 4.349951,
     "end_time": "2021-12-25T12:09:19.380896",
     "exception": false,
     "start_time": "2021-12-25T12:09:15.030945",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 77)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer.fit(X,y)\n",
    "X_pre=preprocessor_woe_powertransformer.transform(X)\n",
    "X_pre.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0a106504",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:19.463615Z",
     "iopub.status.busy": "2021-12-25T12:09:19.462726Z",
     "iopub.status.idle": "2021-12-25T12:09:19.464675Z",
     "shell.execute_reply": "2021-12-25T12:09:19.465043Z",
     "shell.execute_reply.started": "2021-12-25T12:01:53.211476Z"
    },
    "papermill": {
     "duration": 0.05081,
     "end_time": "2021-12-25T12:09:19.465169",
     "exception": false,
     "start_time": "2021-12-25T12:09:19.414359",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    " def objective_xgbr_pipe2(trial,X=X_pre,y=y):\n",
    "    # Optuna+cv :\n",
    "    #https://www.kaggle.com/hamidrezabakhtaki/xgboost-catboost-lighgbm-optuna-final-submission\n",
    "    # https://www.kaggle.com/prashant111/lightgbm-classifier-in-python\n",
    "    # https://www.kaggle.com/tunguz/tps-09-21-histgradientboosting-with-optuna\n",
    "    \n",
    "    X_train,X_test, y_train, y_test = train_test_split(X, y,stratify=y, test_size=0.25,random_state=42,shuffle=True) \n",
    "    \n",
    "    param = {   \"verbosity\": 0,\n",
    "                \"objective\": \"binary:logistic\",\n",
    "                \"eval_metric\": \"auc\",\n",
    "                'random_state': 48,\n",
    "                # regression\n",
    "                #'objective':'reg:squarederror', \n",
    "                # 'eval_metric': 'mae',\n",
    "                #early_stopping_rounds=100 ,\n",
    "                'gpu_id':0, \n",
    "                'predictor':\"gpu_predictor\",\n",
    "                # use exact for small dataset.\n",
    "                #\"tree_method\": \"exact\",\n",
    "                # big data :\n",
    "                 'tree_method': 'gpu_hist',\n",
    "                # defines booster, gblinear for linear functions.\n",
    "                \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n",
    "                # L2 regularization weight.\n",
    "                \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "                # L1 regularization weight.\n",
    "                \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "                # sampling ratio for training data.\n",
    "                \"subsample\": trial.suggest_float(\"subsample\", 0.2, 1.0),\n",
    "                # sampling according to each tree.\n",
    "                \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "                \"learning_rate\" : trial.suggest_float(\"learning_rate\" , 0.005 , 0.2),\n",
    "                \"n_estimators\" : trial.suggest_int(\"n_estimators\" , 1000 ,10000),\n",
    "             }\n",
    "    if  param[\"booster\"] in [\"gbtree\", \"dart\"]:\n",
    "        # maximum depth of the tree, signifies complexity of the tree.\n",
    "        param[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n",
    "        # minimum child weight, larger the term more conservative the tree.\n",
    "        param[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n",
    "        param[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n",
    "        # defines how selective algorithm is.\n",
    "        param[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n",
    "        param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "        \n",
    "    if  param[\"booster\"] == \"dart\":\n",
    "        param[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n",
    "        param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n",
    "        param[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n",
    "        param[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n",
    "             \n",
    "    #pipe_xgbr_pipe1 = Pipeline(\n",
    "     #               steps=[\n",
    "      #                  ('preprocessor', preprocessor_woe_powertransformer),\n",
    "       #                 ('classifier', XGBClassifier(\n",
    "                      #objective=\"mae\",\n",
    "        #               **param))\n",
    "         #           ]\n",
    "          #      )\n",
    "    roc_auc = list()\n",
    "    oof = np.empty((X.shape[0],))\n",
    "    predictions=[]\n",
    "    mean_auc = 0\n",
    "   # for fold, (train_idx, test_idx) in enumerate(cross_validation_design.split(X, y)):    \n",
    "   #     X_train, y_train = X.iloc[list(train_idx), :], y.iloc[list(train_idx)]\n",
    "   #     X_test, y_test = X.iloc[list(test_idx), :],y.iloc[list(test_idx)]\n",
    "        #X_valid = X.iloc[valid_idx][Reduced_Features]\n",
    "        #X_train_weight=X.iloc[train_idx]['weight']\n",
    "   #     pipe_xgbr_pipe1.fit(X_train,\n",
    "    #                        y_train,\n",
    "                            #classifier__eval_metric=\"rmse\"\n",
    "                            #classifier__eval_set = [(X_test, y_test)],\n",
    "                            #classifier__early_stopping_rounds = 100,\n",
    "                            #classifier__sample_weight=X_train_weight)\n",
    "     #                      )\n",
    "     #   preds = pipe_xgbr_pipe1.predict_proba(X_test)[:,1]\n",
    "      #  oof[test_idx] = preds\n",
    "      #  auc_score= roc_auc_score(y_true=y_test, y_score=preds)\n",
    "      #  roc_auc.append(auc_score)\n",
    "       # print(f\"Fold {fold} | AUC: {auc_score}\")\n",
    "       # mean_auc += auc_score / N_FOLD\n",
    "    \n",
    "    #mae =cross_val_score(pipe_xgbr, data, target,cv=cross_validation_design_regression,scoring='neg_mean_absolute_error').mean()\n",
    "    model  = XGBClassifier(**param)\n",
    "    model.fit( X_train, y_train,\n",
    "              eval_set=[(X_test, y_test)],\n",
    "             early_stopping_rounds=100,\n",
    "              verbose=False) \n",
    "    #pipeline_model_lgbm.fit(X_train,y_train)\n",
    "    preds = model.predict_proba(X_test)[:,1]\n",
    "    #y_pred= model.predict_proba(X_test)[:,1]\n",
    "    #mae = mean_absolute_error(y_test, y_pred)\n",
    "    #oof[test_idx] = preds\n",
    "    #average_precesion = average_precision_score(y_true= y_test, y_score= preds)\n",
    "    final_auc= roc_auc_score(y_true=y_test, y_score=preds)   \n",
    "    return final_auc               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5f149c32",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:09:19.538568Z",
     "iopub.status.busy": "2021-12-25T12:09:19.537833Z",
     "iopub.status.idle": "2021-12-25T14:37:25.915999Z",
     "shell.execute_reply": "2021-12-25T14:37:25.915560Z",
     "shell.execute_reply.started": "2021-12-25T12:01:58.380032Z"
    },
    "papermill": {
     "duration": 8886.417048,
     "end_time": "2021-12-25T14:37:25.916118",
     "exception": false,
     "start_time": "2021-12-25T12:09:19.499070",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:09:19,536]\u001b[0m A new study created in memory with name: no-name-2ca78d0e-4ebd-45ed-84f3-50feb0b92cf1\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 12:09:20,574]\u001b[0m Trial 0 finished with value: 0.866714604044994 and parameters: {'booster': 'gbtree', 'lambda': 0.0007838129294351114, 'alpha': 1.583889303293342e-06, 'subsample': 0.3397990627204906, 'colsample_bytree': 0.8444760356520797, 'learning_rate': 0.19853489061421042, 'n_estimators': 2254, 'max_depth': 3, 'min_child_weight': 2, 'eta': 0.002610546386919438, 'gamma': 8.482414635969582e-07, 'grow_policy': 'lossguide'}. Best is trial 0 with value: 0.866714604044994.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 12:09:21,914]\u001b[0m Trial 1 finished with value: 0.8674154709727687 and parameters: {'booster': 'gbtree', 'lambda': 0.022927300568847175, 'alpha': 1.2386683649126889e-07, 'subsample': 0.22385755747916847, 'colsample_bytree': 0.8049864164885507, 'learning_rate': 0.0601019663855617, 'n_estimators': 1317, 'max_depth': 5, 'min_child_weight': 9, 'eta': 0.0015293945054450934, 'gamma': 2.964436942610112e-07, 'grow_policy': 'depthwise'}. Best is trial 1 with value: 0.8674154709727687.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 12:10:06,813]\u001b[0m Trial 2 finished with value: 0.8674906976414289 and parameters: {'booster': 'dart', 'lambda': 2.802421850758759e-08, 'alpha': 0.18231727046831395, 'subsample': 0.2862157965181072, 'colsample_bytree': 0.21883469800543598, 'learning_rate': 0.07088513406612974, 'n_estimators': 1336, 'max_depth': 5, 'min_child_weight': 8, 'eta': 1.7833557778540097e-07, 'gamma': 0.0010411664246381873, 'grow_policy': 'depthwise', 'sample_type': 'weighted', 'normalize_type': 'tree', 'rate_drop': 5.489440321327749e-05, 'skip_drop': 2.9118229221346966e-07}. Best is trial 2 with value: 0.8674906976414289.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 12:10:10,117]\u001b[0m Trial 3 finished with value: 0.8690486890444473 and parameters: {'booster': 'gbtree', 'lambda': 0.006192038585140987, 'alpha': 3.88518230277705e-08, 'subsample': 0.6364534602359602, 'colsample_bytree': 0.2217039319496805, 'learning_rate': 0.02115203575601813, 'n_estimators': 3133, 'max_depth': 5, 'min_child_weight': 10, 'eta': 2.5176520014680957e-06, 'gamma': 2.5332769149375033e-08, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:53:01,014]\u001b[0m Trial 4 finished with value: 0.8646435330083493 and parameters: {'booster': 'dart', 'lambda': 1.9645830543276186e-07, 'alpha': 0.0003999530172790818, 'subsample': 0.30867660065896313, 'colsample_bytree': 0.662524423197197, 'learning_rate': 0.033026271936102816, 'n_estimators': 2708, 'max_depth': 3, 'min_child_weight': 4, 'eta': 0.1276727897032761, 'gamma': 0.0004805458049605306, 'grow_policy': 'lossguide', 'sample_type': 'weighted', 'normalize_type': 'tree', 'rate_drop': 0.029104135690579847, 'skip_drop': 4.8484916114950785e-05}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:54:48,444]\u001b[0m Trial 5 finished with value: 0.8638999441290609 and parameters: {'booster': 'gblinear', 'lambda': 8.498961496954367e-08, 'alpha': 6.464922942778201e-05, 'subsample': 0.7151149304704505, 'colsample_bytree': 0.6553658284182197, 'learning_rate': 0.07476703856567986, 'n_estimators': 1968}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:55:33,138]\u001b[0m Trial 6 finished with value: 0.8629440454528268 and parameters: {'booster': 'gblinear', 'lambda': 0.000819642880823373, 'alpha': 0.0013061435981298782, 'subsample': 0.6197512596284647, 'colsample_bytree': 0.564476294750416, 'learning_rate': 0.191994608138706, 'n_estimators': 3149}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:55:35,777]\u001b[0m Trial 7 finished with value: 0.8660706731118561 and parameters: {'booster': 'gbtree', 'lambda': 2.4625226292042087e-08, 'alpha': 0.6467762207200615, 'subsample': 0.840999049933081, 'colsample_bytree': 0.544013699354173, 'learning_rate': 0.08211024381477379, 'n_estimators': 1859, 'max_depth': 9, 'min_child_weight': 10, 'eta': 2.7044078616420354e-05, 'gamma': 2.172405937100324e-06, 'grow_policy': 'depthwise'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:55:54,996]\u001b[0m Trial 8 finished with value: 0.8416863481465139 and parameters: {'booster': 'gblinear', 'lambda': 0.00023080498736191386, 'alpha': 0.034476500247549606, 'subsample': 0.2256235827787692, 'colsample_bytree': 0.46938593374635185, 'learning_rate': 0.11219726784973634, 'n_estimators': 5964}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:04,582]\u001b[0m Trial 9 finished with value: 0.8638675819514562 and parameters: {'booster': 'gblinear', 'lambda': 0.00038646448633617705, 'alpha': 1.6585088001721887e-06, 'subsample': 0.7582978004355299, 'colsample_bytree': 0.5486924314421686, 'learning_rate': 0.1295682026697106, 'n_estimators': 1328}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:24,385]\u001b[0m Trial 10 finished with value: 0.8681920720099731 and parameters: {'booster': 'gbtree', 'lambda': 0.0725682153785229, 'alpha': 1.0288562619533048e-08, 'subsample': 0.4970303945043861, 'colsample_bytree': 0.20105042211446894, 'learning_rate': 0.005377152137078539, 'n_estimators': 9299, 'max_depth': 9, 'min_child_weight': 6, 'eta': 2.730660308285152e-08, 'gamma': 0.1799271490473076, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:40,556]\u001b[0m Trial 11 finished with value: 0.8677892039501836 and parameters: {'booster': 'gbtree', 'lambda': 0.5709559628356422, 'alpha': 1.061991162430027e-08, 'subsample': 0.48699111450401655, 'colsample_bytree': 0.2208367806046457, 'learning_rate': 0.005078638614739152, 'n_estimators': 9782, 'max_depth': 9, 'min_child_weight': 6, 'eta': 3.1444014066593305e-08, 'gamma': 0.4279720832159163, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:52,084]\u001b[0m Trial 12 finished with value: 0.8680614886848006 and parameters: {'booster': 'gbtree', 'lambda': 0.06125714653804166, 'alpha': 1.3480224305438073e-08, 'subsample': 0.4502153873536019, 'colsample_bytree': 0.36709070931549725, 'learning_rate': 0.006772564172767672, 'n_estimators': 9566, 'max_depth': 7, 'min_child_weight': 6, 'eta': 1.4797992628827787e-06, 'gamma': 1.1394345368145778e-08, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:54,792]\u001b[0m Trial 13 finished with value: 0.8670567046472589 and parameters: {'booster': 'gbtree', 'lambda': 7.713701105544093e-06, 'alpha': 4.328275496348305e-07, 'subsample': 0.9740054508963937, 'colsample_bytree': 0.3459493394517698, 'learning_rate': 0.038919480044326694, 'n_estimators': 4933, 'max_depth': 7, 'min_child_weight': 7, 'eta': 2.691943515324342e-06, 'gamma': 0.7330488000630803, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:57,748]\u001b[0m Trial 14 finished with value: 0.8676353353654379 and parameters: {'booster': 'gbtree', 'lambda': 0.014749573880981748, 'alpha': 1.629156526818051e-05, 'subsample': 0.49108762726892496, 'colsample_bytree': 0.347074931179977, 'learning_rate': 0.03968093469119917, 'n_estimators': 7621, 'max_depth': 7, 'min_child_weight': 4, 'eta': 1.66109948857368e-08, 'gamma': 0.015849626083078616, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:57:58,853]\u001b[0m Trial 15 finished with value: 0.8679852300298567 and parameters: {'booster': 'gbtree', 'lambda': 0.4547521430168615, 'alpha': 8.222433944172878e-08, 'subsample': 0.6050058437435145, 'colsample_bytree': 0.24754477622129095, 'learning_rate': 0.15334423380722845, 'n_estimators': 4241, 'max_depth': 5, 'min_child_weight': 5, 'eta': 9.493112139458407e-07, 'gamma': 1.9730467146790535e-05, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:58:25,666]\u001b[0m Trial 16 finished with value: 0.8656083261520011 and parameters: {'booster': 'dart', 'lambda': 0.005134555075924185, 'alpha': 5.863435704835841e-06, 'subsample': 0.422450302960667, 'colsample_bytree': 0.9432050435602475, 'learning_rate': 0.028238321382534338, 'n_estimators': 6384, 'max_depth': 9, 'min_child_weight': 10, 'eta': 4.31264835796431e-05, 'gamma': 1.0991738624809464e-08, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'forest', 'rate_drop': 1.722217867660228e-08, 'skip_drop': 0.9753006197035599}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:58:29,662]\u001b[0m Trial 17 finished with value: 0.868693434893251 and parameters: {'booster': 'gbtree', 'lambda': 2.1508422865244016e-06, 'alpha': 6.083456427852447e-08, 'subsample': 0.679147701357272, 'colsample_bytree': 0.4341711428462972, 'learning_rate': 0.017193247499385472, 'n_estimators': 8071, 'max_depth': 5, 'min_child_weight': 8, 'eta': 2.1027894875038696e-07, 'gamma': 0.014715105367247565, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 13:58:31,042]\u001b[0m Trial 18 finished with value: 0.8683126832687905 and parameters: {'booster': 'gbtree', 'lambda': 1.1551733451706738e-05, 'alpha': 1.6986870011697265e-07, 'subsample': 0.714510167757411, 'colsample_bytree': 0.42490189812087126, 'learning_rate': 0.10228497816094631, 'n_estimators': 7970, 'max_depth': 5, 'min_child_weight': 8, 'eta': 1.2297782486192243e-05, 'gamma': 0.008144300031666996, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:04,821]\u001b[0m Trial 19 finished with value: 0.8582438325648893 and parameters: {'booster': 'dart', 'lambda': 1.2882089843244318e-05, 'alpha': 0.006352962644669114, 'subsample': 0.8525062004595916, 'colsample_bytree': 0.29950585728731344, 'learning_rate': 0.052582469093847665, 'n_estimators': 3661, 'max_depth': 3, 'min_child_weight': 9, 'eta': 2.6403023444486156e-07, 'gamma': 2.7863361652065592e-05, 'grow_policy': 'depthwise', 'sample_type': 'uniform', 'normalize_type': 'forest', 'rate_drop': 0.9384135195628456, 'skip_drop': 0.44409737452990417}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:06,371]\u001b[0m Trial 20 finished with value: 0.8680793574417796 and parameters: {'booster': 'gbtree', 'lambda': 8.390891145733687e-07, 'alpha': 4.8245208166646155e-05, 'subsample': 0.632629491320225, 'colsample_bytree': 0.4222554333669002, 'learning_rate': 0.09434177340094282, 'n_estimators': 8205, 'max_depth': 5, 'min_child_weight': 8, 'eta': 0.0006747063816192881, 'gamma': 0.009897187454025057, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:07,748]\u001b[0m Trial 21 finished with value: 0.8670509688560908 and parameters: {'booster': 'gbtree', 'lambda': 2.145538139646805e-05, 'alpha': 9.748114162263734e-08, 'subsample': 0.7160268207498001, 'colsample_bytree': 0.45942294582864984, 'learning_rate': 0.16316261241063537, 'n_estimators': 7734, 'max_depth': 5, 'min_child_weight': 8, 'eta': 1.675259547445086e-05, 'gamma': 0.00867275763553024, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:09,216]\u001b[0m Trial 22 finished with value: 0.8677465390144866 and parameters: {'booster': 'gbtree', 'lambda': 1.1628644561081233e-06, 'alpha': 4.731817261688787e-07, 'subsample': 0.8071327184808391, 'colsample_bytree': 0.41108380531205485, 'learning_rate': 0.11276752907414311, 'n_estimators': 6871, 'max_depth': 5, 'min_child_weight': 9, 'eta': 5.144304040130846e-06, 'gamma': 0.0012861479699657805, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:13,817]\u001b[0m Trial 23 finished with value: 0.8671106245051883 and parameters: {'booster': 'gbtree', 'lambda': 5.6910611742149953e-05, 'alpha': 5.8636122494864106e-08, 'subsample': 0.9247348369502839, 'colsample_bytree': 0.4930718273548492, 'learning_rate': 0.022507222290697404, 'n_estimators': 8711, 'max_depth': 7, 'min_child_weight': 10, 'eta': 2.2349748493977042e-07, 'gamma': 0.06496107680779227, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:15,627]\u001b[0m Trial 24 finished with value: 0.8689459351394453 and parameters: {'booster': 'gbtree', 'lambda': 2.9255022358640574e-06, 'alpha': 4.837560600515168e-07, 'subsample': 0.6738879151334073, 'colsample_bytree': 0.6518552071235617, 'learning_rate': 0.053320894820627907, 'n_estimators': 5137, 'max_depth': 3, 'min_child_weight': 7, 'eta': 0.00026767500738954393, 'gamma': 0.053287386520148194, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:17,789]\u001b[0m Trial 25 finished with value: 0.8688545102772378 and parameters: {'booster': 'gbtree', 'lambda': 4.2014346348165837e-07, 'alpha': 2.965680425517601e-06, 'subsample': 0.6659773492085256, 'colsample_bytree': 0.7268220698524042, 'learning_rate': 0.049628201603377144, 'n_estimators': 5190, 'max_depth': 3, 'min_child_weight': 7, 'eta': 0.00020626271894234085, 'gamma': 0.00010366845583694685, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:19,705]\u001b[0m Trial 26 finished with value: 0.8684762787518789 and parameters: {'booster': 'gbtree', 'lambda': 3.772080060028168e-07, 'alpha': 4.149284802782073e-06, 'subsample': 0.5434409725715709, 'colsample_bytree': 0.7513191412454037, 'learning_rate': 0.05909674945378814, 'n_estimators': 5089, 'max_depth': 3, 'min_child_weight': 7, 'eta': 0.00019534251426553788, 'gamma': 0.00010832949736910029, 'grow_policy': 'depthwise'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:00:21,424]\u001b[0m Trial 27 finished with value: 0.8684353642017574 and parameters: {'booster': 'gbtree', 'lambda': 3.169862339719255e-06, 'alpha': 8.61210607823023e-07, 'subsample': 0.5634940346481319, 'colsample_bytree': 0.6863353423106018, 'learning_rate': 0.04846333002587035, 'n_estimators': 4298, 'max_depth': 3, 'min_child_weight': 7, 'eta': 0.05865318382584549, 'gamma': 9.581682728858553e-08, 'grow_policy': 'lossguide'}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:03:15,796]\u001b[0m Trial 28 finished with value: 0.8635918648646689 and parameters: {'booster': 'gblinear', 'lambda': 0.003032652176630049, 'alpha': 1.0314117397266609e-05, 'subsample': 0.666946392676223, 'colsample_bytree': 0.7260871196357821, 'learning_rate': 0.06258264147671037, 'n_estimators': 5548}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:04:21,564]\u001b[0m Trial 29 finished with value: 0.868883850616553 and parameters: {'booster': 'dart', 'lambda': 5.595264469080869e-05, 'alpha': 2.4091787996610724e-06, 'subsample': 0.38977881253850816, 'colsample_bytree': 0.8626402985003747, 'learning_rate': 0.08147417527208888, 'n_estimators': 3894, 'max_depth': 3, 'min_child_weight': 5, 'eta': 0.017363688434374306, 'gamma': 5.522462694830085e-06, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 1.462493297446841e-08, 'skip_drop': 1.9467182156074227e-07}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:05:15,211]\u001b[0m Trial 30 finished with value: 0.8680589115698822 and parameters: {'booster': 'dart', 'lambda': 8.440928428730519e-05, 'alpha': 2.806218166652211e-05, 'subsample': 0.35863112927341967, 'colsample_bytree': 0.9231295861910743, 'learning_rate': 0.08599575702987311, 'n_estimators': 3383, 'max_depth': 3, 'min_child_weight': 3, 'eta': 0.021402892906022687, 'gamma': 3.375756509224118e-06, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 1.56951858941927e-08, 'skip_drop': 1.2045524585622191e-08}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:10:35,144]\u001b[0m Trial 31 finished with value: 0.8688296456795398 and parameters: {'booster': 'dart', 'lambda': 4.771113766575355e-05, 'alpha': 2.52194429388762e-06, 'subsample': 0.5708685924284301, 'colsample_bytree': 0.8669648470070634, 'learning_rate': 0.046847472396434345, 'n_estimators': 4185, 'max_depth': 3, 'min_child_weight': 5, 'eta': 0.7132179236442143, 'gamma': 6.1021817898265785e-06, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 7.89148979824493e-06, 'skip_drop': 1.180717993413937e-05}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:12:37,815]\u001b[0m Trial 32 finished with value: 0.8685387281759989 and parameters: {'booster': 'dart', 'lambda': 4.228285552425683e-06, 'alpha': 2.525663607430652e-07, 'subsample': 0.7858215419212734, 'colsample_bytree': 0.798225362396239, 'learning_rate': 0.06758870206743331, 'n_estimators': 4923, 'max_depth': 3, 'min_child_weight': 5, 'eta': 0.000258196571580818, 'gamma': 1.5531929758572356e-07, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 1.2298729800430944e-06, 'skip_drop': 0.0018206343489344835}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:33:43,078]\u001b[0m Trial 33 finished with value: 0.8688091770013157 and parameters: {'booster': 'dart', 'lambda': 0.0011157282446531808, 'alpha': 9.74583036389134e-07, 'subsample': 0.6891107758484324, 'colsample_bytree': 0.9864154981274244, 'learning_rate': 0.01885649157092663, 'n_estimators': 2786, 'max_depth': 3, 'min_child_weight': 7, 'eta': 0.00837267322778896, 'gamma': 0.0001916063895388605, 'grow_policy': 'lossguide', 'sample_type': 'weighted', 'normalize_type': 'forest', 'rate_drop': 0.00231177693041918, 'skip_drop': 4.576633148675567e-07}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 14:37:25,865]\u001b[0m Trial 34 finished with value: 0.8684566539077432 and parameters: {'booster': 'dart', 'lambda': 8.337904628617345e-08, 'alpha': 0.0003622683084297955, 'subsample': 0.6510633828394352, 'colsample_bytree': 0.8401648591599817, 'learning_rate': 0.03691547023425712, 'n_estimators': 3683, 'max_depth': 3, 'min_child_weight': 4, 'eta': 6.817665254045947e-05, 'gamma': 5.714586064039627e-05, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 7.56135173554374e-07, 'skip_drop': 0.006887843369127247}. Best is trial 3 with value: 0.8690486890444473.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2h 33min 53s, sys: 7.66 s, total: 2h 34min 1s\n",
      "Wall time: 2h 28min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_xgbr_pipe2 = optuna.create_study(direction=\"maximize\")\n",
    "study_xgbr_pipe2.optimize(objective_xgbr_pipe2 ,n_trials=35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0e557ba5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T14:37:26.015652Z",
     "iopub.status.busy": "2021-12-25T14:37:26.010106Z",
     "iopub.status.idle": "2021-12-25T14:37:26.021317Z",
     "shell.execute_reply": "2021-12-25T14:37:26.020887Z",
     "shell.execute_reply.started": "2021-12-25T12:02:40.273777Z"
    },
    "papermill": {
     "duration": 0.061505,
     "end_time": "2021-12-25T14:37:26.021436",
     "exception": false,
     "start_time": "2021-12-25T14:37:25.959931",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of finished trials :  35\n",
      "Best Trials :  {'booster': 'gbtree', 'lambda': 0.006192038585140987, 'alpha': 3.88518230277705e-08, 'subsample': 0.6364534602359602, 'colsample_bytree': 0.2217039319496805, 'learning_rate': 0.02115203575601813, 'n_estimators': 3133, 'max_depth': 5, 'min_child_weight': 10, 'eta': 2.5176520014680957e-06, 'gamma': 2.5332769149375033e-08, 'grow_policy': 'lossguide'}\n",
      "Best Values :  0.8690486890444473\n"
     ]
    }
   ],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_xgbr_pipe2.trials))\n",
    "print(\"Best Trials : \", study_xgbr_pipe2.best_trial.params)\n",
    "print(\"Best Values : \" , study_xgbr_pipe2.best_value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 9192.275204,
   "end_time": "2021-12-25T14:37:27.491600",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-25T12:04:15.216396",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}