credit-part3-modeling-1-pipe1-tune-randomforest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c1711c01",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:49.838330Z",
     "iopub.status.busy": "2021-12-26T17:37:49.837635Z",
     "iopub.status.idle": "2021-12-26T17:37:53.149784Z",
     "shell.execute_reply": "2021-12-26T17:37:53.150264Z",
     "shell.execute_reply.started": "2021-12-26T16:08:36.040310Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.357693,
     "end_time": "2021-12-26T17:37:53.150604",
     "exception": false,
     "start_time": "2021-12-26T17:37:49.792911",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "# Ensemble tree\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75d81096",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.024158,
     "end_time": "2021-12-26T17:37:53.200151",
     "exception": false,
     "start_time": "2021-12-26T17:37:53.175993",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "29efb3e3",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:53.252848Z",
     "iopub.status.busy": "2021-12-26T17:37:53.251923Z",
     "iopub.status.idle": "2021-12-26T17:37:53.771065Z",
     "shell.execute_reply": "2021-12-26T17:37:53.771478Z",
     "shell.execute_reply.started": "2021-12-26T16:08:39.695006Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.547274,
     "end_time": "2021-12-26T17:37:53.771718",
     "exception": false,
     "start_time": "2021-12-26T17:37:53.224444",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 190 ms, sys: 70 ms, total: 260 ms\n",
      "Wall time: 497 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b08f2f84",
   "metadata": {
    "papermill": {
     "duration": 0.025249,
     "end_time": "2021-12-26T17:37:53.823487",
     "exception": false,
     "start_time": "2021-12-26T17:37:53.798238",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce9e040e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:53.883160Z",
     "iopub.status.busy": "2021-12-26T17:37:53.882543Z",
     "iopub.status.idle": "2021-12-26T17:37:53.962378Z",
     "shell.execute_reply": "2021-12-26T17:37:53.961862Z",
     "shell.execute_reply.started": "2021-12-26T16:08:40.151903Z"
    },
    "papermill": {
     "duration": 0.113668,
     "end_time": "2021-12-26T17:37:53.962535",
     "exception": false,
     "start_time": "2021-12-26T17:37:53.848867",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcfa038a",
   "metadata": {
    "papermill": {
     "duration": 0.024852,
     "end_time": "2021-12-26T17:37:54.014053",
     "exception": false,
     "start_time": "2021-12-26T17:37:53.989201",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "29093b1e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:54.068047Z",
     "iopub.status.busy": "2021-12-26T17:37:54.067047Z",
     "iopub.status.idle": "2021-12-26T17:37:54.124838Z",
     "shell.execute_reply": "2021-12-26T17:37:54.124134Z",
     "shell.execute_reply.started": "2021-12-26T16:08:40.244662Z"
    },
    "papermill": {
     "duration": 0.085998,
     "end_time": "2021-12-26T17:37:54.125016",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.039018",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6187c78",
   "metadata": {
    "papermill": {
     "duration": 0.025123,
     "end_time": "2021-12-26T17:37:54.176715",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.151592",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# X and Y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6ffa8e76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:54.230913Z",
     "iopub.status.busy": "2021-12-26T17:37:54.229781Z",
     "iopub.status.idle": "2021-12-26T17:37:54.234648Z",
     "shell.execute_reply": "2021-12-26T17:37:54.235053Z",
     "shell.execute_reply.started": "2021-12-26T16:08:40.310636Z"
    },
    "papermill": {
     "duration": 0.033387,
     "end_time": "2021-12-26T17:37:54.235219",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.201832",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26f7c25c",
   "metadata": {
    "papermill": {
     "duration": 0.02514,
     "end_time": "2021-12-26T17:37:54.285707",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.260567",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "53091970",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:54.340158Z",
     "iopub.status.busy": "2021-12-26T17:37:54.339206Z",
     "iopub.status.idle": "2021-12-26T17:37:54.342644Z",
     "shell.execute_reply": "2021-12-26T17:37:54.343039Z",
     "shell.execute_reply.started": "2021-12-26T16:08:40.317535Z"
    },
    "papermill": {
     "duration": 0.032055,
     "end_time": "2021-12-26T17:37:54.343205",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.311150",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48090805",
   "metadata": {
    "papermill": {
     "duration": 0.025066,
     "end_time": "2021-12-26T17:37:54.393972",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.368906",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Impute NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bbda7a2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:54.448145Z",
     "iopub.status.busy": "2021-12-26T17:37:54.447475Z",
     "iopub.status.idle": "2021-12-26T17:37:54.456401Z",
     "shell.execute_reply": "2021-12-26T17:37:54.455699Z",
     "shell.execute_reply.started": "2021-12-26T16:08:40.330073Z"
    },
    "papermill": {
     "duration": 0.037462,
     "end_time": "2021-12-26T17:37:54.456609",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.419147",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 273 µs, sys: 51 µs, total: 324 µs\n",
      "Wall time: 329 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            'early_stopping_rounds':100 ,\n",
    "           # 'gpu_id':0, \n",
    "           # 'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "            # 'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ede7b12",
   "metadata": {
    "papermill": {
     "duration": 0.025875,
     "end_time": "2021-12-26T17:37:54.509551",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.483676",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# XGBR regression imputer \n",
    "## Predict income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "36944341",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:37:54.569656Z",
     "iopub.status.busy": "2021-12-26T17:37:54.568534Z",
     "iopub.status.idle": "2021-12-26T17:57:00.497857Z",
     "shell.execute_reply": "2021-12-26T17:57:00.498335Z",
     "shell.execute_reply.started": "2021-12-26T16:08:40.346440Z"
    },
    "papermill": {
     "duration": 1145.963365,
     "end_time": "2021-12-26T17:57:00.498521",
     "exception": false,
     "start_time": "2021-12-26T17:37:54.535156",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 14min 8s, sys: 6.25 s, total: 1h 14min 14s\n",
      "Wall time: 19min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e33515ce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:57:00.557082Z",
     "iopub.status.busy": "2021-12-26T17:57:00.556476Z",
     "iopub.status.idle": "2021-12-26T17:57:00.570407Z",
     "shell.execute_reply": "2021-12-26T17:57:00.570978Z",
     "shell.execute_reply.started": "2021-12-26T16:27:37.418387Z"
    },
    "papermill": {
     "duration": 0.045904,
     "end_time": "2021-12-26T17:57:00.571154",
     "exception": false,
     "start_time": "2021-12-26T17:57:00.525250",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    146076 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fb544712",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:57:00.630114Z",
     "iopub.status.busy": "2021-12-26T17:57:00.629225Z",
     "iopub.status.idle": "2021-12-26T17:57:00.639805Z",
     "shell.execute_reply": "2021-12-26T17:57:00.640321Z",
     "shell.execute_reply.started": "2021-12-26T16:27:37.446231Z"
    },
    "papermill": {
     "duration": 0.042847,
     "end_time": "2021-12-26T17:57:00.640481",
     "exception": false,
     "start_time": "2021-12-26T17:57:00.597634",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    98877 non-null   float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "583feb87",
   "metadata": {
    "papermill": {
     "duration": 0.026448,
     "end_time": "2021-12-26T17:57:00.694364",
     "exception": false,
     "start_time": "2021-12-26T17:57:00.667916",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict number of dependents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f34a10a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T17:57:00.750710Z",
     "iopub.status.busy": "2021-12-26T17:57:00.750136Z",
     "iopub.status.idle": "2021-12-26T18:20:03.666272Z",
     "shell.execute_reply": "2021-12-26T18:20:03.666776Z",
     "shell.execute_reply.started": "2021-12-26T16:27:37.517287Z"
    },
    "papermill": {
     "duration": 1382.945823,
     "end_time": "2021-12-26T18:20:03.666957",
     "exception": false,
     "start_time": "2021-12-26T17:57:00.721134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 29min 19s, sys: 5.49 s, total: 1h 29min 24s\n",
      "Wall time: 23min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "935ca82c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:03.725620Z",
     "iopub.status.busy": "2021-12-26T18:20:03.723161Z",
     "iopub.status.idle": "2021-12-26T18:20:03.738128Z",
     "shell.execute_reply": "2021-12-26T18:20:03.737661Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.170200Z"
    },
    "papermill": {
     "duration": 0.044151,
     "end_time": "2021-12-26T18:20:03.738261",
     "exception": false,
     "start_time": "2021-12-26T18:20:03.694110",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    150000 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "38aaea2f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:03.796756Z",
     "iopub.status.busy": "2021-12-26T18:20:03.795862Z",
     "iopub.status.idle": "2021-12-26T18:20:03.809285Z",
     "shell.execute_reply": "2021-12-26T18:20:03.809747Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.189962Z"
    },
    "papermill": {
     "duration": 0.04393,
     "end_time": "2021-12-26T18:20:03.809925",
     "exception": false,
     "start_time": "2021-12-26T18:20:03.765995",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    101503 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "886428d5",
   "metadata": {
    "papermill": {
     "duration": 0.027587,
     "end_time": "2021-12-26T18:20:03.865217",
     "exception": false,
     "start_time": "2021-12-26T18:20:03.837630",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Add bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "57604d13",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:03.923793Z",
     "iopub.status.busy": "2021-12-26T18:20:03.922827Z",
     "iopub.status.idle": "2021-12-26T18:20:04.608773Z",
     "shell.execute_reply": "2021-12-26T18:20:04.609333Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.208919Z"
    },
    "papermill": {
     "duration": 0.716844,
     "end_time": "2021-12-26T18:20:04.609537",
     "exception": false,
     "start_time": "2021-12-26T18:20:03.892693",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) \n",
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bd16bb3",
   "metadata": {
    "papermill": {
     "duration": 0.027283,
     "end_time": "2021-12-26T18:20:04.664762",
     "exception": false,
     "start_time": "2021-12-26T18:20:04.637479",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9dd3ef46",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:04.725177Z",
     "iopub.status.busy": "2021-12-26T18:20:04.724242Z",
     "iopub.status.idle": "2021-12-26T18:20:04.727032Z",
     "shell.execute_reply": "2021-12-26T18:20:04.726540Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.909639Z"
    },
    "papermill": {
     "duration": 0.034486,
     "end_time": "2021-12-26T18:20:04.727165",
     "exception": false,
     "start_time": "2021-12-26T18:20:04.692679",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "66132af9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:04.789395Z",
     "iopub.status.busy": "2021-12-26T18:20:04.788791Z",
     "iopub.status.idle": "2021-12-26T18:20:04.810088Z",
     "shell.execute_reply": "2021-12-26T18:20:04.810614Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.915719Z"
    },
    "papermill": {
     "duration": 0.055361,
     "end_time": "2021-12-26T18:20:04.810815",
     "exception": false,
     "start_time": "2021-12-26T18:20:04.755454",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7b68d687",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:04.872657Z",
     "iopub.status.busy": "2021-12-26T18:20:04.871996Z",
     "iopub.status.idle": "2021-12-26T18:20:04.894209Z",
     "shell.execute_reply": "2021-12-26T18:20:04.893636Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.944069Z"
    },
    "papermill": {
     "duration": 0.054256,
     "end_time": "2021-12-26T18:20:04.894341",
     "exception": false,
     "start_time": "2021-12-26T18:20:04.840085",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "67ebbbcc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:04.956662Z",
     "iopub.status.busy": "2021-12-26T18:20:04.955941Z",
     "iopub.status.idle": "2021-12-26T18:20:04.958837Z",
     "shell.execute_reply": "2021-12-26T18:20:04.959299Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.968382Z"
    },
    "papermill": {
     "duration": 0.03688,
     "end_time": "2021-12-26T18:20:04.959473",
     "exception": false,
     "start_time": "2021-12-26T18:20:04.922593",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08c5da0c",
   "metadata": {
    "papermill": {
     "duration": 0.028175,
     "end_time": "2021-12-26T18:20:05.016064",
     "exception": false,
     "start_time": "2021-12-26T18:20:04.987889",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extract final cat  and num Features \n",
    "## Cat Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5d85949b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.083377Z",
     "iopub.status.busy": "2021-12-26T18:20:05.082737Z",
     "iopub.status.idle": "2021-12-26T18:20:05.086037Z",
     "shell.execute_reply": "2021-12-26T18:20:05.085476Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.973758Z"
    },
    "papermill": {
     "duration": 0.041089,
     "end_time": "2021-12-26T18:20:05.086179",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.045090",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18b435f3",
   "metadata": {
    "papermill": {
     "duration": 0.028254,
     "end_time": "2021-12-26T18:20:05.143141",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.114887",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "48d994e3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.212368Z",
     "iopub.status.busy": "2021-12-26T18:20:05.209784Z",
     "iopub.status.idle": "2021-12-26T18:20:05.215839Z",
     "shell.execute_reply": "2021-12-26T18:20:05.215341Z",
     "shell.execute_reply.started": "2021-12-26T16:50:22.994411Z"
    },
    "papermill": {
     "duration": 0.042385,
     "end_time": "2021-12-26T18:20:05.215972",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.173587",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "909c8227",
   "metadata": {
    "papermill": {
     "duration": 0.028335,
     "end_time": "2021-12-26T18:20:05.273003",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.244668",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5bc1d0cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.341646Z",
     "iopub.status.busy": "2021-12-26T18:20:05.341017Z",
     "iopub.status.idle": "2021-12-26T18:20:05.343928Z",
     "shell.execute_reply": "2021-12-26T18:20:05.343334Z",
     "shell.execute_reply.started": "2021-12-26T16:50:23.008894Z"
    },
    "papermill": {
     "duration": 0.041913,
     "end_time": "2021-12-26T18:20:05.344061",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.302148",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeaturesEngineer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X=X.copy()\n",
    "        # Calculate some metrics across rows\n",
    "        X[\"num_missing\"]  = X.isnull().sum(axis=1)\n",
    "        X[\"std_row\"] = X.std(axis=1)\n",
    "        X[\"sem_row\"] = X.sem(axis=1)\n",
    "        X[\"abs_sum_row\"] = X.abs().sum(axis=1)\n",
    "        X[\"mean_row\"] = X.mean(axis=1)\n",
    "        X[\"max_row\"]= X.max(axis=1)\n",
    "        X[\"min_row\"]= X.min(axis=1)\n",
    "        X['Weighted_Sum_PastDue'] = 2 * X['NumberOfTime30-59DaysPastDueNotWorse'] + 3 * X['NumberOfTime60-89DaysPastDueNotWorse'] + 6 * X['NumberOfTimes90DaysLate']\n",
    "        X['90days_out_of_TotalPastDue'] = X['NumberOfTimes90DaysLate'] / (X['Weighted_Sum_PastDue'])\n",
    "        X.loc[X['Weighted_Sum_PastDue']==0, '90days_out_of_TotalPastDue'] = 0\n",
    "        X['RemainingLines'] = X['NumberOfOpenCreditLinesAndLoans'] - X['NumberRealEstateLoansOrLines']\n",
    "        X['Loans_vs_Other_Lines'] = X['RemainingLines'] / (1 + X['NumberRealEstateLoansOrLines'])\n",
    "        X['Debt_per_Real_Estate_Loan'] = X['DebtRatio'] / X['NumberRealEstateLoansOrLines']\n",
    "        X.loc[X['NumberRealEstateLoansOrLines']==0, 'Debt_per_Real_Estate_Loan'] = 0\n",
    "        X['Disposable_Income_per_person'] = (X['MonthlyIncome'] - X['DebtRatio']) / (X['NumberOfDependents'] + 1)\n",
    "        X.loc[X['NumberOfDependents']==0, 'Disposable_Income_per_person'] = 0\n",
    "        X['RemainingLines_per_person'] = X['RemainingLines'] / (X['NumberOfDependents'] + 1)\n",
    "        X['NumberRE_X_DebtRatio_X_age'] = X['NumberRealEstateLoansOrLines'] * X['DebtRatio'] / X['age']\n",
    "        X['RevolvingUtilization_per_age'] = X['RevolvingUtilizationOfUnsecuredLines'] / X['age']\n",
    "        X[X==np.inf]=np.nan\n",
    "        X=reduce_mem_usage(X)\n",
    "        return X  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "21a5b133",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.409859Z",
     "iopub.status.busy": "2021-12-26T18:20:05.409078Z",
     "iopub.status.idle": "2021-12-26T18:20:05.411569Z",
     "shell.execute_reply": "2021-12-26T18:20:05.411051Z",
     "shell.execute_reply.started": "2021-12-26T16:50:23.024093Z"
    },
    "papermill": {
     "duration": 0.038925,
     "end_time": "2021-12-26T18:20:05.411712",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.372787",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "01ee6c36",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.475830Z",
     "iopub.status.busy": "2021-12-26T18:20:05.475236Z",
     "iopub.status.idle": "2021-12-26T18:20:05.478360Z",
     "shell.execute_reply": "2021-12-26T18:20:05.477810Z",
     "shell.execute_reply.started": "2021-12-26T16:50:23.040447Z"
    },
    "papermill": {
     "duration": 0.037956,
     "end_time": "2021-12-26T18:20:05.478487",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.440531",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "               # ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ce8b6ef",
   "metadata": {
    "papermill": {
     "duration": 0.028968,
     "end_time": "2021-12-26T18:20:05.536663",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.507695",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# First pipe to tune : \n",
    "imputer + transformer + kmeans fe+ploy fe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7dbc598d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.602221Z",
     "iopub.status.busy": "2021-12-26T18:20:05.601483Z",
     "iopub.status.idle": "2021-12-26T18:20:05.664078Z",
     "shell.execute_reply": "2021-12-26T18:20:05.663567Z",
     "shell.execute_reply.started": "2021-12-26T16:50:23.058142Z"
    },
    "papermill": {
     "duration": 0.097346,
     "end_time": "2021-12-26T18:20:05.664228",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.566882",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"256be325-913a-4a5c-b4b9-5acdbae1e40f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"256be325-913a-4a5c-b4b9-5acdbae1e40f\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"15387511-ddc7-4a12-b3d4-648f83b286a9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"15387511-ddc7-4a12-b3d4-648f83b286a9\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c89059bb-2ce3-4449-a923-261bc2526b5c\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c89059bb-2ce3-4449-a923-261bc2526b5c\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"9046ea1a-36d6-41fe-baf2-d6ffbb0435ee\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"9046ea1a-36d6-41fe-baf2-d6ffbb0435ee\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"dccd1dc9-83e3-48a1-a4ac-f94d5b912c05\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"dccd1dc9-83e3-48a1-a4ac-f94d5b912c05\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"9b46f46b-e0f1-41a8-840e-e91693af02b8\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"9b46f46b-e0f1-41a8-840e-e91693af02b8\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c9fd05f1-8708-44b9-ac0d-5d853862f773\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c9fd05f1-8708-44b9-ac0d-5d853862f773\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "11bd8e4c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:05.731566Z",
     "iopub.status.busy": "2021-12-26T18:20:05.730942Z",
     "iopub.status.idle": "2021-12-26T18:20:09.884093Z",
     "shell.execute_reply": "2021-12-26T18:20:09.883207Z",
     "shell.execute_reply.started": "2021-12-26T16:50:23.152150Z"
    },
    "papermill": {
     "duration": 4.18976,
     "end_time": "2021-12-26T18:20:09.884240",
     "exception": false,
     "start_time": "2021-12-26T18:20:05.694480",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 78)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer.fit(X,y)\n",
    "X_pre=preprocessor_woe_powertransformer.transform(X)\n",
    "X_pre.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50562edb",
   "metadata": {
    "papermill": {
     "duration": 0.029668,
     "end_time": "2021-12-26T18:20:09.943699",
     "exception": false,
     "start_time": "2021-12-26T18:20:09.914031",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f3952628",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:10.009789Z",
     "iopub.status.busy": "2021-12-26T18:20:10.008792Z",
     "iopub.status.idle": "2021-12-26T18:20:10.013010Z",
     "shell.execute_reply": "2021-12-26T18:20:10.012356Z",
     "shell.execute_reply.started": "2021-12-26T16:50:27.513105Z"
    },
    "papermill": {
     "duration": 0.039504,
     "end_time": "2021-12-26T18:20:10.013157",
     "exception": false,
     "start_time": "2021-12-26T18:20:09.973653",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StratifiedKFold(n_splits=5, random_state=1, shuffle=True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N_FOLD = 5\n",
    "cross_validation_design = StratifiedKFold(n_splits=N_FOLD,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a6ef09a",
   "metadata": {
    "papermill": {
     "duration": 0.030615,
     "end_time": "2021-12-26T18:20:10.074939",
     "exception": false,
     "start_time": "2021-12-26T18:20:10.044324",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8815cc17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:10.139744Z",
     "iopub.status.busy": "2021-12-26T18:20:10.139055Z",
     "iopub.status.idle": "2021-12-26T18:20:10.145620Z",
     "shell.execute_reply": "2021-12-26T18:20:10.146043Z",
     "shell.execute_reply.started": "2021-12-26T16:50:27.522373Z"
    },
    "papermill": {
     "duration": 0.040668,
     "end_time": "2021-12-26T18:20:10.146212",
     "exception": false,
     "start_time": "2021-12-26T18:20:10.105544",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective_rf_pipe1(trial,X=X_pre,y=y):\n",
    "    # https://www.kaggle.com/hamidrezabakhtaki/xgboost-catboost-lighgbm-optuna-final-submission\n",
    "    # https://www.kaggle.com/prashant111/lightgbm-classifier-in-python\n",
    "    # https://www.kaggle.com/tunguz/tps-09-21-histgradientboosting-with-optuna\n",
    "    # https://github.com/solegalli/optuna-examples/tree/main/catboost\n",
    "    # https://www.kaggle.com/dixhom/bayesian-optimization-with-optuna-stacking\n",
    "    # Handle Pruning : \n",
    "    # https://towardsdatascience.com/exploring-optuna-a-hyper-parameter-framework-using-logistic-regression-84bd622cd3a5\n",
    "   \n",
    "    X_train,X_test, y_train,  y_test = train_test_split(X, y,stratify=y, \n",
    "                                                       test_size=0.25,\n",
    "                                                      random_state=42,\n",
    "                                                        shuffle=True)\n",
    "    param = {\n",
    "        \"n_estimators\": trial.suggest_int(\"n_estimators\", 200, 5000),\n",
    "        \"max_features\": trial.suggest_categorical(\"max_features\", [\"auto\", \"sqrt\"]),\n",
    "        \"max_depth\": trial.suggest_int(\"max_depth\", 10, 100),\n",
    "        \"min_samples_split\": trial.suggest_int(\"min_samples_split\", 2, 10),\n",
    "        \"min_samples_leaf\": trial.suggest_int(\"min_samples_leaf\", 1, 5),\n",
    "        \"bootstrap\": trial.suggest_categorical(\"bootstrap\", [True, False]),\n",
    "    } \n",
    "    model= RandomForestClassifier(**param,\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             )\n",
    "    model.fit( X_train, y_train,\n",
    "             # eval_set=[(X_test, y_test)],\n",
    "              #early_stopping_rounds=100,\n",
    "             # verbose=False\n",
    "             )\n",
    "    preds = model.predict_proba(X_test)[:,1]\n",
    "    final_auc= roc_auc_score(y_true=y_test, y_score=preds)   \n",
    "    return final_auc               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "510d1995",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T18:20:10.210344Z",
     "iopub.status.busy": "2021-12-26T18:20:10.209755Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": false,
     "start_time": "2021-12-26T18:20:10.176962",
     "status": "running"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 18:20:10,210]\u001b[0m A new study created in memory with name: no-name-dd886d61-ae30-4454-b36c-a9c01bda9e1d\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 19:08:58,014]\u001b[0m Trial 0 finished with value: 0.8556079286377266 and parameters: {'n_estimators': 3151, 'max_features': 'sqrt', 'max_depth': 59, 'min_samples_split': 6, 'min_samples_leaf': 4, 'bootstrap': True}. Best is trial 0 with value: 0.8556079286377266.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 19:47:45,937]\u001b[0m Trial 1 finished with value: 0.8451020117871306 and parameters: {'n_estimators': 1564, 'max_features': 'auto', 'max_depth': 49, 'min_samples_split': 2, 'min_samples_leaf': 3, 'bootstrap': False}. Best is trial 0 with value: 0.8556079286377266.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 20:21:51,682]\u001b[0m Trial 2 finished with value: 0.8436500127875074 and parameters: {'n_estimators': 1353, 'max_features': 'sqrt', 'max_depth': 36, 'min_samples_split': 5, 'min_samples_leaf': 2, 'bootstrap': False}. Best is trial 0 with value: 0.8556079286377266.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 21:46:09,378]\u001b[0m Trial 3 finished with value: 0.8430227019649611 and parameters: {'n_estimators': 3320, 'max_features': 'sqrt', 'max_depth': 70, 'min_samples_split': 4, 'min_samples_leaf': 2, 'bootstrap': False}. Best is trial 0 with value: 0.8556079286377266.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 23:47:10,169]\u001b[0m Trial 4 finished with value: 0.8459801009782043 and parameters: {'n_estimators': 4678, 'max_features': 'sqrt', 'max_depth': 81, 'min_samples_split': 10, 'min_samples_leaf': 1, 'bootstrap': False}. Best is trial 0 with value: 0.8556079286377266.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-27 00:32:06,456]\u001b[0m Trial 5 finished with value: 0.8559757661796862 and parameters: {'n_estimators': 2919, 'max_features': 'sqrt', 'max_depth': 91, 'min_samples_split': 6, 'min_samples_leaf': 4, 'bootstrap': True}. Best is trial 5 with value: 0.8559757661796862.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-27 02:23:21,749]\u001b[0m Trial 6 finished with value: 0.8435446019454436 and parameters: {'n_estimators': 4301, 'max_features': 'sqrt', 'max_depth': 54, 'min_samples_split': 7, 'min_samples_leaf': 1, 'bootstrap': False}. Best is trial 5 with value: 0.8559757661796862.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_rf = optuna.create_study(direction=\"maximize\")\n",
    "study_rf.optimize(objective_rf_pipe1 ,n_trials=10)\n",
    "# with open('study_rf.pkl', 'wb') as f:\n",
    "#     pickle.dump(study_rf, f)\n",
    "# with open('study_rf.pkl', 'rb') as f:\n",
    "#     study_rf = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17aba6ac",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_rf.trials))\n",
    "print(\"Best Trials : \", study_rf.best_trial.params)\n",
    "print(\"Best Values : \" , study_rf.best_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "081937e7",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "    Trial 0 finished with value: 0.848301163565105 and parameters: {'n_estimators': 1265, 'max_features': 'auto', 'max_depth': 75, 'min_samples_split': 10, 'min_samples_leaf': 4, 'bootstrap': False}. Best is trial 0 with value: 0.848301163565105.\n",
    "\n",
    "    Trial 2 finished with value: 0.855246676422605 and parameters: {'n_estimators': 1006, 'max_features': 'sqrt', 'max_depth': 68, 'min_samples_split': 10, 'min_samples_leaf': 3, 'bootstrap': True}. Best is trial 2 with value: 0.855246676422605.\n",
    "    \n",
    "    \n",
    "     Trial 0 finished with value: 0.8619153033690738 and parameters: {'n_estimators': 3595, 'max_features': 'sqrt', 'max_depth': 18, 'min_samples_split': 8, 'min_samples_leaf': 4, 'bootstrap': False}. Best is trial 0 with value: 0.8619153033690738\n",
    "     \n",
    "     Trial 3 finished with value: 0.8644444679856417 and parameters: {'n_estimators': 1331, 'max_features': 'sqrt', 'max_depth': 18, 'min_samples_split': 10, 'min_samples_leaf': 3, 'bootstrap': True}. Best is trial 3 with value: 0.8644444679856417."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "papermill": {
   "default_parameters": {},
   "duration": null,
   "end_time": null,
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-26T17:37:41.303124",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}