credit-part3-modeling-1-pipe1-tune-lr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d5e13b30",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:23.451173Z",
     "iopub.status.busy": "2021-12-26T11:54:23.449926Z",
     "iopub.status.idle": "2021-12-26T11:54:27.205719Z",
     "shell.execute_reply": "2021-12-26T11:54:27.205074Z",
     "shell.execute_reply.started": "2021-12-26T11:50:33.341216Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.791968,
     "end_time": "2021-12-26T11:54:27.205918",
     "exception": false,
     "start_time": "2021-12-26T11:54:23.413950",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1c4fe93",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.029422,
     "end_time": "2021-12-26T11:54:27.266689",
     "exception": false,
     "start_time": "2021-12-26T11:54:27.237267",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0262f4aa",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:27.331251Z",
     "iopub.status.busy": "2021-12-26T11:54:27.330305Z",
     "iopub.status.idle": "2021-12-26T11:54:27.809661Z",
     "shell.execute_reply": "2021-12-26T11:54:27.809162Z",
     "shell.execute_reply.started": "2021-12-26T11:50:37.075072Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.512581,
     "end_time": "2021-12-26T11:54:27.809843",
     "exception": false,
     "start_time": "2021-12-26T11:54:27.297262",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 243 ms, sys: 83.4 ms, total: 327 ms\n",
      "Wall time: 456 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9afd7612",
   "metadata": {
    "papermill": {
     "duration": 0.030754,
     "end_time": "2021-12-26T11:54:27.871789",
     "exception": false,
     "start_time": "2021-12-26T11:54:27.841035",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "179ee9fe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:27.938168Z",
     "iopub.status.busy": "2021-12-26T11:54:27.937148Z",
     "iopub.status.idle": "2021-12-26T11:54:28.029923Z",
     "shell.execute_reply": "2021-12-26T11:54:28.029378Z",
     "shell.execute_reply.started": "2021-12-26T11:50:37.505810Z"
    },
    "papermill": {
     "duration": 0.127428,
     "end_time": "2021-12-26T11:54:28.030069",
     "exception": false,
     "start_time": "2021-12-26T11:54:27.902641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecef7782",
   "metadata": {
    "papermill": {
     "duration": 0.030704,
     "end_time": "2021-12-26T11:54:28.091876",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.061172",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1988be82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:28.174430Z",
     "iopub.status.busy": "2021-12-26T11:54:28.158316Z",
     "iopub.status.idle": "2021-12-26T11:54:28.221351Z",
     "shell.execute_reply": "2021-12-26T11:54:28.222027Z",
     "shell.execute_reply.started": "2021-12-26T11:50:37.595002Z"
    },
    "papermill": {
     "duration": 0.097809,
     "end_time": "2021-12-26T11:54:28.222212",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.124403",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01ab4188",
   "metadata": {
    "papermill": {
     "duration": 0.031033,
     "end_time": "2021-12-26T11:54:28.284533",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.253500",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# X and Y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "98600760",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:28.350756Z",
     "iopub.status.busy": "2021-12-26T11:54:28.350072Z",
     "iopub.status.idle": "2021-12-26T11:54:28.354672Z",
     "shell.execute_reply": "2021-12-26T11:54:28.355385Z",
     "shell.execute_reply.started": "2021-12-26T11:50:37.658718Z"
    },
    "papermill": {
     "duration": 0.039947,
     "end_time": "2021-12-26T11:54:28.355638",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.315691",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "003f9032",
   "metadata": {
    "papermill": {
     "duration": 0.030896,
     "end_time": "2021-12-26T11:54:28.419137",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.388241",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "55546b0f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:28.484842Z",
     "iopub.status.busy": "2021-12-26T11:54:28.484206Z",
     "iopub.status.idle": "2021-12-26T11:54:28.487086Z",
     "shell.execute_reply": "2021-12-26T11:54:28.487600Z",
     "shell.execute_reply.started": "2021-12-26T11:50:37.666265Z"
    },
    "papermill": {
     "duration": 0.037104,
     "end_time": "2021-12-26T11:54:28.487769",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.450665",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0c1956f",
   "metadata": {
    "papermill": {
     "duration": 0.03121,
     "end_time": "2021-12-26T11:54:28.549851",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.518641",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Impute NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "adbd6494",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:28.616003Z",
     "iopub.status.busy": "2021-12-26T11:54:28.615327Z",
     "iopub.status.idle": "2021-12-26T11:54:28.625357Z",
     "shell.execute_reply": "2021-12-26T11:54:28.626057Z",
     "shell.execute_reply.started": "2021-12-26T11:50:37.683979Z"
    },
    "papermill": {
     "duration": 0.045561,
     "end_time": "2021-12-26T11:54:28.626274",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.580713",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 357 µs, sys: 0 ns, total: 357 µs\n",
      "Wall time: 370 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            'early_stopping_rounds':100 ,\n",
    "           # 'gpu_id':0, \n",
    "           # 'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "            # 'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "534d4733",
   "metadata": {
    "papermill": {
     "duration": 0.030829,
     "end_time": "2021-12-26T11:54:28.689357",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.658528",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# XGBR regression imputer \n",
    "## Predict income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e91f0c5a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T11:54:28.754657Z",
     "iopub.status.busy": "2021-12-26T11:54:28.754011Z",
     "iopub.status.idle": "2021-12-26T12:13:26.269027Z",
     "shell.execute_reply": "2021-12-26T12:13:26.270039Z"
    },
    "papermill": {
     "duration": 1137.549872,
     "end_time": "2021-12-26T12:13:26.270358",
     "exception": false,
     "start_time": "2021-12-26T11:54:28.720486",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 14min 1s, sys: 5.89 s, total: 1h 14min 6s\n",
      "Wall time: 18min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "25e9ce8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:13:26.338971Z",
     "iopub.status.busy": "2021-12-26T12:13:26.338329Z",
     "iopub.status.idle": "2021-12-26T12:13:26.355258Z",
     "shell.execute_reply": "2021-12-26T12:13:26.356295Z"
    },
    "papermill": {
     "duration": 0.052793,
     "end_time": "2021-12-26T12:13:26.356570",
     "exception": false,
     "start_time": "2021-12-26T12:13:26.303777",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    146076 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f2951382",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:13:26.427369Z",
     "iopub.status.busy": "2021-12-26T12:13:26.426723Z",
     "iopub.status.idle": "2021-12-26T12:13:26.440503Z",
     "shell.execute_reply": "2021-12-26T12:13:26.441100Z"
    },
    "papermill": {
     "duration": 0.049391,
     "end_time": "2021-12-26T12:13:26.441277",
     "exception": false,
     "start_time": "2021-12-26T12:13:26.391886",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    98877 non-null   float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53cbad1b",
   "metadata": {
    "papermill": {
     "duration": 0.034505,
     "end_time": "2021-12-26T12:13:26.508705",
     "exception": false,
     "start_time": "2021-12-26T12:13:26.474200",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict number of dependents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8b0c9d64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:13:26.578596Z",
     "iopub.status.busy": "2021-12-26T12:13:26.577977Z",
     "iopub.status.idle": "2021-12-26T12:36:16.512274Z",
     "shell.execute_reply": "2021-12-26T12:36:16.512791Z"
    },
    "papermill": {
     "duration": 1369.970764,
     "end_time": "2021-12-26T12:36:16.512979",
     "exception": false,
     "start_time": "2021-12-26T12:13:26.542215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 29min 16s, sys: 5.25 s, total: 1h 29min 21s\n",
      "Wall time: 22min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "66c9e5eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:16.582685Z",
     "iopub.status.busy": "2021-12-26T12:36:16.582084Z",
     "iopub.status.idle": "2021-12-26T12:36:16.597001Z",
     "shell.execute_reply": "2021-12-26T12:36:16.596480Z"
    },
    "papermill": {
     "duration": 0.050821,
     "end_time": "2021-12-26T12:36:16.597143",
     "exception": false,
     "start_time": "2021-12-26T12:36:16.546322",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    150000 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0530569f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:16.668619Z",
     "iopub.status.busy": "2021-12-26T12:36:16.668019Z",
     "iopub.status.idle": "2021-12-26T12:36:16.682244Z",
     "shell.execute_reply": "2021-12-26T12:36:16.681585Z"
    },
    "papermill": {
     "duration": 0.0513,
     "end_time": "2021-12-26T12:36:16.682383",
     "exception": false,
     "start_time": "2021-12-26T12:36:16.631083",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    101503 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f745807a",
   "metadata": {
    "papermill": {
     "duration": 0.033771,
     "end_time": "2021-12-26T12:36:16.750267",
     "exception": false,
     "start_time": "2021-12-26T12:36:16.716496",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Add bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6060ead5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:16.821368Z",
     "iopub.status.busy": "2021-12-26T12:36:16.820758Z",
     "iopub.status.idle": "2021-12-26T12:36:17.509166Z",
     "shell.execute_reply": "2021-12-26T12:36:17.508597Z"
    },
    "papermill": {
     "duration": 0.725064,
     "end_time": "2021-12-26T12:36:17.509302",
     "exception": false,
     "start_time": "2021-12-26T12:36:16.784238",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) \n",
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f587a422",
   "metadata": {
    "papermill": {
     "duration": 0.033715,
     "end_time": "2021-12-26T12:36:17.577653",
     "exception": false,
     "start_time": "2021-12-26T12:36:17.543938",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ed24d16c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:17.648938Z",
     "iopub.status.busy": "2021-12-26T12:36:17.648271Z",
     "iopub.status.idle": "2021-12-26T12:36:17.651542Z",
     "shell.execute_reply": "2021-12-26T12:36:17.652156Z"
    },
    "papermill": {
     "duration": 0.040662,
     "end_time": "2021-12-26T12:36:17.652318",
     "exception": false,
     "start_time": "2021-12-26T12:36:17.611656",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ca7e5fc7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:17.724535Z",
     "iopub.status.busy": "2021-12-26T12:36:17.723925Z",
     "iopub.status.idle": "2021-12-26T12:36:17.749594Z",
     "shell.execute_reply": "2021-12-26T12:36:17.750136Z"
    },
    "papermill": {
     "duration": 0.063494,
     "end_time": "2021-12-26T12:36:17.750303",
     "exception": false,
     "start_time": "2021-12-26T12:36:17.686809",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "24999eb2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:17.824553Z",
     "iopub.status.busy": "2021-12-26T12:36:17.823951Z",
     "iopub.status.idle": "2021-12-26T12:36:17.849321Z",
     "shell.execute_reply": "2021-12-26T12:36:17.848633Z"
    },
    "papermill": {
     "duration": 0.064204,
     "end_time": "2021-12-26T12:36:17.849461",
     "exception": false,
     "start_time": "2021-12-26T12:36:17.785257",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7377cee4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:17.927402Z",
     "iopub.status.busy": "2021-12-26T12:36:17.926373Z",
     "iopub.status.idle": "2021-12-26T12:36:17.928605Z",
     "shell.execute_reply": "2021-12-26T12:36:17.929146Z"
    },
    "papermill": {
     "duration": 0.044196,
     "end_time": "2021-12-26T12:36:17.929316",
     "exception": false,
     "start_time": "2021-12-26T12:36:17.885120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a18310c5",
   "metadata": {
    "papermill": {
     "duration": 0.035386,
     "end_time": "2021-12-26T12:36:18.000896",
     "exception": false,
     "start_time": "2021-12-26T12:36:17.965510",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extract final cat  and num Features \n",
    "## Cat Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f31fc0c9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.082559Z",
     "iopub.status.busy": "2021-12-26T12:36:18.081896Z",
     "iopub.status.idle": "2021-12-26T12:36:18.086668Z",
     "shell.execute_reply": "2021-12-26T12:36:18.087282Z"
    },
    "papermill": {
     "duration": 0.050893,
     "end_time": "2021-12-26T12:36:18.087449",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.036556",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1727fb6",
   "metadata": {
    "papermill": {
     "duration": 0.035648,
     "end_time": "2021-12-26T12:36:18.159324",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.123676",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1ad591b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.234618Z",
     "iopub.status.busy": "2021-12-26T12:36:18.233941Z",
     "iopub.status.idle": "2021-12-26T12:36:18.246138Z",
     "shell.execute_reply": "2021-12-26T12:36:18.246756Z"
    },
    "papermill": {
     "duration": 0.051702,
     "end_time": "2021-12-26T12:36:18.246928",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.195226",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38944033",
   "metadata": {
    "papermill": {
     "duration": 0.037325,
     "end_time": "2021-12-26T12:36:18.320923",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.283598",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2eedba71",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.402568Z",
     "iopub.status.busy": "2021-12-26T12:36:18.401904Z",
     "iopub.status.idle": "2021-12-26T12:36:18.410161Z",
     "shell.execute_reply": "2021-12-26T12:36:18.409634Z"
    },
    "papermill": {
     "duration": 0.0528,
     "end_time": "2021-12-26T12:36:18.410296",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.357496",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeaturesEngineer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X=X.copy()\n",
    "        # Calculate some metrics across rows\n",
    "        X[\"num_missing\"]  = X.isnull().sum(axis=1)\n",
    "        X[\"std_row\"] = X.std(axis=1)\n",
    "        X[\"sem_row\"] = X.sem(axis=1)\n",
    "        X[\"abs_sum_row\"] = X.abs().sum(axis=1)\n",
    "        X[\"mean_row\"] = X.mean(axis=1)\n",
    "        X[\"max_row\"]= X.max(axis=1)\n",
    "        X[\"min_row\"]= X.min(axis=1)\n",
    "        X['Weighted_Sum_PastDue'] = 2 * X['NumberOfTime30-59DaysPastDueNotWorse'] + 3 * X['NumberOfTime60-89DaysPastDueNotWorse'] + 6 * X['NumberOfTimes90DaysLate']\n",
    "        X['90days_out_of_TotalPastDue'] = X['NumberOfTimes90DaysLate'] / (X['Weighted_Sum_PastDue'])\n",
    "        X.loc[X['Weighted_Sum_PastDue']==0, '90days_out_of_TotalPastDue'] = 0\n",
    "        X['RemainingLines'] = X['NumberOfOpenCreditLinesAndLoans'] - X['NumberRealEstateLoansOrLines']\n",
    "        X['Loans_vs_Other_Lines'] = X['RemainingLines'] / (1 + X['NumberRealEstateLoansOrLines'])\n",
    "        X['Debt_per_Real_Estate_Loan'] = X['DebtRatio'] / X['NumberRealEstateLoansOrLines']\n",
    "        X.loc[X['NumberRealEstateLoansOrLines']==0, 'Debt_per_Real_Estate_Loan'] = 0\n",
    "        X['Disposable_Income_per_person'] = (X['MonthlyIncome'] - X['DebtRatio']) / (X['NumberOfDependents'] + 1)\n",
    "        X.loc[X['NumberOfDependents']==0, 'Disposable_Income_per_person'] = 0\n",
    "        X['RemainingLines_per_person'] = X['RemainingLines'] / (X['NumberOfDependents'] + 1)\n",
    "        X['NumberRE_X_DebtRatio_X_age'] = X['NumberRealEstateLoansOrLines'] * X['DebtRatio'] / X['age']\n",
    "        X['RevolvingUtilization_per_age'] = X['RevolvingUtilizationOfUnsecuredLines'] / X['age']\n",
    "        X[X==np.inf]=np.nan\n",
    "        X=reduce_mem_usage(X)\n",
    "        return X  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "19a3e940",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.493855Z",
     "iopub.status.busy": "2021-12-26T12:36:18.487134Z",
     "iopub.status.idle": "2021-12-26T12:36:18.496577Z",
     "shell.execute_reply": "2021-12-26T12:36:18.495955Z"
    },
    "papermill": {
     "duration": 0.049321,
     "end_time": "2021-12-26T12:36:18.496715",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.447394",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e13294dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.579333Z",
     "iopub.status.busy": "2021-12-26T12:36:18.578556Z",
     "iopub.status.idle": "2021-12-26T12:36:18.581279Z",
     "shell.execute_reply": "2021-12-26T12:36:18.580790Z"
    },
    "papermill": {
     "duration": 0.047357,
     "end_time": "2021-12-26T12:36:18.581418",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.534061",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "               # ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7833b333",
   "metadata": {
    "papermill": {
     "duration": 0.036539,
     "end_time": "2021-12-26T12:36:18.654914",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.618375",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# First pipe to tune : \n",
    "imputer + transformer + kmeans fe+ploy fe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a33d4790",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.733359Z",
     "iopub.status.busy": "2021-12-26T12:36:18.732435Z",
     "iopub.status.idle": "2021-12-26T12:36:18.810292Z",
     "shell.execute_reply": "2021-12-26T12:36:18.810878Z"
    },
    "papermill": {
     "duration": 0.119454,
     "end_time": "2021-12-26T12:36:18.811049",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.691595",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"28946013-7d9f-403e-975e-56dab010f7ef\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"28946013-7d9f-403e-975e-56dab010f7ef\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"73f4cdd8-8346-4a6b-810c-b21e6ebb0c7c\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"73f4cdd8-8346-4a6b-810c-b21e6ebb0c7c\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"dd5d0115-90e3-4148-9a78-c52973fd412f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"dd5d0115-90e3-4148-9a78-c52973fd412f\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"f6dda3a6-00f2-4497-8f74-8bd48d91bb39\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"f6dda3a6-00f2-4497-8f74-8bd48d91bb39\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ddc2eb1d-fda3-46f9-9896-cd330471a0d6\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ddc2eb1d-fda3-46f9-9896-cd330471a0d6\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c2baf2d1-077c-4aac-9355-956141a965d4\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c2baf2d1-077c-4aac-9355-956141a965d4\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b2df608d-665e-45d9-8c7a-e1ee214ddeb7\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b2df608d-665e-45d9-8c7a-e1ee214ddeb7\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9a64a053",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:18.894098Z",
     "iopub.status.busy": "2021-12-26T12:36:18.893460Z",
     "iopub.status.idle": "2021-12-26T12:36:23.275472Z",
     "shell.execute_reply": "2021-12-26T12:36:23.275947Z"
    },
    "papermill": {
     "duration": 4.426967,
     "end_time": "2021-12-26T12:36:23.276127",
     "exception": false,
     "start_time": "2021-12-26T12:36:18.849160",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 78)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer.fit(X,y)\n",
    "X_pre=preprocessor_woe_powertransformer.transform(X)\n",
    "X_pre.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6139fd51",
   "metadata": {
    "papermill": {
     "duration": 0.038078,
     "end_time": "2021-12-26T12:36:23.353107",
     "exception": false,
     "start_time": "2021-12-26T12:36:23.315029",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "97548c49",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:23.436513Z",
     "iopub.status.busy": "2021-12-26T12:36:23.435629Z",
     "iopub.status.idle": "2021-12-26T12:36:23.439608Z",
     "shell.execute_reply": "2021-12-26T12:36:23.439080Z"
    },
    "papermill": {
     "duration": 0.048458,
     "end_time": "2021-12-26T12:36:23.439838",
     "exception": false,
     "start_time": "2021-12-26T12:36:23.391380",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StratifiedKFold(n_splits=5, random_state=1, shuffle=True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N_FOLD = 5\n",
    "cross_validation_design = StratifiedKFold(n_splits=N_FOLD,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8483775",
   "metadata": {
    "papermill": {
     "duration": 0.038203,
     "end_time": "2021-12-26T12:36:23.517024",
     "exception": false,
     "start_time": "2021-12-26T12:36:23.478821",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f8512add",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:23.597980Z",
     "iopub.status.busy": "2021-12-26T12:36:23.597000Z",
     "iopub.status.idle": "2021-12-26T12:36:23.604657Z",
     "shell.execute_reply": "2021-12-26T12:36:23.605126Z"
    },
    "papermill": {
     "duration": 0.04952,
     "end_time": "2021-12-26T12:36:23.605295",
     "exception": false,
     "start_time": "2021-12-26T12:36:23.555775",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective_lr_pipe1(trial,X=X_pre,y=y):\n",
    "    # https://www.kaggle.com/hamidrezabakhtaki/xgboost-catboost-lighgbm-optuna-final-submission\n",
    "    # https://www.kaggle.com/prashant111/lightgbm-classifier-in-python\n",
    "    # https://www.kaggle.com/tunguz/tps-09-21-histgradientboosting-with-optuna\n",
    "    # https://github.com/solegalli/optuna-examples/tree/main/catboost\n",
    "    # https://www.kaggle.com/dixhom/bayesian-optimization-with-optuna-stacking\n",
    "    X_train,X_test, y_train,  y_test = train_test_split(X, y,stratify=y, \n",
    "                                                       test_size=0.25,\n",
    "                                                      random_state=42,\n",
    "                                                        shuffle=True)\n",
    "    param = { \n",
    "        \"random_state\": 42,\n",
    "        'solver':'liblinear',\n",
    "        \"C\": trial.suggest_loguniform(\"C\", 2 ** -10, 2 ** 15),\n",
    "        \"penalty\": trial.suggest_categorical('penalty', ['l2', 'l1']),\n",
    "       # \"regularization\":trial.suggest_uniform('logistic-regularization', 0.01, 10)\n",
    "        \n",
    "    }\n",
    "    #if  param[\"penalty\"]  == 'l1':\n",
    "     #   param[\"solver\"] =  'saga'\n",
    "    #else:\n",
    "       # param[\"solver\"] = 'lbfgs'\n",
    "    \n",
    "    model= LogisticRegression(**param,\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             )\n",
    "    model.fit( X_train, y_train,\n",
    "             # eval_set=[(X_test, y_test)],\n",
    "             # early_stopping_rounds=100,\n",
    "             # verbose=False\n",
    "             )\n",
    "    preds = model.predict_proba(X_test)[:,1]\n",
    "    final_auc= roc_auc_score(y_true=y_test, y_score=preds)   \n",
    "    return final_auc               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "03d8d0c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T12:36:23.687977Z",
     "iopub.status.busy": "2021-12-26T12:36:23.687350Z",
     "iopub.status.idle": "2021-12-26T14:15:52.225231Z",
     "shell.execute_reply": "2021-12-26T14:15:52.224251Z"
    },
    "papermill": {
     "duration": 5968.581535,
     "end_time": "2021-12-26T14:15:52.225474",
     "exception": false,
     "start_time": "2021-12-26T12:36:23.643939",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-26 12:36:23,685]\u001b[0m A new study created in memory with name: no-name-9d222a36-a649-4386-ab28-021162ff7231\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:39:48,046]\u001b[0m Trial 0 finished with value: 0.8635650902371087 and parameters: {'C': 0.13008485717466153, 'penalty': 'l1'}. Best is trial 0 with value: 0.8635650902371087.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:39:56,798]\u001b[0m Trial 1 finished with value: 0.8633610306288513 and parameters: {'C': 0.04776132373823716, 'penalty': 'l2'}. Best is trial 0 with value: 0.8635650902371087.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:40:19,497]\u001b[0m Trial 2 finished with value: 0.8639989121838285 and parameters: {'C': 0.7873048727824917, 'penalty': 'l2'}. Best is trial 2 with value: 0.8639989121838285.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:41:25,092]\u001b[0m Trial 3 finished with value: 0.8645600960620727 and parameters: {'C': 13005.075128755812, 'penalty': 'l2'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:42:55,547]\u001b[0m Trial 4 finished with value: 0.8645566066940857 and parameters: {'C': 2855.1476631191067, 'penalty': 'l2'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:44:30,287]\u001b[0m Trial 5 finished with value: 0.8635597193471679 and parameters: {'C': 0.05153171179006784, 'penalty': 'l1'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 12:46:17,171]\u001b[0m Trial 6 finished with value: 0.8645592522279842 and parameters: {'C': 619.1164811819291, 'penalty': 'l2'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 13:09:59,247]\u001b[0m Trial 7 finished with value: 0.8644649708733558 and parameters: {'C': 473.5077797665874, 'penalty': 'l1'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 13:41:11,806]\u001b[0m Trial 8 finished with value: 0.8644893280302848 and parameters: {'C': 1.7375054148024474, 'penalty': 'l1'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:05:01,930]\u001b[0m Trial 9 finished with value: 0.8644650278891729 and parameters: {'C': 1104.7577186970188, 'penalty': 'l1'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:06:47,752]\u001b[0m Trial 10 finished with value: 0.8645600960620726 and parameters: {'C': 15655.44959010592, 'penalty': 'l2'}. Best is trial 3 with value: 0.8645600960620727.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:07:53,959]\u001b[0m Trial 11 finished with value: 0.864560586398097 and parameters: {'C': 22934.021606965187, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:09:30,673]\u001b[0m Trial 12 finished with value: 0.8645498218118888 and parameters: {'C': 30.32489649025664, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:11:00,024]\u001b[0m Trial 13 finished with value: 0.8643469139231301 and parameters: {'C': 20.28464976887166, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:11:04,136]\u001b[0m Trial 14 finished with value: 0.8551919412384956 and parameters: {'C': 0.0017834833270246335, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:11:51,391]\u001b[0m Trial 15 finished with value: 0.8645602785126864 and parameters: {'C': 22467.25855926268, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:13:38,511]\u001b[0m Trial 16 finished with value: 0.8645590697773705 and parameters: {'C': 30205.496184598986, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:14:23,119]\u001b[0m Trial 17 finished with value: 0.8645581803306288 and parameters: {'C': 106.18808080479776, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:15:05,393]\u001b[0m Trial 18 finished with value: 0.8645603469316665 and parameters: {'C': 2953.48684189889, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 14:15:52,132]\u001b[0m Trial 19 finished with value: 0.864559982030439 and parameters: {'C': 3047.2617179058225, 'penalty': 'l2'}. Best is trial 11 with value: 0.864560586398097.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 39min 29s, sys: 4.83 s, total: 1h 39min 33s\n",
      "Wall time: 1h 39min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_lr_pipe1 = optuna.create_study(direction=\"maximize\")\n",
    "study_lr_pipe1.optimize(objective_lr_pipe1 ,n_trials=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b775b514",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T14:15:52.327914Z",
     "iopub.status.busy": "2021-12-26T14:15:52.327077Z",
     "iopub.status.idle": "2021-12-26T14:15:52.331946Z",
     "shell.execute_reply": "2021-12-26T14:15:52.331401Z"
    },
    "papermill": {
     "duration": 0.057756,
     "end_time": "2021-12-26T14:15:52.332096",
     "exception": false,
     "start_time": "2021-12-26T14:15:52.274340",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of finished trials :  20\n",
      "Best Trials :  {'C': 22934.021606965187, 'penalty': 'l2'}\n",
      "Best Values :  0.864560586398097\n"
     ]
    }
   ],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_lr_pipe1.trials))\n",
    "print(\"Best Trials : \", study_lr_pipe1.best_trial.params)\n",
    "print(\"Best Values : \" , study_lr_pipe1.best_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a1c2744",
   "metadata": {
    "papermill": {
     "duration": 0.045933,
     "end_time": "2021-12-26T14:15:52.424603",
     "exception": false,
     "start_time": "2021-12-26T14:15:52.378670",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    " Trial 0 finished with value: 0.8637496504360272 and parameters: {'C': 0.372623733011658, 'penalty': 'l2'}. Best is trial 0 with value: 0.8637496504360272."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 8499.847673,
   "end_time": "2021-12-26T14:15:53.500109",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-26T11:54:13.652436",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}