credit-part3-modeling-1-pipe1-tune-hsbc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "19441b4e",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:53.252197Z",
     "iopub.status.busy": "2021-12-25T21:06:53.250978Z",
     "iopub.status.idle": "2021-12-25T21:06:57.139820Z",
     "shell.execute_reply": "2021-12-25T21:06:57.138862Z",
     "shell.execute_reply.started": "2021-12-25T21:02:58.358363Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.924067,
     "end_time": "2021-12-25T21:06:57.140013",
     "exception": false,
     "start_time": "2021-12-25T21:06:53.215946",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c63e8ef",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.030837,
     "end_time": "2021-12-25T21:06:57.204816",
     "exception": false,
     "start_time": "2021-12-25T21:06:57.173979",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "db82e441",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:57.273424Z",
     "iopub.status.busy": "2021-12-25T21:06:57.272741Z",
     "iopub.status.idle": "2021-12-25T21:06:57.734157Z",
     "shell.execute_reply": "2021-12-25T21:06:57.734675Z",
     "shell.execute_reply.started": "2021-12-25T21:03:02.927456Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.498803,
     "end_time": "2021-12-25T21:06:57.734852",
     "exception": false,
     "start_time": "2021-12-25T21:06:57.236049",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 254 ms, sys: 76.3 ms, total: 330 ms\n",
      "Wall time: 442 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61b0d421",
   "metadata": {
    "papermill": {
     "duration": 0.032041,
     "end_time": "2021-12-25T21:06:57.799913",
     "exception": false,
     "start_time": "2021-12-25T21:06:57.767872",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c858620",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:57.867366Z",
     "iopub.status.busy": "2021-12-25T21:06:57.866716Z",
     "iopub.status.idle": "2021-12-25T21:06:57.963121Z",
     "shell.execute_reply": "2021-12-25T21:06:57.962005Z",
     "shell.execute_reply.started": "2021-12-25T21:03:03.522220Z"
    },
    "papermill": {
     "duration": 0.131282,
     "end_time": "2021-12-25T21:06:57.963275",
     "exception": false,
     "start_time": "2021-12-25T21:06:57.831993",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f104453",
   "metadata": {
    "papermill": {
     "duration": 0.033101,
     "end_time": "2021-12-25T21:06:58.030444",
     "exception": false,
     "start_time": "2021-12-25T21:06:57.997343",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d5bcf39e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:58.116386Z",
     "iopub.status.busy": "2021-12-25T21:06:58.114668Z",
     "iopub.status.idle": "2021-12-25T21:06:58.164130Z",
     "shell.execute_reply": "2021-12-25T21:06:58.164640Z",
     "shell.execute_reply.started": "2021-12-25T21:03:03.656884Z"
    },
    "papermill": {
     "duration": 0.101882,
     "end_time": "2021-12-25T21:06:58.164822",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.062940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af5e73c4",
   "metadata": {
    "papermill": {
     "duration": 0.031797,
     "end_time": "2021-12-25T21:06:58.230057",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.198260",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# X and Y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "254fe3a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:58.298917Z",
     "iopub.status.busy": "2021-12-25T21:06:58.298230Z",
     "iopub.status.idle": "2021-12-25T21:06:58.303477Z",
     "shell.execute_reply": "2021-12-25T21:06:58.303991Z",
     "shell.execute_reply.started": "2021-12-25T21:03:03.747427Z"
    },
    "papermill": {
     "duration": 0.041806,
     "end_time": "2021-12-25T21:06:58.304156",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.262350",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03f65baa",
   "metadata": {
    "papermill": {
     "duration": 0.031771,
     "end_time": "2021-12-25T21:06:58.368006",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.336235",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fcb0f021",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:58.435544Z",
     "iopub.status.busy": "2021-12-25T21:06:58.434926Z",
     "iopub.status.idle": "2021-12-25T21:06:58.439166Z",
     "shell.execute_reply": "2021-12-25T21:06:58.438639Z",
     "shell.execute_reply.started": "2021-12-25T21:03:03.757282Z"
    },
    "papermill": {
     "duration": 0.039017,
     "end_time": "2021-12-25T21:06:58.439295",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.400278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef60d3c",
   "metadata": {
    "papermill": {
     "duration": 0.033348,
     "end_time": "2021-12-25T21:06:58.504610",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.471262",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Impute NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f183c52d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:58.594748Z",
     "iopub.status.busy": "2021-12-25T21:06:58.587801Z",
     "iopub.status.idle": "2021-12-25T21:06:58.598570Z",
     "shell.execute_reply": "2021-12-25T21:06:58.599172Z",
     "shell.execute_reply.started": "2021-12-25T21:03:03.780183Z"
    },
    "papermill": {
     "duration": 0.05781,
     "end_time": "2021-12-25T21:06:58.599337",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.541527",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 396 µs, sys: 15 µs, total: 411 µs\n",
      "Wall time: 417 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            'early_stopping_rounds':100 ,\n",
    "           # 'gpu_id':0, \n",
    "           # 'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "            # 'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57888996",
   "metadata": {
    "papermill": {
     "duration": 0.033265,
     "end_time": "2021-12-25T21:06:58.669052",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.635787",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# XGBR regression imputer \n",
    "## Predict income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e356d757",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:06:58.743918Z",
     "iopub.status.busy": "2021-12-25T21:06:58.742913Z",
     "iopub.status.idle": "2021-12-25T21:26:04.049973Z",
     "shell.execute_reply": "2021-12-25T21:26:04.050678Z"
    },
    "papermill": {
     "duration": 1145.346931,
     "end_time": "2021-12-25T21:26:04.051060",
     "exception": false,
     "start_time": "2021-12-25T21:06:58.704129",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 14min 18s, sys: 6.92 s, total: 1h 14min 25s\n",
      "Wall time: 19min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a3f64ad7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:26:04.128902Z",
     "iopub.status.busy": "2021-12-25T21:26:04.128128Z",
     "iopub.status.idle": "2021-12-25T21:26:04.145504Z",
     "shell.execute_reply": "2021-12-25T21:26:04.144885Z"
    },
    "papermill": {
     "duration": 0.059451,
     "end_time": "2021-12-25T21:26:04.145651",
     "exception": false,
     "start_time": "2021-12-25T21:26:04.086200",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    146076 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d146ca48",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:26:04.220028Z",
     "iopub.status.busy": "2021-12-25T21:26:04.219325Z",
     "iopub.status.idle": "2021-12-25T21:26:04.233863Z",
     "shell.execute_reply": "2021-12-25T21:26:04.233299Z"
    },
    "papermill": {
     "duration": 0.054189,
     "end_time": "2021-12-25T21:26:04.234002",
     "exception": false,
     "start_time": "2021-12-25T21:26:04.179813",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    98877 non-null   float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4793e59c",
   "metadata": {
    "papermill": {
     "duration": 0.033227,
     "end_time": "2021-12-25T21:26:04.301113",
     "exception": false,
     "start_time": "2021-12-25T21:26:04.267886",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict number of dependents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "22d443c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:26:04.377758Z",
     "iopub.status.busy": "2021-12-25T21:26:04.377019Z",
     "iopub.status.idle": "2021-12-25T21:49:04.852587Z",
     "shell.execute_reply": "2021-12-25T21:49:04.853115Z"
    },
    "papermill": {
     "duration": 1380.517721,
     "end_time": "2021-12-25T21:49:04.853294",
     "exception": false,
     "start_time": "2021-12-25T21:26:04.335573",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 29min 51s, sys: 5.57 s, total: 1h 29min 57s\n",
      "Wall time: 23min\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "629d5c22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:04.929766Z",
     "iopub.status.busy": "2021-12-25T21:49:04.928801Z",
     "iopub.status.idle": "2021-12-25T21:49:04.941673Z",
     "shell.execute_reply": "2021-12-25T21:49:04.942450Z"
    },
    "papermill": {
     "duration": 0.054875,
     "end_time": "2021-12-25T21:49:04.942678",
     "exception": false,
     "start_time": "2021-12-25T21:49:04.887803",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    150000 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ac70d13d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:05.017366Z",
     "iopub.status.busy": "2021-12-25T21:49:05.016749Z",
     "iopub.status.idle": "2021-12-25T21:49:05.030237Z",
     "shell.execute_reply": "2021-12-25T21:49:05.030948Z"
    },
    "papermill": {
     "duration": 0.051626,
     "end_time": "2021-12-25T21:49:05.031166",
     "exception": false,
     "start_time": "2021-12-25T21:49:04.979540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    101503 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3e00d6f",
   "metadata": {
    "papermill": {
     "duration": 0.035242,
     "end_time": "2021-12-25T21:49:05.102616",
     "exception": false,
     "start_time": "2021-12-25T21:49:05.067374",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Add bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d5a7e57b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:05.177592Z",
     "iopub.status.busy": "2021-12-25T21:49:05.176920Z",
     "iopub.status.idle": "2021-12-25T21:49:05.875015Z",
     "shell.execute_reply": "2021-12-25T21:49:05.874459Z"
    },
    "papermill": {
     "duration": 0.736373,
     "end_time": "2021-12-25T21:49:05.875158",
     "exception": false,
     "start_time": "2021-12-25T21:49:05.138785",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) \n",
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06584d97",
   "metadata": {
    "papermill": {
     "duration": 0.035292,
     "end_time": "2021-12-25T21:49:05.945919",
     "exception": false,
     "start_time": "2021-12-25T21:49:05.910627",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "19870d1d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.021791Z",
     "iopub.status.busy": "2021-12-25T21:49:06.021114Z",
     "iopub.status.idle": "2021-12-25T21:49:06.024692Z",
     "shell.execute_reply": "2021-12-25T21:49:06.024159Z"
    },
    "papermill": {
     "duration": 0.043349,
     "end_time": "2021-12-25T21:49:06.024841",
     "exception": false,
     "start_time": "2021-12-25T21:49:05.981492",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1828e9bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.099893Z",
     "iopub.status.busy": "2021-12-25T21:49:06.098972Z",
     "iopub.status.idle": "2021-12-25T21:49:06.123971Z",
     "shell.execute_reply": "2021-12-25T21:49:06.124608Z"
    },
    "papermill": {
     "duration": 0.064078,
     "end_time": "2021-12-25T21:49:06.124784",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.060706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "13d82681",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.200044Z",
     "iopub.status.busy": "2021-12-25T21:49:06.199143Z",
     "iopub.status.idle": "2021-12-25T21:49:06.224297Z",
     "shell.execute_reply": "2021-12-25T21:49:06.223794Z"
    },
    "papermill": {
     "duration": 0.063959,
     "end_time": "2021-12-25T21:49:06.224460",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.160501",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8f555725",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.301439Z",
     "iopub.status.busy": "2021-12-25T21:49:06.300807Z",
     "iopub.status.idle": "2021-12-25T21:49:06.304268Z",
     "shell.execute_reply": "2021-12-25T21:49:06.304818Z"
    },
    "papermill": {
     "duration": 0.043786,
     "end_time": "2021-12-25T21:49:06.304977",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.261191",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8f7c1dd",
   "metadata": {
    "papermill": {
     "duration": 0.036921,
     "end_time": "2021-12-25T21:49:06.380799",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.343878",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extract final cat  and num Features \n",
    "## Cat Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "61c2285c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.457182Z",
     "iopub.status.busy": "2021-12-25T21:49:06.456575Z",
     "iopub.status.idle": "2021-12-25T21:49:06.468547Z",
     "shell.execute_reply": "2021-12-25T21:49:06.469118Z"
    },
    "papermill": {
     "duration": 0.051569,
     "end_time": "2021-12-25T21:49:06.469278",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.417709",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e234e569",
   "metadata": {
    "papermill": {
     "duration": 0.037345,
     "end_time": "2021-12-25T21:49:06.544768",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.507423",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a266b242",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.622472Z",
     "iopub.status.busy": "2021-12-25T21:49:06.621840Z",
     "iopub.status.idle": "2021-12-25T21:49:06.632954Z",
     "shell.execute_reply": "2021-12-25T21:49:06.633556Z"
    },
    "papermill": {
     "duration": 0.051529,
     "end_time": "2021-12-25T21:49:06.633731",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.582202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8842ef63",
   "metadata": {
    "papermill": {
     "duration": 0.037524,
     "end_time": "2021-12-25T21:49:06.708796",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.671272",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e90732a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.787324Z",
     "iopub.status.busy": "2021-12-25T21:49:06.786533Z",
     "iopub.status.idle": "2021-12-25T21:49:06.799345Z",
     "shell.execute_reply": "2021-12-25T21:49:06.799899Z"
    },
    "papermill": {
     "duration": 0.053684,
     "end_time": "2021-12-25T21:49:06.800064",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.746380",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeaturesEngineer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X=X.copy()\n",
    "        # Calculate some metrics across rows\n",
    "        X[\"num_missing\"]  = X.isnull().sum(axis=1)\n",
    "        X[\"std_row\"] = X.std(axis=1)\n",
    "        X[\"sem_row\"] = X.sem(axis=1)\n",
    "        X[\"abs_sum_row\"] = X.abs().sum(axis=1)\n",
    "        X[\"mean_row\"] = X.mean(axis=1)\n",
    "        X[\"max_row\"]= X.max(axis=1)\n",
    "        X[\"min_row\"]= X.min(axis=1)\n",
    "        X['Weighted_Sum_PastDue'] = 2 * X['NumberOfTime30-59DaysPastDueNotWorse'] + 3 * X['NumberOfTime60-89DaysPastDueNotWorse'] + 6 * X['NumberOfTimes90DaysLate']\n",
    "        X['90days_out_of_TotalPastDue'] = X['NumberOfTimes90DaysLate'] / (X['Weighted_Sum_PastDue'])\n",
    "        X.loc[X['Weighted_Sum_PastDue']==0, '90days_out_of_TotalPastDue'] = 0\n",
    "        X['RemainingLines'] = X['NumberOfOpenCreditLinesAndLoans'] - X['NumberRealEstateLoansOrLines']\n",
    "        X['Loans_vs_Other_Lines'] = X['RemainingLines'] / (1 + X['NumberRealEstateLoansOrLines'])\n",
    "        X['Debt_per_Real_Estate_Loan'] = X['DebtRatio'] / X['NumberRealEstateLoansOrLines']\n",
    "        X.loc[X['NumberRealEstateLoansOrLines']==0, 'Debt_per_Real_Estate_Loan'] = 0\n",
    "        X['Disposable_Income_per_person'] = (X['MonthlyIncome'] - X['DebtRatio']) / (X['NumberOfDependents'] + 1)\n",
    "        X.loc[X['NumberOfDependents']==0, 'Disposable_Income_per_person'] = 0\n",
    "        X['RemainingLines_per_person'] = X['RemainingLines'] / (X['NumberOfDependents'] + 1)\n",
    "        X['NumberRE_X_DebtRatio_X_age'] = X['NumberRealEstateLoansOrLines'] * X['DebtRatio'] / X['age']\n",
    "        X['RevolvingUtilization_per_age'] = X['RevolvingUtilizationOfUnsecuredLines'] / X['age']\n",
    "        X[X==np.inf]=np.nan\n",
    "        X=reduce_mem_usage(X)\n",
    "        return X  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f1106500",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.879088Z",
     "iopub.status.busy": "2021-12-25T21:49:06.878466Z",
     "iopub.status.idle": "2021-12-25T21:49:06.886751Z",
     "shell.execute_reply": "2021-12-25T21:49:06.887274Z"
    },
    "papermill": {
     "duration": 0.04948,
     "end_time": "2021-12-25T21:49:06.887450",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.837970",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "69e97280",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:06.966224Z",
     "iopub.status.busy": "2021-12-25T21:49:06.965621Z",
     "iopub.status.idle": "2021-12-25T21:49:06.972235Z",
     "shell.execute_reply": "2021-12-25T21:49:06.972730Z"
    },
    "papermill": {
     "duration": 0.047642,
     "end_time": "2021-12-25T21:49:06.972907",
     "exception": false,
     "start_time": "2021-12-25T21:49:06.925265",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "               # ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66970b3e",
   "metadata": {
    "papermill": {
     "duration": 0.038261,
     "end_time": "2021-12-25T21:49:07.049140",
     "exception": false,
     "start_time": "2021-12-25T21:49:07.010879",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# First pipe to tune : \n",
    "imputer + transformer + kmeans fe+ploy fe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "00e51304",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:07.128504Z",
     "iopub.status.busy": "2021-12-25T21:49:07.127884Z",
     "iopub.status.idle": "2021-12-25T21:49:07.206575Z",
     "shell.execute_reply": "2021-12-25T21:49:07.205959Z"
    },
    "papermill": {
     "duration": 0.119484,
     "end_time": "2021-12-25T21:49:07.206712",
     "exception": false,
     "start_time": "2021-12-25T21:49:07.087228",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"2922ba36-de15-4643-8802-c226b9b67c8c\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"2922ba36-de15-4643-8802-c226b9b67c8c\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"d3834f0d-3dd8-4c89-baf7-b83a39651f60\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"d3834f0d-3dd8-4c89-baf7-b83a39651f60\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"e2d6e657-024f-4811-a988-0b053b89ae06\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"e2d6e657-024f-4811-a988-0b053b89ae06\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"84b5ef7c-d2cd-4074-90f4-b7d985e34e18\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"84b5ef7c-d2cd-4074-90f4-b7d985e34e18\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"5573abdb-af66-43da-977f-1583b9bd3b32\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"5573abdb-af66-43da-977f-1583b9bd3b32\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b676d809-3f49-4eb0-bff9-42966daa16a5\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b676d809-3f49-4eb0-bff9-42966daa16a5\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"751ec6bb-d343-4b98-828a-beb6028c8ebd\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"751ec6bb-d343-4b98-828a-beb6028c8ebd\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "937d48a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:07.287009Z",
     "iopub.status.busy": "2021-12-25T21:49:07.286409Z",
     "iopub.status.idle": "2021-12-25T21:49:11.743473Z",
     "shell.execute_reply": "2021-12-25T21:49:11.742920Z"
    },
    "papermill": {
     "duration": 4.498118,
     "end_time": "2021-12-25T21:49:11.743620",
     "exception": false,
     "start_time": "2021-12-25T21:49:07.245502",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 78)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer.fit(X,y)\n",
    "X_pre=preprocessor_woe_powertransformer.transform(X)\n",
    "X_pre.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c94b3b8",
   "metadata": {
    "papermill": {
     "duration": 0.038991,
     "end_time": "2021-12-25T21:49:11.822460",
     "exception": false,
     "start_time": "2021-12-25T21:49:11.783469",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "031a5b8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:11.908793Z",
     "iopub.status.busy": "2021-12-25T21:49:11.907958Z",
     "iopub.status.idle": "2021-12-25T21:49:11.911172Z",
     "shell.execute_reply": "2021-12-25T21:49:11.911690Z"
    },
    "papermill": {
     "duration": 0.050172,
     "end_time": "2021-12-25T21:49:11.911861",
     "exception": false,
     "start_time": "2021-12-25T21:49:11.861689",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StratifiedKFold(n_splits=5, random_state=1, shuffle=True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N_FOLD = 5\n",
    "cross_validation_design = StratifiedKFold(n_splits=N_FOLD,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "114090af",
   "metadata": {
    "papermill": {
     "duration": 0.040769,
     "end_time": "2021-12-25T21:49:11.992328",
     "exception": false,
     "start_time": "2021-12-25T21:49:11.951559",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0210dd2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:12.077855Z",
     "iopub.status.busy": "2021-12-25T21:49:12.076871Z",
     "iopub.status.idle": "2021-12-25T21:49:12.086268Z",
     "shell.execute_reply": "2021-12-25T21:49:12.086894Z"
    },
    "papermill": {
     "duration": 0.054382,
     "end_time": "2021-12-25T21:49:12.087060",
     "exception": false,
     "start_time": "2021-12-25T21:49:12.032678",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective_hsbc_pipe1(trial,X=X_pre,y=y):\n",
    "    # https://neptune.ai/blog/optuna-guide-how-to-monitor-hyper-parameter-optimization-runs\n",
    "    # https://www.kaggle.com/hamidrezabakhtaki/xgboost-catboost-lighgbm-optuna-final-submission\n",
    "    # https://www.kaggle.com/prashant111/lightgbm-classifier-in-python\n",
    "    # https://www.kaggle.com/tunguz/tps-09-21-histgradientboosting-with-optuna\n",
    "    # https://github.com/solegalli/optuna-examples/tree/main/catboost\n",
    "    X_train,X_test, y_train,  y_test = train_test_split(X, y,stratify=y, \n",
    "                                                       test_size=0.25,\n",
    "                                                      random_state=42,\n",
    "                                                        shuffle=True)\n",
    "    params = {\n",
    "        'l2_regularization': trial.suggest_loguniform('l2_regularization',1e-10,10.0),\n",
    "        'early_stopping': trial.suggest_categorical('early_stopping', ['False']),\n",
    "        'learning_rate': trial.suggest_loguniform('learning_rate', 0.001,0.1),\n",
    "        #\"iterations\":trial.suggest_int(\"iterations\" ,1000, 15000),\n",
    "        #'max_iter': trial.suggest_categorical('max_iter', [1000]),\n",
    "        'max_iter': trial.suggest_int(\"iterations\" ,1000, 15000),\n",
    "        'max_depth': trial.suggest_int('max_depth', 2,30),\n",
    "        'max_bins': trial.suggest_int('max_bins', 100,255),\n",
    "        'min_samples_leaf': trial.suggest_int('min_samples_leaf', 20,100000),\n",
    "        'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 20,80),\n",
    "    }\n",
    "\n",
    "    model = HistGradientBoostingClassifier(**params)\n",
    "    model.fit( X_train, y_train,\n",
    "             # eval_set=[(X_test, y_test)],\n",
    "             # early_stopping_rounds=100,\n",
    "              #verbose=False\n",
    "             )\n",
    "    preds = model.predict_proba(X_test)[:,1]\n",
    "    final_auc= roc_auc_score(y_true=y_test, y_score=preds)   \n",
    "    return final_auc               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f9966c7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:49:12.171032Z",
     "iopub.status.busy": "2021-12-25T21:49:12.170025Z",
     "iopub.status.idle": "2021-12-26T00:14:02.979632Z",
     "shell.execute_reply": "2021-12-26T00:14:02.978915Z"
    },
    "papermill": {
     "duration": 8690.852919,
     "end_time": "2021-12-26T00:14:02.979841",
     "exception": false,
     "start_time": "2021-12-25T21:49:12.126922",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 21:49:12,171]\u001b[0m A new study created in memory with name: no-name-57c086aa-eed2-4d51-a83e-e237df761b10\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:49:55,882]\u001b[0m Trial 0 finished with value: 0.8339362394857702 and parameters: {'l2_regularization': 0.0009119120828175131, 'early_stopping': 'False', 'learning_rate': 0.0014751057596489629, 'iterations': 2865, 'max_depth': 27, 'max_bins': 230, 'min_samples_leaf': 33696, 'max_leaf_nodes': 30}. Best is trial 0 with value: 0.8339362394857702.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:50:14,404]\u001b[0m Trial 1 finished with value: 0.8553509697546602 and parameters: {'l2_regularization': 1.6813657718627733e-10, 'early_stopping': 'False', 'learning_rate': 0.06150320619133296, 'iterations': 5177, 'max_depth': 16, 'max_bins': 201, 'min_samples_leaf': 36315, 'max_leaf_nodes': 29}. Best is trial 1 with value: 0.8553509697546602.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:50:15,910]\u001b[0m Trial 2 finished with value: 0.5 and parameters: {'l2_regularization': 0.0022962346759314396, 'early_stopping': 'False', 'learning_rate': 0.0024569061913364064, 'iterations': 8027, 'max_depth': 3, 'max_bins': 132, 'min_samples_leaf': 75841, 'max_leaf_nodes': 26}. Best is trial 1 with value: 0.8553509697546602.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:50:17,440]\u001b[0m Trial 3 finished with value: 0.5 and parameters: {'l2_regularization': 0.008801806526610011, 'early_stopping': 'False', 'learning_rate': 0.06471835049095495, 'iterations': 1631, 'max_depth': 12, 'max_bins': 188, 'min_samples_leaf': 56782, 'max_leaf_nodes': 52}. Best is trial 1 with value: 0.8553509697546602.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:50:19,072]\u001b[0m Trial 4 finished with value: 0.5 and parameters: {'l2_regularization': 0.9633528214202622, 'early_stopping': 'False', 'learning_rate': 0.0017079779961256454, 'iterations': 1166, 'max_depth': 2, 'max_bins': 237, 'min_samples_leaf': 95327, 'max_leaf_nodes': 77}. Best is trial 1 with value: 0.8553509697546602.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:51:03,142]\u001b[0m Trial 5 finished with value: 0.8637243753244486 and parameters: {'l2_regularization': 9.844779966048623e-08, 'early_stopping': 'False', 'learning_rate': 0.007606775623603334, 'iterations': 4515, 'max_depth': 21, 'max_bins': 226, 'min_samples_leaf': 13523, 'max_leaf_nodes': 47}. Best is trial 5 with value: 0.8637243753244486.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:51:04,661]\u001b[0m Trial 6 finished with value: 0.5 and parameters: {'l2_regularization': 8.276018594767585e-06, 'early_stopping': 'False', 'learning_rate': 0.001682458661426076, 'iterations': 5285, 'max_depth': 3, 'max_bins': 196, 'min_samples_leaf': 58304, 'max_leaf_nodes': 67}. Best is trial 5 with value: 0.8637243753244486.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:52:16,907]\u001b[0m Trial 7 finished with value: 0.8609723529848304 and parameters: {'l2_regularization': 1.2379136250445262e-08, 'early_stopping': 'False', 'learning_rate': 0.0025561565579094872, 'iterations': 4019, 'max_depth': 2, 'max_bins': 118, 'min_samples_leaf': 11006, 'max_leaf_nodes': 33}. Best is trial 5 with value: 0.8637243753244486.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:52:46,986]\u001b[0m Trial 8 finished with value: 0.8587902550481691 and parameters: {'l2_regularization': 0.9700171011121308, 'early_stopping': 'False', 'learning_rate': 0.03035735233932774, 'iterations': 9062, 'max_depth': 11, 'max_bins': 101, 'min_samples_leaf': 31506, 'max_leaf_nodes': 64}. Best is trial 5 with value: 0.8637243753244486.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:52:57,480]\u001b[0m Trial 9 finished with value: 0.8584535652469165 and parameters: {'l2_regularization': 1.3379412687190348e-06, 'early_stopping': 'False', 'learning_rate': 0.04440466945444355, 'iterations': 14548, 'max_depth': 7, 'max_bins': 223, 'min_samples_leaf': 28172, 'max_leaf_nodes': 48}. Best is trial 5 with value: 0.8637243753244486.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:53:28,751]\u001b[0m Trial 10 finished with value: 0.8659399814566319 and parameters: {'l2_regularization': 1.1963115680307446e-07, 'early_stopping': 'False', 'learning_rate': 0.008519699035165371, 'iterations': 11087, 'max_depth': 25, 'max_bins': 145, 'min_samples_leaf': 4386, 'max_leaf_nodes': 45}. Best is trial 10 with value: 0.8659399814566319.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:54:01,130]\u001b[0m Trial 11 finished with value: 0.8665559005189855 and parameters: {'l2_regularization': 2.3645973812170054e-08, 'early_stopping': 'False', 'learning_rate': 0.008567254818793396, 'iterations': 11893, 'max_depth': 26, 'max_bins': 155, 'min_samples_leaf': 3368, 'max_leaf_nodes': 45}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:54:21,472]\u001b[0m Trial 12 finished with value: 0.8664291144472105 and parameters: {'l2_regularization': 9.59264659029924e-10, 'early_stopping': 'False', 'learning_rate': 0.011034357658739266, 'iterations': 12169, 'max_depth': 28, 'max_bins': 155, 'min_samples_leaf': 1339, 'max_leaf_nodes': 40}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:54:30,332]\u001b[0m Trial 13 finished with value: 0.8644035933465916 and parameters: {'l2_regularization': 1.2741600276292304e-10, 'early_stopping': 'False', 'learning_rate': 0.019018701213468003, 'iterations': 13070, 'max_depth': 30, 'max_bins': 159, 'min_samples_leaf': 3321, 'max_leaf_nodes': 38}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:55:20,690]\u001b[0m Trial 14 finished with value: 0.8618085127442439 and parameters: {'l2_regularization': 3.690749116328915e-09, 'early_stopping': 'False', 'learning_rate': 0.004759812273029884, 'iterations': 11105, 'max_depth': 23, 'max_bins': 171, 'min_samples_leaf': 18084, 'max_leaf_nodes': 57}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:56:11,601]\u001b[0m Trial 15 finished with value: 0.8135867414233728 and parameters: {'l2_regularization': 5.637453159092646e-09, 'early_stopping': 'False', 'learning_rate': 0.020253161681639424, 'iterations': 11283, 'max_depth': 19, 'max_bins': 155, 'min_samples_leaf': 47541, 'max_leaf_nodes': 21}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:56:42,587]\u001b[0m Trial 16 finished with value: 0.8619041111642397 and parameters: {'l2_regularization': 7.013272951026445e-05, 'early_stopping': 'False', 'learning_rate': 0.012846870038784135, 'iterations': 13282, 'max_depth': 30, 'max_bins': 130, 'min_samples_leaf': 19029, 'max_leaf_nodes': 42}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:57:39,026]\u001b[0m Trial 17 finished with value: 0.8657476956145395 and parameters: {'l2_regularization': 3.2083907057325726e-07, 'early_stopping': 'False', 'learning_rate': 0.003947032393552379, 'iterations': 9109, 'max_depth': 27, 'max_bins': 255, 'min_samples_leaf': 1798, 'max_leaf_nodes': 37}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:57:40,569]\u001b[0m Trial 18 finished with value: 0.5 and parameters: {'l2_regularization': 1.1076691669019675e-09, 'early_stopping': 'False', 'learning_rate': 0.005687324490629725, 'iterations': 12232, 'max_depth': 19, 'max_bins': 171, 'min_samples_leaf': 71762, 'max_leaf_nodes': 58}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:58:05,533]\u001b[0m Trial 19 finished with value: 0.8598177313807895 and parameters: {'l2_regularization': 7.138161109678297e-06, 'early_stopping': 'False', 'learning_rate': 0.014326034374638183, 'iterations': 14385, 'max_depth': 25, 'max_bins': 146, 'min_samples_leaf': 22270, 'max_leaf_nodes': 53}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:58:28,510]\u001b[0m Trial 20 finished with value: 0.8499350145123499 and parameters: {'l2_regularization': 2.3687668677905422e-08, 'early_stopping': 'False', 'learning_rate': 0.02716176057390375, 'iterations': 6974, 'max_depth': 14, 'max_bins': 184, 'min_samples_leaf': 43797, 'max_leaf_nodes': 41}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:58:50,711]\u001b[0m Trial 21 finished with value: 0.8661395596216904 and parameters: {'l2_regularization': 3.740968355287137e-07, 'early_stopping': 'False', 'learning_rate': 0.008689875306081049, 'iterations': 11258, 'max_depth': 26, 'max_bins': 147, 'min_samples_leaf': 771, 'max_leaf_nodes': 45}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:59:17,977]\u001b[0m Trial 22 finished with value: 0.8637882957566412 and parameters: {'l2_regularization': 1.3776498932730997e-06, 'early_stopping': 'False', 'learning_rate': 0.011243474631270062, 'iterations': 9918, 'max_depth': 27, 'max_bins': 161, 'min_samples_leaf': 10108, 'max_leaf_nodes': 35}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 21:59:44,509]\u001b[0m Trial 23 finished with value: 0.865734952579489 and parameters: {'l2_regularization': 1.0521605290406199e-09, 'early_stopping': 'False', 'learning_rate': 0.006603861455826522, 'iterations': 12410, 'max_depth': 23, 'max_bins': 131, 'min_samples_leaf': 1468, 'max_leaf_nodes': 40}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:01:25,931]\u001b[0m Trial 24 finished with value: 0.8599790975454418 and parameters: {'l2_regularization': 4.416153571832147e-08, 'early_stopping': 'False', 'learning_rate': 0.00317294569798822, 'iterations': 10325, 'max_depth': 30, 'max_bins': 147, 'min_samples_leaf': 21718, 'max_leaf_nodes': 45}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:01:52,369]\u001b[0m Trial 25 finished with value: 0.8639165129254173 and parameters: {'l2_regularization': 9.034104090339663e-05, 'early_stopping': 'False', 'learning_rate': 0.010749838305498027, 'iterations': 13736, 'max_depth': 24, 'max_bins': 114, 'min_samples_leaf': 8434, 'max_leaf_nodes': 56}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:02:18,568]\u001b[0m Trial 26 finished with value: 0.863366276083995 and parameters: {'l2_regularization': 6.396045014773606e-07, 'early_stopping': 'False', 'learning_rate': 0.016992111766156875, 'iterations': 12078, 'max_depth': 21, 'max_bins': 169, 'min_samples_leaf': 14715, 'max_leaf_nodes': 63}. Best is trial 11 with value: 0.8665559005189855.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:02:39,111]\u001b[0m Trial 27 finished with value: 0.8668950020892877 and parameters: {'l2_regularization': 7.599572380329895e-10, 'early_stopping': 'False', 'learning_rate': 0.00910078717370759, 'iterations': 7719, 'max_depth': 28, 'max_bins': 141, 'min_samples_leaf': 285, 'max_leaf_nodes': 43}. Best is trial 27 with value: 0.8668950020892877.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:03:00,489]\u001b[0m Trial 28 finished with value: 0.8605419177776276 and parameters: {'l2_regularization': 7.901416892303495e-10, 'early_stopping': 'False', 'learning_rate': 0.02703821325354806, 'iterations': 6863, 'max_depth': 28, 'max_bins': 206, 'min_samples_leaf': 25651, 'max_leaf_nodes': 23}. Best is trial 27 with value: 0.8668950020892877.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:04:52,922]\u001b[0m Trial 29 finished with value: 0.8538165828997888 and parameters: {'l2_regularization': 3.699948701710313e-10, 'early_stopping': 'False', 'learning_rate': 0.004888632410513789, 'iterations': 7475, 'max_depth': 28, 'max_bins': 137, 'min_samples_leaf': 37474, 'max_leaf_nodes': 32}. Best is trial 27 with value: 0.8668950020892877.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:04:54,423]\u001b[0m Trial 30 finished with value: 0.5 and parameters: {'l2_regularization': 3.3612223235558585e-09, 'early_stopping': 'False', 'learning_rate': 0.09587799260530602, 'iterations': 8902, 'max_depth': 21, 'max_bins': 118, 'min_samples_leaf': 98037, 'max_leaf_nodes': 50}. Best is trial 27 with value: 0.8668950020892877.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:05:20,756]\u001b[0m Trial 31 finished with value: 0.8633284346863976 and parameters: {'l2_regularization': 1.853069880620221e-08, 'early_stopping': 'False', 'learning_rate': 0.007537634265137625, 'iterations': 10122, 'max_depth': 26, 'max_bins': 156, 'min_samples_leaf': 7780, 'max_leaf_nodes': 43}. Best is trial 27 with value: 0.8668950020892877.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:05:40,844]\u001b[0m Trial 32 finished with value: 0.8679321597075973 and parameters: {'l2_regularization': 1.8211948290688826e-10, 'early_stopping': 'False', 'learning_rate': 0.009429166930620776, 'iterations': 6349, 'max_depth': 28, 'max_bins': 140, 'min_samples_leaf': 127, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:08:00,972]\u001b[0m Trial 33 finished with value: 0.8607544727425853 and parameters: {'l2_regularization': 1.4078867058830957e-10, 'early_stopping': 'False', 'learning_rate': 0.0010206382111630457, 'iterations': 6218, 'max_depth': 29, 'max_bins': 138, 'min_samples_leaf': 14882, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:08:16,725]\u001b[0m Trial 34 finished with value: 0.8668065705574611 and parameters: {'l2_regularization': 2.1150635921246295e-09, 'early_stopping': 'False', 'learning_rate': 0.015110057644925732, 'iterations': 8153, 'max_depth': 28, 'max_bins': 180, 'min_samples_leaf': 73, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:08:43,878]\u001b[0m Trial 35 finished with value: 0.8639062044657433 and parameters: {'l2_regularization': 0.04988534131550968, 'early_stopping': 'False', 'learning_rate': 0.014611237612293397, 'iterations': 5980, 'max_depth': 18, 'max_bins': 183, 'min_samples_leaf': 8205, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:08:45,421]\u001b[0m Trial 36 finished with value: 0.5 and parameters: {'l2_regularization': 2.481638800471675e-09, 'early_stopping': 'False', 'learning_rate': 0.02280271932571469, 'iterations': 3137, 'max_depth': 23, 'max_bins': 212, 'min_samples_leaf': 74069, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:08:46,843]\u001b[0m Trial 37 finished with value: 0.5 and parameters: {'l2_regularization': 3.03367483012004e-10, 'early_stopping': 'False', 'learning_rate': 0.03634162290454811, 'iterations': 8146, 'max_depth': 25, 'max_bins': 125, 'min_samples_leaf': 63622, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:10:04,007]\u001b[0m Trial 38 finished with value: 0.8522277516414739 and parameters: {'l2_regularization': 0.0005184040113356969, 'early_stopping': 'False', 'learning_rate': 0.006243495762085572, 'iterations': 5139, 'max_depth': 29, 'max_bins': 197, 'min_samples_leaf': 36440, 'max_leaf_nodes': 78}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:11:22,182]\u001b[0m Trial 39 finished with value: 0.8635076296969573 and parameters: {'l2_regularization': 6.842872537450797, 'early_stopping': 'False', 'learning_rate': 0.0034567359401222243, 'iterations': 8262, 'max_depth': 16, 'max_bins': 175, 'min_samples_leaf': 14266, 'max_leaf_nodes': 20}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:11:50,557]\u001b[0m Trial 40 finished with value: 0.8660773610671644 and parameters: {'l2_regularization': 8.68528176859084e-09, 'early_stopping': 'False', 'learning_rate': 0.009123748350140286, 'iterations': 7438, 'max_depth': 8, 'max_bins': 164, 'min_samples_leaf': 6351, 'max_leaf_nodes': 33}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:12:07,565]\u001b[0m Trial 41 finished with value: 0.867043665129961 and parameters: {'l2_regularization': 1.00854989315159e-10, 'early_stopping': 'False', 'learning_rate': 0.013099543787649806, 'iterations': 5762, 'max_depth': 28, 'max_bins': 153, 'min_samples_leaf': 199, 'max_leaf_nodes': 36}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:12:33,002]\u001b[0m Trial 42 finished with value: 0.8636651529955586 and parameters: {'l2_regularization': 1.2201467820753155e-10, 'early_stopping': 'False', 'learning_rate': 0.01513277625429824, 'iterations': 4330, 'max_depth': 27, 'max_bins': 140, 'min_samples_leaf': 12048, 'max_leaf_nodes': 37}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:12:47,406]\u001b[0m Trial 43 finished with value: 0.8678110923222457 and parameters: {'l2_regularization': 6.273162491914922e-10, 'early_stopping': 'False', 'learning_rate': 0.012069796283564076, 'iterations': 5911, 'max_depth': 26, 'max_bins': 193, 'min_samples_leaf': 166, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:13:02,644]\u001b[0m Trial 44 finished with value: 0.8671629593234111 and parameters: {'l2_regularization': 4.34835203900559e-10, 'early_stopping': 'False', 'learning_rate': 0.012531043096168497, 'iterations': 5818, 'max_depth': 29, 'max_bins': 189, 'min_samples_leaf': 80, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:13:25,280]\u001b[0m Trial 45 finished with value: 0.8641200536897421 and parameters: {'l2_regularization': 3.130770495115638e-10, 'early_stopping': 'False', 'learning_rate': 0.012054015249895139, 'iterations': 5522, 'max_depth': 29, 'max_bins': 191, 'min_samples_leaf': 7886, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:13:41,188]\u001b[0m Trial 46 finished with value: 0.8618116999284018 and parameters: {'l2_regularization': 7.333534136211893e-10, 'early_stopping': 'False', 'learning_rate': 0.021494845856143814, 'iterations': 2137, 'max_depth': 24, 'max_bins': 204, 'min_samples_leaf': 17701, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:14:18,334]\u001b[0m Trial 47 finished with value: 0.8654210120891321 and parameters: {'l2_regularization': 5.593291099334336e-10, 'early_stopping': 'False', 'learning_rate': 0.007324567864298121, 'iterations': 3785, 'max_depth': 30, 'max_bins': 215, 'min_samples_leaf': 6232, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:14:19,867]\u001b[0m Trial 48 finished with value: 0.5 and parameters: {'l2_regularization': 1.23496240493268e-10, 'early_stopping': 'False', 'learning_rate': 0.00996884663597894, 'iterations': 6198, 'max_depth': 22, 'max_bins': 189, 'min_samples_leaf': 88322, 'max_leaf_nodes': 34}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:15:28,760]\u001b[0m Trial 49 finished with value: 0.8591742565741859 and parameters: {'l2_regularization': 6.641309754749656e-08, 'early_stopping': 'False', 'learning_rate': 0.004956925362327601, 'iterations': 5168, 'max_depth': 26, 'max_bins': 104, 'min_samples_leaf': 25852, 'max_leaf_nodes': 74}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:15:35,058]\u001b[0m Trial 50 finished with value: 0.8655450728048648 and parameters: {'l2_regularization': 5.14881187920062e-09, 'early_stopping': 'False', 'learning_rate': 0.048780439208069765, 'iterations': 6726, 'max_depth': 27, 'max_bins': 195, 'min_samples_leaf': 5688, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:15:47,042]\u001b[0m Trial 51 finished with value: 0.8664635976131992 and parameters: {'l2_regularization': 2.240211006969252e-09, 'early_stopping': 'False', 'learning_rate': 0.016748239820116754, 'iterations': 4838, 'max_depth': 28, 'max_bins': 180, 'min_samples_leaf': 827, 'max_leaf_nodes': 31}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:16:00,846]\u001b[0m Trial 52 finished with value: 0.8676075629610841 and parameters: {'l2_regularization': 1.4683004478663684e-09, 'early_stopping': 'False', 'learning_rate': 0.013005197988231946, 'iterations': 5772, 'max_depth': 29, 'max_bins': 177, 'min_samples_leaf': 166, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:16:23,997]\u001b[0m Trial 53 finished with value: 0.8633814308880953 and parameters: {'l2_regularization': 3.490294656635072e-10, 'early_stopping': 'False', 'learning_rate': 0.013240938240604374, 'iterations': 5679, 'max_depth': 29, 'max_bins': 163, 'min_samples_leaf': 11777, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:16:53,371]\u001b[0m Trial 54 finished with value: 0.8655392058773181 and parameters: {'l2_regularization': 1.0577131051488125e-10, 'early_stopping': 'False', 'learning_rate': 0.009932404627521565, 'iterations': 3596, 'max_depth': 30, 'max_bins': 152, 'min_samples_leaf': 5906, 'max_leaf_nodes': 36}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:17:13,020]\u001b[0m Trial 55 finished with value: 0.8659421651624145 and parameters: {'l2_regularization': 1.266247031100372e-08, 'early_stopping': 'False', 'learning_rate': 0.01783399976505682, 'iterations': 7371, 'max_depth': 25, 'max_bins': 167, 'min_samples_leaf': 4074, 'max_leaf_nodes': 39}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:17:59,357]\u001b[0m Trial 56 finished with value: 0.863797897220187 and parameters: {'l2_regularization': 1.3494691508562259e-09, 'early_stopping': 'False', 'learning_rate': 0.011578403062208252, 'iterations': 6378, 'max_depth': 27, 'max_bins': 174, 'min_samples_leaf': 11215, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:18:17,360]\u001b[0m Trial 57 finished with value: 0.8666308592133066 and parameters: {'l2_regularization': 3.2398564723871327e-10, 'early_stopping': 'False', 'learning_rate': 0.007778927277748798, 'iterations': 5744, 'max_depth': 29, 'max_bins': 124, 'min_samples_leaf': 75, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:19:29,678]\u001b[0m Trial 58 finished with value: 0.862560414529619 and parameters: {'l2_regularization': 1.0836173065698265e-08, 'early_stopping': 'False', 'learning_rate': 0.005668976920215162, 'iterations': 4356, 'max_depth': 24, 'max_bins': 232, 'min_samples_leaf': 17976, 'max_leaf_nodes': 33}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:19:31,357]\u001b[0m Trial 59 finished with value: 0.5 and parameters: {'l2_regularization': 0.04059797692900436, 'early_stopping': 'False', 'learning_rate': 0.024289289043591258, 'iterations': 4922, 'max_depth': 9, 'max_bins': 198, 'min_samples_leaf': 55213, 'max_leaf_nodes': 35}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:20:07,876]\u001b[0m Trial 60 finished with value: 0.8666850641503199 and parameters: {'l2_regularization': 3.2087913465537935e-08, 'early_stopping': 'False', 'learning_rate': 0.009125383924338211, 'iterations': 6610, 'max_depth': 26, 'max_bins': 134, 'min_samples_leaf': 3842, 'max_leaf_nodes': 48}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:20:22,267]\u001b[0m Trial 61 finished with value: 0.86704644750182 and parameters: {'l2_regularization': 1.757676167468181e-09, 'early_stopping': 'False', 'learning_rate': 0.015167810670119405, 'iterations': 7831, 'max_depth': 28, 'max_bins': 185, 'min_samples_leaf': 677, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:20:49,656]\u001b[0m Trial 62 finished with value: 0.8657395024416681 and parameters: {'l2_regularization': 1.642733721176617e-09, 'early_stopping': 'False', 'learning_rate': 0.011852920099072396, 'iterations': 7011, 'max_depth': 28, 'max_bins': 185, 'min_samples_leaf': 3070, 'max_leaf_nodes': 31}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:21:12,210]\u001b[0m Trial 63 finished with value: 0.8638063811737239 and parameters: {'l2_regularization': 1.202525098948354e-07, 'early_stopping': 'False', 'learning_rate': 0.019019850301383614, 'iterations': 8871, 'max_depth': 30, 'max_bins': 191, 'min_samples_leaf': 9929, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:21:35,807]\u001b[0m Trial 64 finished with value: 0.8666695843560641 and parameters: {'l2_regularization': 4.1636338756453485e-09, 'early_stopping': 'False', 'learning_rate': 0.012780954972155185, 'iterations': 7571, 'max_depth': 13, 'max_bins': 151, 'min_samples_leaf': 3997, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:21:37,311]\u001b[0m Trial 65 finished with value: 0.5 and parameters: {'l2_regularization': 5.757847058133577e-10, 'early_stopping': 'False', 'learning_rate': 0.010185839123540935, 'iterations': 9469, 'max_depth': 27, 'max_bins': 174, 'min_samples_leaf': 82476, 'max_leaf_nodes': 43}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:21:59,006]\u001b[0m Trial 66 finished with value: 0.8628782092891902 and parameters: {'l2_regularization': 2.167512490473986e-10, 'early_stopping': 'False', 'learning_rate': 0.016267360689949956, 'iterations': 8620, 'max_depth': 4, 'max_bins': 142, 'min_samples_leaf': 15574, 'max_leaf_nodes': 20}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:22:07,948]\u001b[0m Trial 67 finished with value: 0.8671869971917657 and parameters: {'l2_regularization': 9.125125649874848e-10, 'early_stopping': 'False', 'learning_rate': 0.033325227890365294, 'iterations': 7765, 'max_depth': 29, 'max_bins': 215, 'min_samples_leaf': 284, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:22:18,471]\u001b[0m Trial 68 finished with value: 0.8640221005165132 and parameters: {'l2_regularization': 6.377213678346561e-06, 'early_stopping': 'False', 'learning_rate': 0.035730650583262316, 'iterations': 7034, 'max_depth': 29, 'max_bins': 204, 'min_samples_leaf': 10487, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:22:26,115]\u001b[0m Trial 69 finished with value: 0.8596296019917404 and parameters: {'l2_regularization': 7.767712917599608e-09, 'early_stopping': 'False', 'learning_rate': 0.05246774580058464, 'iterations': 4663, 'max_depth': 25, 'max_bins': 246, 'min_samples_leaf': 22635, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:22:47,425]\u001b[0m Trial 70 finished with value: 0.8577861437972653 and parameters: {'l2_regularization': 1.7132032947376211e-09, 'early_stopping': 'False', 'learning_rate': 0.0330444982609538, 'iterations': 5945, 'max_depth': 26, 'max_bins': 216, 'min_samples_leaf': 31765, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:23:10,329]\u001b[0m Trial 71 finished with value: 0.8664711351041777 and parameters: {'l2_regularization': 8.235079430842125e-10, 'early_stopping': 'False', 'learning_rate': 0.006660455094098522, 'iterations': 7800, 'max_depth': 30, 'max_bins': 212, 'min_samples_leaf': 100, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:23:31,783]\u001b[0m Trial 72 finished with value: 0.8654226427414918 and parameters: {'l2_regularization': 2.2616314531488649e-10, 'early_stopping': 'False', 'learning_rate': 0.013748930362499688, 'iterations': 6468, 'max_depth': 28, 'max_bins': 223, 'min_samples_leaf': 3363, 'max_leaf_nodes': 32}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:23:49,678]\u001b[0m Trial 73 finished with value: 0.8638212451971586 and parameters: {'l2_regularization': 4.917074245182218e-10, 'early_stopping': 'False', 'learning_rate': 0.026413675947095247, 'iterations': 5373, 'max_depth': 28, 'max_bins': 126, 'min_samples_leaf': 7814, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:24:19,058]\u001b[0m Trial 74 finished with value: 0.866851727084351 and parameters: {'l2_regularization': 3.3134033467257867e-09, 'early_stopping': 'False', 'learning_rate': 0.008025978259897024, 'iterations': 8542, 'max_depth': 29, 'max_bins': 186, 'min_samples_leaf': 2757, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:24:31,498]\u001b[0m Trial 75 finished with value: 0.8659152993095476 and parameters: {'l2_regularization': 2.0578025332470003e-10, 'early_stopping': 'False', 'learning_rate': 0.019616220029427634, 'iterations': 9428, 'max_depth': 27, 'max_bins': 179, 'min_samples_leaf': 5617, 'max_leaf_nodes': 38}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:24:42,399]\u001b[0m Trial 76 finished with value: 0.8513861867826298 and parameters: {'l2_regularization': 1.0630642335993154e-09, 'early_stopping': 'False', 'learning_rate': 0.07532345850044501, 'iterations': 7247, 'max_depth': 27, 'max_bins': 208, 'min_samples_leaf': 41234, 'max_leaf_nodes': 35}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:25:16,302]\u001b[0m Trial 77 finished with value: 0.8637970191766086 and parameters: {'l2_regularization': 5.859545739197822e-09, 'early_stopping': 'False', 'learning_rate': 0.01078679186950991, 'iterations': 7877, 'max_depth': 30, 'max_bins': 200, 'min_samples_leaf': 9082, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:25:23,178]\u001b[0m Trial 78 finished with value: 0.8666962848630624 and parameters: {'l2_regularization': 4.698348502802677e-10, 'early_stopping': 'False', 'learning_rate': 0.04100906214414236, 'iterations': 5724, 'max_depth': 26, 'max_bins': 193, 'min_samples_leaf': 2303, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:26:03,376]\u001b[0m Trial 79 finished with value: 0.8639810035157776 and parameters: {'l2_regularization': 1.791056941777807e-08, 'early_stopping': 'False', 'learning_rate': 0.008991331580927823, 'iterations': 6131, 'max_depth': 29, 'max_bins': 149, 'min_samples_leaf': 12138, 'max_leaf_nodes': 21}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:26:18,846]\u001b[0m Trial 80 finished with value: 0.8654928862277657 and parameters: {'l2_regularization': 1.9752339890812048e-10, 'early_stopping': 'False', 'learning_rate': 0.015864041811888448, 'iterations': 6734, 'max_depth': 24, 'max_bins': 158, 'min_samples_leaf': 5743, 'max_leaf_nodes': 41}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:26:41,410]\u001b[0m Trial 81 finished with value: 0.8664292740914974 and parameters: {'l2_regularization': 3.222173490136778e-09, 'early_stopping': 'False', 'learning_rate': 0.008144412556966572, 'iterations': 8304, 'max_depth': 29, 'max_bins': 185, 'min_samples_leaf': 2530, 'max_leaf_nodes': 31}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:27:10,975]\u001b[0m Trial 82 finished with value: 0.8675421601176551 and parameters: {'l2_regularization': 1.0593680100876227e-09, 'early_stopping': 'False', 'learning_rate': 0.007262230792901344, 'iterations': 8553, 'max_depth': 28, 'max_bins': 187, 'min_samples_leaf': 256, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:27:44,878]\u001b[0m Trial 83 finished with value: 0.8671522973656731 and parameters: {'l2_regularization': 1.1969450393762046e-09, 'early_stopping': 'False', 'learning_rate': 0.005923860669673911, 'iterations': 7659, 'max_depth': 28, 'max_bins': 168, 'min_samples_leaf': 321, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:28:28,073]\u001b[0m Trial 84 finished with value: 0.8669383341100407 and parameters: {'l2_regularization': 1.1198621446474263e-09, 'early_stopping': 'False', 'learning_rate': 0.004289071843223198, 'iterations': 9402, 'max_depth': 28, 'max_bins': 168, 'min_samples_leaf': 119, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:29:15,042]\u001b[0m Trial 85 finished with value: 0.8640931593289667 and parameters: {'l2_regularization': 1.0524292490336119e-10, 'early_stopping': 'False', 'learning_rate': 0.005741760165833418, 'iterations': 7143, 'max_depth': 27, 'max_bins': 179, 'min_samples_leaf': 8017, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:29:51,430]\u001b[0m Trial 86 finished with value: 0.8653767450089835 and parameters: {'l2_regularization': 1.8872193750570443e-09, 'early_stopping': 'False', 'learning_rate': 0.007192725332492325, 'iterations': 7888, 'max_depth': 26, 'max_bins': 173, 'min_samples_leaf': 4960, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:30:19,798]\u001b[0m Trial 87 finished with value: 0.8629379903730845 and parameters: {'l2_regularization': 3.1704756523715477e-10, 'early_stopping': 'False', 'learning_rate': 0.0138024492410512, 'iterations': 6389, 'max_depth': 30, 'max_bins': 167, 'min_samples_leaf': 14350, 'max_leaf_nodes': 32}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:31:01,189]\u001b[0m Trial 88 finished with value: 0.8654170038772124 and parameters: {'l2_regularization': 0.0018868972893229246, 'early_stopping': 'False', 'learning_rate': 0.006385018605554164, 'iterations': 5223, 'max_depth': 28, 'max_bins': 189, 'min_samples_leaf': 6494, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:32:34,624]\u001b[0m Trial 89 finished with value: 0.8669523143883153 and parameters: {'l2_regularization': 5.771092858054046e-10, 'early_stopping': 'False', 'learning_rate': 0.0025890778373384056, 'iterations': 10662, 'max_depth': 25, 'max_bins': 194, 'min_samples_leaf': 1960, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:32:36,101]\u001b[0m Trial 90 finished with value: 0.5 and parameters: {'l2_regularization': 1.2257138385749054e-09, 'early_stopping': 'False', 'learning_rate': 0.012517641406824043, 'iterations': 8597, 'max_depth': 19, 'max_bins': 161, 'min_samples_leaf': 68217, 'max_leaf_nodes': 34}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:33:49,308]\u001b[0m Trial 91 finished with value: 0.866253990365969 and parameters: {'l2_regularization': 5.282198685066046e-10, 'early_stopping': 'False', 'learning_rate': 0.00239782134972842, 'iterations': 10797, 'max_depth': 25, 'max_bins': 201, 'min_samples_leaf': 1723, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:35:33,453]\u001b[0m Trial 92 finished with value: 0.8662488076282236 and parameters: {'l2_regularization': 1.8517991916894532e-10, 'early_stopping': 'False', 'learning_rate': 0.0023633347939590065, 'iterations': 11638, 'max_depth': 28, 'max_bins': 182, 'min_samples_leaf': 3952, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:37:40,974]\u001b[0m Trial 93 finished with value: 0.8676165144443186 and parameters: {'l2_regularization': 7.763008659818068e-10, 'early_stopping': 'False', 'learning_rate': 0.0016009340237239454, 'iterations': 14971, 'max_depth': 26, 'max_bins': 195, 'min_samples_leaf': 45, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:38:14,906]\u001b[0m Trial 94 finished with value: 0.864106278668408 and parameters: {'l2_regularization': 3.2678521065937415e-09, 'early_stopping': 'False', 'learning_rate': 0.01003689425906768, 'iterations': 12623, 'max_depth': 29, 'max_bins': 177, 'min_samples_leaf': 9065, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:38:30,779]\u001b[0m Trial 95 finished with value: 0.865402932373631 and parameters: {'l2_regularization': 8.998056783298944e-10, 'early_stopping': 'False', 'learning_rate': 0.022005267474040257, 'iterations': 13572, 'max_depth': 27, 'max_bins': 188, 'min_samples_leaf': 7015, 'max_leaf_nodes': 31}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:41:20,184]\u001b[0m Trial 96 finished with value: 0.8660564419639879 and parameters: {'l2_regularization': 6.183647078949455e-09, 'early_stopping': 'False', 'learning_rate': 0.0013327835092396022, 'iterations': 6009, 'max_depth': 22, 'max_bins': 197, 'min_samples_leaf': 4735, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:43:50,092]\u001b[0m Trial 97 finished with value: 0.8629640237950266 and parameters: {'l2_regularization': 2.458755189767829e-09, 'early_stopping': 'False', 'learning_rate': 0.0018042774561184273, 'iterations': 14960, 'max_depth': 30, 'max_bins': 171, 'min_samples_leaf': 12482, 'max_leaf_nodes': 33}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:44:00,533]\u001b[0m Trial 98 finished with value: 0.8670208930127391 and parameters: {'l2_regularization': 1.0360876901345175e-10, 'early_stopping': 'False', 'learning_rate': 0.01788668973732723, 'iterations': 1428, 'max_depth': 26, 'max_bins': 183, 'min_samples_leaf': 105, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:44:21,277]\u001b[0m Trial 99 finished with value: 0.8664798471209818 and parameters: {'l2_regularization': 4.8876058357768286e-08, 'early_stopping': 'False', 'learning_rate': 0.011124185899736171, 'iterations': 5540, 'max_depth': 27, 'max_bins': 209, 'min_samples_leaf': 2449, 'max_leaf_nodes': 21}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:46:04,603]\u001b[0m Trial 100 finished with value: 0.8598771133539664 and parameters: {'l2_regularization': 1.4802850594326238e-08, 'early_stopping': 'False', 'learning_rate': 0.0010437674357360429, 'iterations': 3982, 'max_depth': 29, 'max_bins': 220, 'min_samples_leaf': 9727, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:46:16,209]\u001b[0m Trial 101 finished with value: 0.8668646810779237 and parameters: {'l2_regularization': 1.6564752443944064e-10, 'early_stopping': 'False', 'learning_rate': 0.014969532617969107, 'iterations': 14188, 'max_depth': 26, 'max_bins': 192, 'min_samples_leaf': 45, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:46:32,489]\u001b[0m Trial 102 finished with value: 0.8658211718976246 and parameters: {'l2_regularization': 1.0217634461225485e-10, 'early_stopping': 'False', 'learning_rate': 0.020515286938674465, 'iterations': 2148, 'max_depth': 28, 'max_bins': 182, 'min_samples_leaf': 4793, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:46:39,521]\u001b[0m Trial 103 finished with value: 0.8664499423250802 and parameters: {'l2_regularization': 2.8699213206420315e-10, 'early_stopping': 'False', 'learning_rate': 0.024925138416880346, 'iterations': 4824, 'max_depth': 24, 'max_bins': 177, 'min_samples_leaf': 54, 'max_leaf_nodes': 34}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:46:52,886]\u001b[0m Trial 104 finished with value: 0.866420145859231 and parameters: {'l2_regularization': 4.1893288028104105e-10, 'early_stopping': 'False', 'learning_rate': 0.01811679496234278, 'iterations': 2047, 'max_depth': 15, 'max_bins': 187, 'min_samples_leaf': 2719, 'max_leaf_nodes': 32}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:47:08,679]\u001b[0m Trial 105 finished with value: 0.8646876404442106 and parameters: {'l2_regularization': 1.4797203049713493e-09, 'early_stopping': 'False', 'learning_rate': 0.017210534745465248, 'iterations': 6811, 'max_depth': 25, 'max_bins': 200, 'min_samples_leaf': 7036, 'max_leaf_nodes': 36}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:47:26,091]\u001b[0m Trial 106 finished with value: 0.8660357851335683 and parameters: {'l2_regularization': 6.220896129604215e-10, 'early_stopping': 'False', 'learning_rate': 0.012510501371574128, 'iterations': 3567, 'max_depth': 26, 'max_bins': 234, 'min_samples_leaf': 2259, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:47:41,714]\u001b[0m Trial 107 finished with value: 0.8624375910571103 and parameters: {'l2_regularization': 2.735976139816846e-10, 'early_stopping': 'False', 'learning_rate': 0.030096404987195607, 'iterations': 1319, 'max_depth': 28, 'max_bins': 183, 'min_samples_leaf': 16255, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:48:09,579]\u001b[0m Trial 108 finished with value: 0.8663574455655173 and parameters: {'l2_regularization': 2.6081978970761993e-05, 'early_stopping': 'False', 'learning_rate': 0.009903998928994621, 'iterations': 7495, 'max_depth': 29, 'max_bins': 164, 'min_samples_leaf': 4141, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:48:11,094]\u001b[0m Trial 109 finished with value: 0.5 and parameters: {'l2_regularization': 8.34902403589838e-10, 'early_stopping': 'False', 'learning_rate': 0.014065518433805858, 'iterations': 2839, 'max_depth': 27, 'max_bins': 154, 'min_samples_leaf': 50683, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:49:41,006]\u001b[0m Trial 110 finished with value: 0.8636723426900546 and parameters: {'l2_regularization': 1.8172228305403004e-10, 'early_stopping': 'False', 'learning_rate': 0.0037601854788918976, 'iterations': 4291, 'max_depth': 29, 'max_bins': 189, 'min_samples_leaf': 10509, 'max_leaf_nodes': 20}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:50:56,057]\u001b[0m Trial 111 finished with value: 0.865582315536386 and parameters: {'l2_regularization': 5.469099719605761e-10, 'early_stopping': 'False', 'learning_rate': 0.0029093310067496627, 'iterations': 9763, 'max_depth': 25, 'max_bins': 194, 'min_samples_leaf': 1760, 'max_leaf_nodes': 60}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:52:42,060]\u001b[0m Trial 112 finished with value: 0.8668409225870712 and parameters: {'l2_regularization': 1.6539425201755133e-09, 'early_stopping': 'False', 'learning_rate': 0.001873171747175323, 'iterations': 8956, 'max_depth': 23, 'max_bins': 203, 'min_samples_leaf': 1514, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:53:03,566]\u001b[0m Trial 113 finished with value: 0.8652170037951095 and parameters: {'l2_regularization': 1.0012738675010035e-10, 'early_stopping': 'False', 'learning_rate': 0.011711312356272164, 'iterations': 8133, 'max_depth': 27, 'max_bins': 195, 'min_samples_leaf': 5787, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:54:49,418]\u001b[0m Trial 114 finished with value: 0.8659699717762585 and parameters: {'l2_regularization': 3.634106595148393e-10, 'early_stopping': 'False', 'learning_rate': 0.0021595523153383643, 'iterations': 12760, 'max_depth': 11, 'max_bins': 192, 'min_samples_leaf': 3785, 'max_leaf_nodes': 52}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:56:58,659]\u001b[0m Trial 115 finished with value: 0.86682780324763 and parameters: {'l2_regularization': 7.783724398503489e-10, 'early_stopping': 'False', 'learning_rate': 0.001470843510011279, 'iterations': 10955, 'max_depth': 30, 'max_bins': 145, 'min_samples_leaf': 1402, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:58:03,448]\u001b[0m Trial 116 finished with value: 0.8656294505121185 and parameters: {'l2_regularization': 4.335879350740481e-09, 'early_stopping': 'False', 'learning_rate': 0.005476882168691183, 'iterations': 6479, 'max_depth': 26, 'max_bins': 171, 'min_samples_leaf': 7243, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:00:21,817]\u001b[0m Trial 117 finished with value: 0.8654587451566774 and parameters: {'l2_regularization': 1.9497894307280437e-09, 'early_stopping': 'False', 'learning_rate': 0.001214053793726145, 'iterations': 11569, 'max_depth': 28, 'max_bins': 227, 'min_samples_leaf': 112, 'max_leaf_nodes': 73}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:01:04,605]\u001b[0m Trial 118 finished with value: 0.8658869738517709 and parameters: {'l2_regularization': 8.842689518188001e-09, 'early_stopping': 'False', 'learning_rate': 0.006949779253344256, 'iterations': 8398, 'max_depth': 27, 'max_bins': 177, 'min_samples_leaf': 4932, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:01:35,588]\u001b[0m Trial 119 finished with value: 0.8632663729698321 and parameters: {'l2_regularization': 1.7943402337534328e-10, 'early_stopping': 'False', 'learning_rate': 0.008793071328472023, 'iterations': 10497, 'max_depth': 26, 'max_bins': 181, 'min_samples_leaf': 8936, 'max_leaf_nodes': 31}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:02:41,812]\u001b[0m Trial 120 finished with value: 0.8663868258159044 and parameters: {'l2_regularization': 4.1925102973398566e-10, 'early_stopping': 'False', 'learning_rate': 0.0031816838575695525, 'iterations': 5857, 'max_depth': 25, 'max_bins': 243, 'min_samples_leaf': 2284, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:03:10,116]\u001b[0m Trial 121 finished with value: 0.8668193592051648 and parameters: {'l2_regularization': 1.1784133074987906e-09, 'early_stopping': 'False', 'learning_rate': 0.004958345737766423, 'iterations': 9454, 'max_depth': 28, 'max_bins': 185, 'min_samples_leaf': 451, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:04:22,556]\u001b[0m Trial 122 finished with value: 0.8668681077285123 and parameters: {'l2_regularization': 7.900497802999531e-10, 'early_stopping': 'False', 'learning_rate': 0.002651234146613934, 'iterations': 8689, 'max_depth': 28, 'max_bins': 167, 'min_samples_leaf': 60, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:04:46,618]\u001b[0m Trial 123 finished with value: 0.8663393373421078 and parameters: {'l2_regularization': 0.00023427057031527728, 'early_stopping': 'False', 'learning_rate': 0.015404211153980564, 'iterations': 7307, 'max_depth': 29, 'max_bins': 175, 'min_samples_leaf': 3663, 'max_leaf_nodes': 30}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:05:36,216]\u001b[0m Trial 124 finished with value: 0.8648771211081173 and parameters: {'l2_regularization': 2.505298522847948e-09, 'early_stopping': 'False', 'learning_rate': 0.0044348340946073475, 'iterations': 7924, 'max_depth': 28, 'max_bins': 158, 'min_samples_leaf': 6265, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:06:25,620]\u001b[0m Trial 125 finished with value: 0.8668073801820592 and parameters: {'l2_regularization': 1.1624985499525587e-09, 'early_stopping': 'False', 'learning_rate': 0.004167074335313235, 'iterations': 9197, 'max_depth': 30, 'max_bins': 190, 'min_samples_leaf': 2332, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:06:32,978]\u001b[0m Trial 126 finished with value: 0.8660290116545347 and parameters: {'l2_regularization': 2.467089533576148e-10, 'early_stopping': 'False', 'learning_rate': 0.061966415347199114, 'iterations': 7633, 'max_depth': 27, 'max_bins': 197, 'min_samples_leaf': 5005, 'max_leaf_nodes': 33}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:07:14,821]\u001b[0m Trial 127 finished with value: 0.8669296962137985 and parameters: {'l2_regularization': 5.735229179544785e-10, 'early_stopping': 'False', 'learning_rate': 0.0036641480073274805, 'iterations': 9697, 'max_depth': 24, 'max_bins': 168, 'min_samples_leaf': 109, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:07:38,600]\u001b[0m Trial 128 finished with value: 0.8661558832500349 and parameters: {'l2_regularization': 0.5401057738804664, 'early_stopping': 'False', 'learning_rate': 0.013311863562509107, 'iterations': 10359, 'max_depth': 29, 'max_bins': 163, 'min_samples_leaf': 3571, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:08:11,309]\u001b[0m Trial 129 finished with value: 0.8642427061148004 and parameters: {'l2_regularization': 4.850514309930888e-09, 'early_stopping': 'False', 'learning_rate': 0.009516737821883814, 'iterations': 9989, 'max_depth': 27, 'max_bins': 184, 'min_samples_leaf': 8004, 'max_leaf_nodes': 31}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:09:34,912]\u001b[0m Trial 130 finished with value: 0.8668960454787347 and parameters: {'l2_regularization': 1.5500545800715872e-10, 'early_stopping': 'False', 'learning_rate': 0.00210573574979709, 'iterations': 5381, 'max_depth': 26, 'max_bins': 179, 'min_samples_leaf': 2101, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:10:40,804]\u001b[0m Trial 131 finished with value: 0.8677991075975582 and parameters: {'l2_regularization': 5.323682407643744e-10, 'early_stopping': 'False', 'learning_rate': 0.0029656618418034005, 'iterations': 9663, 'max_depth': 24, 'max_bins': 172, 'min_samples_leaf': 229, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:11:56,977]\u001b[0m Trial 132 finished with value: 0.8658939183782549 and parameters: {'l2_regularization': 3.567368334616844e-10, 'early_stopping': 'False', 'learning_rate': 0.0032280623199705902, 'iterations': 8879, 'max_depth': 28, 'max_bins': 172, 'min_samples_leaf': 4773, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:12:36,254]\u001b[0m Trial 133 finished with value: 0.8669483688937941 and parameters: {'l2_regularization': 1.089828567534041e-09, 'early_stopping': 'False', 'learning_rate': 0.004572069995639154, 'iterations': 9243, 'max_depth': 25, 'max_bins': 187, 'min_samples_leaf': 34, 'max_leaf_nodes': 21}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:13:16,578]\u001b[0m Trial 134 finished with value: 0.866207414145241 and parameters: {'l2_regularization': 2.155774089466099e-09, 'early_stopping': 'False', 'learning_rate': 0.006132036929110973, 'iterations': 6130, 'max_depth': 25, 'max_bins': 187, 'min_samples_leaf': 2680, 'max_leaf_nodes': 21}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:14:49,000]\u001b[0m Trial 135 finished with value: 0.8648393709358271 and parameters: {'l2_regularization': 4.5602067449719816e-10, 'early_stopping': 'False', 'learning_rate': 0.0027872053058196062, 'iterations': 4978, 'max_depth': 24, 'max_bins': 193, 'min_samples_leaf': 6651, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:15:17,565]\u001b[0m Trial 136 finished with value: 0.8657834901443142 and parameters: {'l2_regularization': 2.331899395543657e-10, 'early_stopping': 'False', 'learning_rate': 0.0053103524665657174, 'iterations': 9061, 'max_depth': 23, 'max_bins': 181, 'min_samples_leaf': 1755, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:15:40,938]\u001b[0m Trial 137 finished with value: 0.865736457797052 and parameters: {'l2_regularization': 7.723073082186216e-10, 'early_stopping': 'False', 'learning_rate': 0.010801593390198243, 'iterations': 6941, 'max_depth': 26, 'max_bins': 188, 'min_samples_leaf': 4328, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:16:08,706]\u001b[0m Trial 138 finished with value: 0.8673839868387425 and parameters: {'l2_regularization': 1.3093744989373397e-09, 'early_stopping': 'False', 'learning_rate': 0.00792650130853679, 'iterations': 8269, 'max_depth': 25, 'max_bins': 206, 'min_samples_leaf': 120, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:16:10,206]\u001b[0m Trial 139 finished with value: 0.5 and parameters: {'l2_regularization': 3.435696186906543e-09, 'early_stopping': 'False', 'learning_rate': 0.00804430097504143, 'iterations': 8382, 'max_depth': 22, 'max_bins': 204, 'min_samples_leaf': 79241, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:16:46,905]\u001b[0m Trial 140 finished with value: 0.8655329854517074 and parameters: {'l2_regularization': 1.417817101833145e-10, 'early_stopping': 'False', 'learning_rate': 0.01055853648422352, 'iterations': 7178, 'max_depth': 23, 'max_bins': 211, 'min_samples_leaf': 7071, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:17:21,095]\u001b[0m Trial 141 finished with value: 0.866879374053908 and parameters: {'l2_regularization': 2.470151792061249e-06, 'early_stopping': 'False', 'learning_rate': 0.005958712807651738, 'iterations': 8227, 'max_depth': 25, 'max_bins': 218, 'min_samples_leaf': 2007, 'max_leaf_nodes': 20}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:19:34,473]\u001b[0m Trial 142 finished with value: 0.8677946489606861 and parameters: {'l2_regularization': 1.3925251756269873e-09, 'early_stopping': 'False', 'learning_rate': 0.0015595103038148635, 'iterations': 7821, 'max_depth': 24, 'max_bins': 199, 'min_samples_leaf': 293, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:21:30,480]\u001b[0m Trial 143 finished with value: 0.8655165991059647 and parameters: {'l2_regularization': 1.4160165345184556e-09, 'early_stopping': 'False', 'learning_rate': 0.001629995595171236, 'iterations': 8014, 'max_depth': 17, 'max_bins': 197, 'min_samples_leaf': 3251, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:24:17,926]\u001b[0m Trial 144 finished with value: 0.8662772300128887 and parameters: {'l2_regularization': 5.902103158559599e-10, 'early_stopping': 'False', 'learning_rate': 0.0013898373363621314, 'iterations': 7493, 'max_depth': 24, 'max_bins': 200, 'min_samples_leaf': 5073, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:25:51,490]\u001b[0m Trial 145 finished with value: 0.8666337328104724 and parameters: {'l2_regularization': 3.072760624515967e-10, 'early_stopping': 'False', 'learning_rate': 0.001921925257161755, 'iterations': 6353, 'max_depth': 26, 'max_bins': 202, 'min_samples_leaf': 1547, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:27:30,521]\u001b[0m Trial 146 finished with value: 0.8664046831697201 and parameters: {'l2_regularization': 2.5199938949280628e-09, 'early_stopping': 'False', 'learning_rate': 0.0016968130604551426, 'iterations': 7709, 'max_depth': 27, 'max_bins': 209, 'min_samples_leaf': 67, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:28:20,889]\u001b[0m Trial 147 finished with value: 0.8639870814018465 and parameters: {'l2_regularization': 1.5325282125754014e-09, 'early_stopping': 'False', 'learning_rate': 0.007276092124727009, 'iterations': 8624, 'max_depth': 24, 'max_bins': 134, 'min_samples_leaf': 9719, 'max_leaf_nodes': 29}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:31:51,293]\u001b[0m Trial 148 finished with value: 0.8662778685900367 and parameters: {'l2_regularization': 6.514573298540782e-10, 'early_stopping': 'False', 'learning_rate': 0.0011548304836847191, 'iterations': 6718, 'max_depth': 20, 'max_bins': 195, 'min_samples_leaf': 3548, 'max_leaf_nodes': 32}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:32:13,279]\u001b[0m Trial 149 finished with value: 0.8652098255037769 and parameters: {'l2_regularization': 1.0015564517532939e-10, 'early_stopping': 'False', 'learning_rate': 0.012325300898832573, 'iterations': 5632, 'max_depth': 29, 'max_bins': 206, 'min_samples_leaf': 5874, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:32:30,811]\u001b[0m Trial 150 finished with value: 0.8676055617059151 and parameters: {'l2_regularization': 6.8508720107748656e-09, 'early_stopping': 'False', 'learning_rate': 0.016362369188899828, 'iterations': 1032, 'max_depth': 30, 'max_bins': 215, 'min_samples_leaf': 2174, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:32:43,231]\u001b[0m Trial 151 finished with value: 0.8671334479366456 and parameters: {'l2_regularization': 7.453798848884729e-09, 'early_stopping': 'False', 'learning_rate': 0.016551228849884074, 'iterations': 1778, 'max_depth': 30, 'max_bins': 220, 'min_samples_leaf': 1800, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:32:57,143]\u001b[0m Trial 152 finished with value: 0.8672189203475813 and parameters: {'l2_regularization': 2.5935223875822772e-08, 'early_stopping': 'False', 'learning_rate': 0.016115202544263466, 'iterations': 1234, 'max_depth': 30, 'max_bins': 215, 'min_samples_leaf': 105, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:33:15,792]\u001b[0m Trial 153 finished with value: 0.8660016383608983 and parameters: {'l2_regularization': 3.209721535368583e-08, 'early_stopping': 'False', 'learning_rate': 0.01438878283415154, 'iterations': 2225, 'max_depth': 30, 'max_bins': 219, 'min_samples_leaf': 3493, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:33:30,086]\u001b[0m Trial 154 finished with value: 0.8663987079121215 and parameters: {'l2_regularization': 1.1556665283375492e-08, 'early_stopping': 'False', 'learning_rate': 0.016926366764351147, 'iterations': 1602, 'max_depth': 29, 'max_bins': 216, 'min_samples_leaf': 2171, 'max_leaf_nodes': 26}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:33:46,209]\u001b[0m Trial 155 finished with value: 0.8656206187620992 and parameters: {'l2_regularization': 6.686788288677823e-09, 'early_stopping': 'False', 'learning_rate': 0.016296460287950534, 'iterations': 1071, 'max_depth': 30, 'max_bins': 223, 'min_samples_leaf': 5502, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:34:06,292]\u001b[0m Trial 156 finished with value: 0.8643359953942166 and parameters: {'l2_regularization': 3.2689275509299134e-09, 'early_stopping': 'False', 'learning_rate': 0.020103685495591238, 'iterations': 1226, 'max_depth': 30, 'max_bins': 226, 'min_samples_leaf': 8321, 'max_leaf_nodes': 28}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:34:26,543]\u001b[0m Trial 157 finished with value: 0.8661503584173889 and parameters: {'l2_regularization': 7.090492117574632e-09, 'early_stopping': 'False', 'learning_rate': 0.01331287686048838, 'iterations': 7739, 'max_depth': 29, 'max_bins': 215, 'min_samples_leaf': 3542, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:34:46,387]\u001b[0m Trial 158 finished with value: 0.8676363103359047 and parameters: {'l2_regularization': 4.755603133488136e-09, 'early_stopping': 'False', 'learning_rate': 0.012016402310804402, 'iterations': 1820, 'max_depth': 30, 'max_bins': 212, 'min_samples_leaf': 193, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:35:04,865]\u001b[0m Trial 159 finished with value: 0.8669036513886932 and parameters: {'l2_regularization': 2.1567570934077772e-08, 'early_stopping': 'False', 'learning_rate': 0.011955859968435371, 'iterations': 2434, 'max_depth': 30, 'max_bins': 211, 'min_samples_leaf': 1395, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:35:25,504]\u001b[0m Trial 160 finished with value: 0.867231834430082 and parameters: {'l2_regularization': 1.1020635229099656e-07, 'early_stopping': 'False', 'learning_rate': 0.008688524053376293, 'iterations': 1848, 'max_depth': 30, 'max_bins': 212, 'min_samples_leaf': 100, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:35:40,946]\u001b[0m Trial 161 finished with value: 0.8673590994347178 and parameters: {'l2_regularization': 1.2246219477444892e-08, 'early_stopping': 'False', 'learning_rate': 0.015011216120050146, 'iterations': 1978, 'max_depth': 30, 'max_bins': 212, 'min_samples_leaf': 59, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:36:02,222]\u001b[0m Trial 162 finished with value: 0.866677042024899 and parameters: {'l2_regularization': 2.6735535180644906e-07, 'early_stopping': 'False', 'learning_rate': 0.008423085731040506, 'iterations': 1678, 'max_depth': 30, 'max_bins': 213, 'min_samples_leaf': 1851, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:36:31,088]\u001b[0m Trial 163 finished with value: 0.8674676803561947 and parameters: {'l2_regularization': 7.057849727931641e-08, 'early_stopping': 'False', 'learning_rate': 0.008846560556910165, 'iterations': 1718, 'max_depth': 30, 'max_bins': 206, 'min_samples_leaf': 235, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:36:55,505]\u001b[0m Trial 164 finished with value: 0.8655732443199361 and parameters: {'l2_regularization': 2.9301618221490878e-08, 'early_stopping': 'False', 'learning_rate': 0.009237800840145837, 'iterations': 1925, 'max_depth': 29, 'max_bins': 207, 'min_samples_leaf': 4338, 'max_leaf_nodes': 24}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:37:18,776]\u001b[0m Trial 165 finished with value: 0.867056328342868 and parameters: {'l2_regularization': 8.267928224579108e-08, 'early_stopping': 'False', 'learning_rate': 0.007715746331582354, 'iterations': 2488, 'max_depth': 30, 'max_bins': 213, 'min_samples_leaf': 464, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:37:44,362]\u001b[0m Trial 166 finished with value: 0.866570302714304 and parameters: {'l2_regularization': 2.115656690987243e-07, 'early_stopping': 'False', 'learning_rate': 0.010948917119974071, 'iterations': 1530, 'max_depth': 30, 'max_bins': 209, 'min_samples_leaf': 3976, 'max_leaf_nodes': 22}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:38:05,500]\u001b[0m Trial 167 finished with value: 0.8674733306236376 and parameters: {'l2_regularization': 2.3350311440061698e-09, 'early_stopping': 'False', 'learning_rate': 0.008408796411123494, 'iterations': 2919, 'max_depth': 29, 'max_bins': 207, 'min_samples_leaf': 111, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:38:27,312]\u001b[0m Trial 168 finished with value: 0.8667949564355828 and parameters: {'l2_regularization': 5.590705942408304e-08, 'early_stopping': 'False', 'learning_rate': 0.00818650569112602, 'iterations': 1028, 'max_depth': 29, 'max_bins': 207, 'min_samples_leaf': 108, 'max_leaf_nodes': 25}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:38:28,873]\u001b[0m Trial 169 finished with value: 0.5 and parameters: {'l2_regularization': 1.5284612842034758e-08, 'early_stopping': 'False', 'learning_rate': 0.009726960508012313, 'iterations': 2798, 'max_depth': 29, 'max_bins': 205, 'min_samples_leaf': 90696, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:38:34,472]\u001b[0m Trial 170 finished with value: 0.8651957882096856 and parameters: {'l2_regularization': 1.0112193028884949e-08, 'early_stopping': 'False', 'learning_rate': 0.07662964855223944, 'iterations': 2524, 'max_depth': 30, 'max_bins': 215, 'min_samples_leaf': 6240, 'max_leaf_nodes': 23}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:39:00,476]\u001b[0m Trial 171 finished with value: 0.8664151512736809 and parameters: {'l2_regularization': 4.4101148105782236e-09, 'early_stopping': 'False', 'learning_rate': 0.008618892715562935, 'iterations': 3083, 'max_depth': 29, 'max_bins': 212, 'min_samples_leaf': 2772, 'max_leaf_nodes': 27}. Best is trial 32 with value: 0.8679321597075973.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:39:26,013]\u001b[0m Trial 172 finished with value: 0.8681191259739842 and parameters: {'l2_regularization': 1.939640534964219e-08, 'early_stopping': 'False', 'learning_rate': 0.006683250049408931, 'iterations': 1841, 'max_depth': 28, 'max_bins': 114, 'min_samples_leaf': 50, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:39:47,044]\u001b[0m Trial 173 finished with value: 0.8672144845170356 and parameters: {'l2_regularization': 2.6721739409556532e-08, 'early_stopping': 'False', 'learning_rate': 0.007367033959021827, 'iterations': 1312, 'max_depth': 29, 'max_bins': 107, 'min_samples_leaf': 153, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:40:08,842]\u001b[0m Trial 174 finished with value: 0.8650234065892314 and parameters: {'l2_regularization': 3.1451147226277234e-08, 'early_stopping': 'False', 'learning_rate': 0.008824880782391164, 'iterations': 1351, 'max_depth': 30, 'max_bins': 101, 'min_samples_leaf': 3408, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:40:40,249]\u001b[0m Trial 175 finished with value: 0.8662795562582135 and parameters: {'l2_regularization': 2.3991313173907733e-08, 'early_stopping': 'False', 'learning_rate': 0.006987713982910214, 'iterations': 1917, 'max_depth': 29, 'max_bins': 110, 'min_samples_leaf': 1927, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:41:16,117]\u001b[0m Trial 176 finished with value: 0.8660184352205219 and parameters: {'l2_regularization': 1.7228124423694602e-07, 'early_stopping': 'False', 'learning_rate': 0.007363822380718292, 'iterations': 1686, 'max_depth': 28, 'max_bins': 108, 'min_samples_leaf': 4974, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:41:44,900]\u001b[0m Trial 177 finished with value: 0.8673105447651476 and parameters: {'l2_regularization': 6.174303123239568e-07, 'early_stopping': 'False', 'learning_rate': 0.006641907908360416, 'iterations': 1370, 'max_depth': 30, 'max_bins': 115, 'min_samples_leaf': 104, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:41:46,386]\u001b[0m Trial 178 finished with value: 0.5 and parameters: {'l2_regularization': 4.1745868391596827e-07, 'early_stopping': 'False', 'learning_rate': 0.0066333495777064835, 'iterations': 1367, 'max_depth': 30, 'max_bins': 119, 'min_samples_leaf': 58606, 'max_leaf_nodes': 67}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:42:11,984]\u001b[0m Trial 179 finished with value: 0.8664615222374684 and parameters: {'l2_regularization': 9.387455412861994e-08, 'early_stopping': 'False', 'learning_rate': 0.006479046396000419, 'iterations': 2321, 'max_depth': 30, 'max_bins': 115, 'min_samples_leaf': 68, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:42:43,059]\u001b[0m Trial 180 finished with value: 0.8660770759880807 and parameters: {'l2_regularization': 1.5150642420101636e-07, 'early_stopping': 'False', 'learning_rate': 0.007730725203399983, 'iterations': 2007, 'max_depth': 29, 'max_bins': 104, 'min_samples_leaf': 2855, 'max_leaf_nodes': 21}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:43:03,644]\u001b[0m Trial 181 finished with value: 0.8676328608789897 and parameters: {'l2_regularization': 5.0502877162172276e-08, 'early_stopping': 'False', 'learning_rate': 0.009713923156249759, 'iterations': 1190, 'max_depth': 29, 'max_bins': 118, 'min_samples_leaf': 118, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:43:21,001]\u001b[0m Trial 182 finished with value: 0.8661783759897548 and parameters: {'l2_regularization': 4.948774113901246e-08, 'early_stopping': 'False', 'learning_rate': 0.01008810902398476, 'iterations': 1041, 'max_depth': 30, 'max_bins': 113, 'min_samples_leaf': 2067, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:43:42,346]\u001b[0m Trial 183 finished with value: 0.8661494575674835 and parameters: {'l2_regularization': 1.6189430390305508e-08, 'early_stopping': 'False', 'learning_rate': 0.009475045892254393, 'iterations': 1583, 'max_depth': 28, 'max_bins': 117, 'min_samples_leaf': 4439, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:44:02,858]\u001b[0m Trial 184 finished with value: 0.8669528560385749 and parameters: {'l2_regularization': 8.424023882863938e-08, 'early_stopping': 'False', 'learning_rate': 0.008308067352734897, 'iterations': 1322, 'max_depth': 5, 'max_bins': 108, 'min_samples_leaf': 33, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:44:20,507]\u001b[0m Trial 185 finished with value: 0.8673700293667946 and parameters: {'l2_regularization': 3.2525296809176123e-08, 'early_stopping': 'False', 'learning_rate': 0.011278867568496777, 'iterations': 1815, 'max_depth': 29, 'max_bins': 125, 'min_samples_leaf': 45, 'max_leaf_nodes': 22}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:44:40,614]\u001b[0m Trial 186 finished with value: 0.8666261211989323 and parameters: {'l2_regularization': 4.1791235116759433e-07, 'early_stopping': 'False', 'learning_rate': 0.01059022247256319, 'iterations': 1885, 'max_depth': 29, 'max_bins': 112, 'min_samples_leaf': 2332, 'max_leaf_nodes': 22}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:45:00,547]\u001b[0m Trial 187 finished with value: 0.8648243814775955 and parameters: {'l2_regularization': 4.795120098135753e-08, 'early_stopping': 'False', 'learning_rate': 0.011250587234194837, 'iterations': 2128, 'max_depth': 28, 'max_bins': 122, 'min_samples_leaf': 6420, 'max_leaf_nodes': 20}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:46:01,507]\u001b[0m Trial 188 finished with value: 0.8652962614820161 and parameters: {'l2_regularization': 1.034067925259656e-06, 'early_stopping': 'False', 'learning_rate': 0.0015797937280150128, 'iterations': 1662, 'max_depth': 30, 'max_bins': 121, 'min_samples_leaf': 39, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:46:50,372]\u001b[0m Trial 189 finished with value: 0.8593433711883387 and parameters: {'l2_regularization': 1.2110595804568155e-07, 'early_stopping': 'False', 'learning_rate': 0.011692449210650954, 'iterations': 2647, 'max_depth': 30, 'max_bins': 126, 'min_samples_leaf': 28975, 'max_leaf_nodes': 22}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:47:19,332]\u001b[0m Trial 190 finished with value: 0.8508257384084221 and parameters: {'l2_regularization': 1.3129768518202192e-08, 'early_stopping': 'False', 'learning_rate': 0.014454664647668935, 'iterations': 1831, 'max_depth': 28, 'max_bins': 201, 'min_samples_leaf': 39331, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:47:43,498]\u001b[0m Trial 191 finished with value: 0.8657637455669632 and parameters: {'l2_regularization': 3.9082656556422464e-08, 'early_stopping': 'False', 'learning_rate': 0.006965764606099188, 'iterations': 1375, 'max_depth': 29, 'max_bins': 130, 'min_samples_leaf': 1925, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:48:18,692]\u001b[0m Trial 192 finished with value: 0.8667388471702892 and parameters: {'l2_regularization': 2.4241656023184657e-08, 'early_stopping': 'False', 'learning_rate': 0.009478253659092465, 'iterations': 1143, 'max_depth': 29, 'max_bins': 110, 'min_samples_leaf': 3588, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:48:34,183]\u001b[0m Trial 193 finished with value: 0.8661846420280188 and parameters: {'l2_regularization': 6.325170287215915e-08, 'early_stopping': 'False', 'learning_rate': 0.012961313230100982, 'iterations': 1590, 'max_depth': 29, 'max_bins': 117, 'min_samples_leaf': 1526, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:48:58,977]\u001b[0m Trial 194 finished with value: 0.8667974423251944 and parameters: {'l2_regularization': 1.809829069650242e-08, 'early_stopping': 'False', 'learning_rate': 0.0088576887880874, 'iterations': 2233, 'max_depth': 30, 'max_bins': 210, 'min_samples_leaf': 36, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:49:20,954]\u001b[0m Trial 195 finished with value: 0.8673306941547977 and parameters: {'l2_regularization': 1.0037457760272293e-08, 'early_stopping': 'False', 'learning_rate': 0.007829419506146169, 'iterations': 1060, 'max_depth': 28, 'max_bins': 105, 'min_samples_leaf': 38, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:49:49,320]\u001b[0m Trial 196 finished with value: 0.8655235094229585 and parameters: {'l2_regularization': 9.456064472409618e-09, 'early_stopping': 'False', 'learning_rate': 0.010382528152254752, 'iterations': 1013, 'max_depth': 27, 'max_bins': 204, 'min_samples_leaf': 3816, 'max_leaf_nodes': 55}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:50:13,753]\u001b[0m Trial 197 finished with value: 0.8669196557284634 and parameters: {'l2_regularization': 4.452645255078705e-09, 'early_stopping': 'False', 'learning_rate': 0.007705514198925769, 'iterations': 1722, 'max_depth': 28, 'max_bins': 125, 'min_samples_leaf': 2185, 'max_leaf_nodes': 29}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:50:33,692]\u001b[0m Trial 198 finished with value: 0.8657556778288888 and parameters: {'l2_regularization': 6.362856467419673e-09, 'early_stopping': 'False', 'learning_rate': 0.0116157009616786, 'iterations': 1409, 'max_depth': 28, 'max_bins': 200, 'min_samples_leaf': 4681, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:50:56,821]\u001b[0m Trial 199 finished with value: 0.8664985198009774 and parameters: {'l2_regularization': 3.0178619573380672e-09, 'early_stopping': 'False', 'learning_rate': 0.008404896066448496, 'iterations': 2000, 'max_depth': 30, 'max_bins': 115, 'min_samples_leaf': 2538, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:51:11,694]\u001b[0m Trial 200 finished with value: 0.8654968659317769 and parameters: {'l2_regularization': 1.0192142304132892e-08, 'early_stopping': 'False', 'learning_rate': 0.018525612316655272, 'iterations': 3234, 'max_depth': 29, 'max_bins': 105, 'min_samples_leaf': 5961, 'max_leaf_nodes': 80}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:51:33,691]\u001b[0m Trial 201 finished with value: 0.8668280199077337 and parameters: {'l2_regularization': 2.4470017327815573e-08, 'early_stopping': 'False', 'learning_rate': 0.0072727841905787525, 'iterations': 1221, 'max_depth': 29, 'max_bins': 108, 'min_samples_leaf': 130, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:52:02,298]\u001b[0m Trial 202 finished with value: 0.8672551424959818 and parameters: {'l2_regularization': 3.8764197393969935e-08, 'early_stopping': 'False', 'learning_rate': 0.006549820323555106, 'iterations': 1470, 'max_depth': 29, 'max_bins': 103, 'min_samples_leaf': 75, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:52:27,243]\u001b[0m Trial 203 finished with value: 0.8656146263997555 and parameters: {'l2_regularization': 4.2898071734509144e-08, 'early_stopping': 'False', 'learning_rate': 0.006230405901409756, 'iterations': 1532, 'max_depth': 30, 'max_bins': 218, 'min_samples_leaf': 2001, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:52:43,378]\u001b[0m Trial 204 finished with value: 0.8408468301554921 and parameters: {'l2_regularization': 6.910987193895108e-08, 'early_stopping': 'False', 'learning_rate': 0.009415040775800305, 'iterations': 1021, 'max_depth': 28, 'max_bins': 103, 'min_samples_leaf': 45969, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:52:57,429]\u001b[0m Trial 205 finished with value: 0.8661756620368759 and parameters: {'l2_regularization': 1.650937180568092e-08, 'early_stopping': 'False', 'learning_rate': 0.014351306876127684, 'iterations': 1895, 'max_depth': 30, 'max_bins': 100, 'min_samples_leaf': 1894, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:53:30,175]\u001b[0m Trial 206 finished with value: 0.8669782166738788 and parameters: {'l2_regularization': 2.1211252833924556e-09, 'early_stopping': 'False', 'learning_rate': 0.0052323388588396596, 'iterations': 1508, 'max_depth': 29, 'max_bins': 112, 'min_samples_leaf': 58, 'max_leaf_nodes': 30}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:54:12,714]\u001b[0m Trial 207 finished with value: 0.8656869395601782 and parameters: {'l2_regularization': 5.1053122190085405e-09, 'early_stopping': 'False', 'learning_rate': 0.00646367743071256, 'iterations': 2294, 'max_depth': 27, 'max_bins': 209, 'min_samples_leaf': 3747, 'max_leaf_nodes': 47}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:54:43,733]\u001b[0m Trial 208 finished with value: 0.8657883935045575 and parameters: {'l2_regularization': 1.1502283829919274e-07, 'early_stopping': 'False', 'learning_rate': 0.008030839403747287, 'iterations': 1844, 'max_depth': 30, 'max_bins': 206, 'min_samples_leaf': 3459, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:55:30,328]\u001b[0m Trial 209 finished with value: 0.8633448894511206 and parameters: {'l2_regularization': 1.985553476863951e-05, 'early_stopping': 'False', 'learning_rate': 0.0012654263755393432, 'iterations': 1333, 'max_depth': 28, 'max_bins': 121, 'min_samples_leaf': 23, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:55:52,375]\u001b[0m Trial 210 finished with value: 0.86552935924576 and parameters: {'l2_regularization': 3.889131146157785e-08, 'early_stopping': 'False', 'learning_rate': 0.012513069421145469, 'iterations': 1607, 'max_depth': 29, 'max_bins': 214, 'min_samples_leaf': 5271, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:56:15,171]\u001b[0m Trial 211 finished with value: 0.8672292915246536 and parameters: {'l2_regularization': 2.4259901529439626e-08, 'early_stopping': 'False', 'learning_rate': 0.007545904251249012, 'iterations': 1256, 'max_depth': 29, 'max_bins': 106, 'min_samples_leaf': 38, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:56:47,354]\u001b[0m Trial 212 finished with value: 0.866848311836926 and parameters: {'l2_regularization': 1.3723484793415363e-08, 'early_stopping': 'False', 'learning_rate': 0.006763443544648465, 'iterations': 1228, 'max_depth': 29, 'max_bins': 221, 'min_samples_leaf': 2062, 'max_leaf_nodes': 21}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:57:11,218]\u001b[0m Trial 213 finished with value: 0.8673422113497874 and parameters: {'l2_regularization': 7.583290863880608e-08, 'early_stopping': 'False', 'learning_rate': 0.008953354261219553, 'iterations': 2146, 'max_depth': 29, 'max_bins': 105, 'min_samples_leaf': 1989, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:57:34,885]\u001b[0m Trial 214 finished with value: 0.866554720291578 and parameters: {'l2_regularization': 6.338541105175811e-08, 'early_stopping': 'False', 'learning_rate': 0.00890617770663421, 'iterations': 2198, 'max_depth': 28, 'max_bins': 110, 'min_samples_leaf': 2044, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:58:09,641]\u001b[0m Trial 215 finished with value: 0.8665238120173013 and parameters: {'l2_regularization': 2.441913342088802e-07, 'early_stopping': 'False', 'learning_rate': 0.0076322758462034285, 'iterations': 2684, 'max_depth': 29, 'max_bins': 107, 'min_samples_leaf': 3583, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:58:37,324]\u001b[0m Trial 216 finished with value: 0.8666076993885304 and parameters: {'l2_regularization': 3.453059946666813e-06, 'early_stopping': 'False', 'learning_rate': 0.009808631646330645, 'iterations': 2081, 'max_depth': 27, 'max_bins': 104, 'min_samples_leaf': 1974, 'max_leaf_nodes': 22}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:58:56,811]\u001b[0m Trial 217 finished with value: 0.8670307168379703 and parameters: {'l2_regularization': 1.4841291722911087e-09, 'early_stopping': 'False', 'learning_rate': 0.008310797091269959, 'iterations': 1697, 'max_depth': 28, 'max_bins': 100, 'min_samples_leaf': 77, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:59:20,531]\u001b[0m Trial 218 finished with value: 0.8663340291695655 and parameters: {'l2_regularization': 8.498026422623479e-09, 'early_stopping': 'False', 'learning_rate': 0.010742546530580963, 'iterations': 2462, 'max_depth': 29, 'max_bins': 113, 'min_samples_leaf': 5069, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:59:53,906]\u001b[0m Trial 219 finished with value: 0.8670167992770942 and parameters: {'l2_regularization': 1.3915327638590577e-07, 'early_stopping': 'False', 'learning_rate': 0.00559921967904647, 'iterations': 1923, 'max_depth': 30, 'max_bins': 104, 'min_samples_leaf': 1925, 'max_leaf_nodes': 22}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:00:24,398]\u001b[0m Trial 220 finished with value: 0.8651138051667369 and parameters: {'l2_regularization': 6.986864325822358e-08, 'early_stopping': 'False', 'learning_rate': 0.008646761715979067, 'iterations': 1034, 'max_depth': 28, 'max_bins': 198, 'min_samples_leaf': 7140, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:00:52,555]\u001b[0m Trial 221 finished with value: 0.8672364641144046 and parameters: {'l2_regularization': 3.536318579080412e-08, 'early_stopping': 'False', 'learning_rate': 0.00797418208389235, 'iterations': 1522, 'max_depth': 30, 'max_bins': 211, 'min_samples_leaf': 250, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:01:18,290]\u001b[0m Trial 222 finished with value: 0.867198679732624 and parameters: {'l2_regularization': 3.0401381474866385e-08, 'early_stopping': 'False', 'learning_rate': 0.007649895818253464, 'iterations': 1573, 'max_depth': 30, 'max_bins': 203, 'min_samples_leaf': 26, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:01:50,962]\u001b[0m Trial 223 finished with value: 0.8661411503629788 and parameters: {'l2_regularization': 6.944769922871355e-07, 'early_stopping': 'False', 'learning_rate': 0.007211168150868971, 'iterations': 13237, 'max_depth': 29, 'max_bins': 211, 'min_samples_leaf': 3243, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:02:08,177]\u001b[0m Trial 224 finished with value: 0.8661708099908678 and parameters: {'l2_regularization': 1.7893710586162398e-08, 'early_stopping': 'False', 'learning_rate': 0.009200262790375032, 'iterations': 1470, 'max_depth': 29, 'max_bins': 207, 'min_samples_leaf': 1813, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:03:10,793]\u001b[0m Trial 225 finished with value: 0.8623031078500698 and parameters: {'l2_regularization': 4.9882444221119995e-08, 'early_stopping': 'False', 'learning_rate': 0.0014638583500381382, 'iterations': 1802, 'max_depth': 30, 'max_bins': 116, 'min_samples_leaf': 3616, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:03:41,670]\u001b[0m Trial 226 finished with value: 0.8668720190135433 and parameters: {'l2_regularization': 2.6149528307757167e-09, 'early_stopping': 'False', 'learning_rate': 0.006595804076127283, 'iterations': 2223, 'max_depth': 30, 'max_bins': 138, 'min_samples_leaf': 1804, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:04:02,381]\u001b[0m Trial 227 finished with value: 0.8668866948847826 and parameters: {'l2_regularization': 1.097029475805312e-09, 'early_stopping': 'False', 'learning_rate': 0.010225172810888201, 'iterations': 1014, 'max_depth': 29, 'max_bins': 119, 'min_samples_leaf': 1620, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:04:24,581]\u001b[0m Trial 228 finished with value: 0.867174664670596 and parameters: {'l2_regularization': 4.054292886958876e-08, 'early_stopping': 'False', 'learning_rate': 0.008335279700370008, 'iterations': 1407, 'max_depth': 30, 'max_bins': 102, 'min_samples_leaf': 22, 'max_leaf_nodes': 29}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:04:59,069]\u001b[0m Trial 229 finished with value: 0.8664113768266102 and parameters: {'l2_regularization': 4.132166294083322e-09, 'early_stopping': 'False', 'learning_rate': 0.008082988752759824, 'iterations': 2004, 'max_depth': 28, 'max_bins': 107, 'min_samples_leaf': 4054, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:05:15,185]\u001b[0m Trial 230 finished with value: 0.8666964958215844 and parameters: {'l2_regularization': 9.906044634400216e-08, 'early_stopping': 'False', 'learning_rate': 0.009434118408203187, 'iterations': 14179, 'max_depth': 27, 'max_bins': 111, 'min_samples_leaf': 36, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:05:30,575]\u001b[0m Trial 231 finished with value: 0.8676148381793053 and parameters: {'l2_regularization': 2.295667465009779e-08, 'early_stopping': 'False', 'learning_rate': 0.015577128810897418, 'iterations': 1263, 'max_depth': 30, 'max_bins': 211, 'min_samples_leaf': 91, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:05:45,177]\u001b[0m Trial 232 finished with value: 0.8665823501563898 and parameters: {'l2_regularization': 1.0852137879730137e-08, 'early_stopping': 'False', 'learning_rate': 0.013853386018907229, 'iterations': 1672, 'max_depth': 30, 'max_bins': 212, 'min_samples_leaf': 1880, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:06:12,401]\u001b[0m Trial 233 finished with value: 0.8661411047503251 and parameters: {'l2_regularization': 2.044309146172117e-08, 'early_stopping': 'False', 'learning_rate': 0.006108775666521929, 'iterations': 1278, 'max_depth': 29, 'max_bins': 217, 'min_samples_leaf': 2028, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:06:35,578]\u001b[0m Trial 234 finished with value: 0.8661084460904732 and parameters: {'l2_regularization': 4.167825472162115e-08, 'early_stopping': 'False', 'learning_rate': 0.011728989204003299, 'iterations': 1325, 'max_depth': 29, 'max_bins': 206, 'min_samples_leaf': 3761, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:07:02,523]\u001b[0m Trial 235 finished with value: 0.8675325358477824 and parameters: {'l2_regularization': 6.855342718725313e-09, 'early_stopping': 'False', 'learning_rate': 0.007062626744137217, 'iterations': 1804, 'max_depth': 30, 'max_bins': 129, 'min_samples_leaf': 143, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:07:22,325]\u001b[0m Trial 236 finished with value: 0.8666712720242408 and parameters: {'l2_regularization': 4.780382342671339e-09, 'early_stopping': 'False', 'learning_rate': 0.01319618247549134, 'iterations': 1889, 'max_depth': 30, 'max_bins': 208, 'min_samples_leaf': 1940, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:07:51,984]\u001b[0m Trial 237 finished with value: 0.8655025561102916 and parameters: {'l2_regularization': 8.302989082721677e-09, 'early_stopping': 'False', 'learning_rate': 0.007025209915460557, 'iterations': 2360, 'max_depth': 30, 'max_bins': 127, 'min_samples_leaf': 5088, 'max_leaf_nodes': 30}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:09:08,381]\u001b[0m Trial 238 finished with value: 0.866837775313985 and parameters: {'l2_regularization': 1.8207099253074266e-09, 'early_stopping': 'False', 'learning_rate': 0.0021235951900322645, 'iterations': 3212, 'max_depth': 24, 'max_bins': 129, 'min_samples_leaf': 47, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:09:28,814]\u001b[0m Trial 239 finished with value: 0.86581561855707 and parameters: {'l2_regularization': 7.800970640071904e-10, 'early_stopping': 'False', 'learning_rate': 0.015140283303770647, 'iterations': 1707, 'max_depth': 30, 'max_bins': 211, 'min_samples_leaf': 3208, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:09:49,108]\u001b[0m Trial 240 finished with value: 0.8666533405498632 and parameters: {'l2_regularization': 2.515970685612169e-09, 'early_stopping': 'False', 'learning_rate': 0.010120007008110764, 'iterations': 2125, 'max_depth': 30, 'max_bins': 132, 'min_samples_leaf': 1781, 'max_leaf_nodes': 25}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:10:11,662]\u001b[0m Trial 241 finished with value: 0.8678332885797182 and parameters: {'l2_regularization': 1.6504656104522262e-08, 'early_stopping': 'False', 'learning_rate': 0.007793822587997274, 'iterations': 1444, 'max_depth': 29, 'max_bins': 121, 'min_samples_leaf': 259, 'max_leaf_nodes': 24}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:10:31,998]\u001b[0m Trial 242 finished with value: 0.8669779829090299 and parameters: {'l2_regularization': 1.1288404079291224e-08, 'early_stopping': 'False', 'learning_rate': 0.007967780444313597, 'iterations': 1571, 'max_depth': 29, 'max_bins': 122, 'min_samples_leaf': 139, 'max_leaf_nodes': 26}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:10:56,175]\u001b[0m Trial 243 finished with value: 0.8667296847285324 and parameters: {'l2_regularization': 1.6049446835255593e-08, 'early_stopping': 'False', 'learning_rate': 0.008946769707045799, 'iterations': 4514, 'max_depth': 29, 'max_bins': 120, 'min_samples_leaf': 2087, 'max_leaf_nodes': 22}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:11:28,648]\u001b[0m Trial 244 finished with value: 0.8675505528458851 and parameters: {'l2_regularization': 6.369236515798644e-09, 'early_stopping': 'False', 'learning_rate': 0.006665200397019193, 'iterations': 1458, 'max_depth': 30, 'max_bins': 122, 'min_samples_leaf': 75, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:12:02,842]\u001b[0m Trial 245 finished with value: 0.8655116615362315 and parameters: {'l2_regularization': 6.4739785778780955e-09, 'early_stopping': 'False', 'learning_rate': 0.006133197332190358, 'iterations': 1484, 'max_depth': 28, 'max_bins': 125, 'min_samples_leaf': 3465, 'max_leaf_nodes': 28}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:12:32,575]\u001b[0m Trial 246 finished with value: 0.8664922480611315 and parameters: {'l2_regularization': 5.93741372774577e-09, 'early_stopping': 'False', 'learning_rate': 0.006693869602695694, 'iterations': 1002, 'max_depth': 30, 'max_bins': 119, 'min_samples_leaf': 1708, 'max_leaf_nodes': 29}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:13:07,389]\u001b[0m Trial 247 finished with value: 0.8658352376996244 and parameters: {'l2_regularization': 0.0068273781164795945, 'early_stopping': 'False', 'learning_rate': 0.006970845641285344, 'iterations': 1398, 'max_depth': 25, 'max_bins': 123, 'min_samples_leaf': 5116, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:13:31,669]\u001b[0m Trial 248 finished with value: 0.8673947970376042 and parameters: {'l2_regularization': 3.4433963768276026e-09, 'early_stopping': 'False', 'learning_rate': 0.007582412544365289, 'iterations': 1703, 'max_depth': 29, 'max_bins': 116, 'min_samples_leaf': 71, 'max_leaf_nodes': 27}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:14:02,634]\u001b[0m Trial 249 finished with value: 0.8653386185323025 and parameters: {'l2_regularization': 3.6745907115542102e-09, 'early_stopping': 'False', 'learning_rate': 0.006038780575803087, 'iterations': 1875, 'max_depth': 29, 'max_bins': 115, 'min_samples_leaf': 3072, 'max_leaf_nodes': 23}. Best is trial 172 with value: 0.8681191259739842.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9h 27min 31s, sys: 1min 28s, total: 9h 29min\n",
      "Wall time: 2h 24min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_hsbc_pipe1 = optuna.create_study(direction=\"maximize\")\n",
    "study_hsbc_pipe1.optimize(objective_hsbc_pipe1 ,n_trials=250)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e116b6d5",
   "metadata": {
    "papermill": {
     "duration": 0.142188,
     "end_time": "2021-12-26T00:14:03.313223",
     "exception": false,
     "start_time": "2021-12-26T00:14:03.171035",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "        Trial 19 finished with value: 0.8672914102570359 and parameters: {'l2_regularization': 0.0006534209250711092, 'early_stopping': 'False', 'learning_rate': 0.005016459124104803, 'iterations': 8503, 'max_depth': 3, 'max_bins': 127, 'min_samples_leaf': 577, 'max_leaf_nodes': 66}. Best is trial 19 with value: 0.8672914102570359.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "45c1e041",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T00:14:03.607918Z",
     "iopub.status.busy": "2021-12-26T00:14:03.607124Z",
     "iopub.status.idle": "2021-12-26T00:14:03.672456Z",
     "shell.execute_reply": "2021-12-26T00:14:03.673505Z"
    },
    "papermill": {
     "duration": 0.218766,
     "end_time": "2021-12-26T00:14:03.673703",
     "exception": false,
     "start_time": "2021-12-26T00:14:03.454937",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of finished trials :  250\n",
      "Best Trials :  {'l2_regularization': 1.939640534964219e-08, 'early_stopping': 'False', 'learning_rate': 0.006683250049408931, 'iterations': 1841, 'max_depth': 28, 'max_bins': 114, 'min_samples_leaf': 50, 'max_leaf_nodes': 28}\n",
      "Best Values :  0.8681191259739842\n"
     ]
    }
   ],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_hsbc_pipe1.trials))\n",
    "print(\"Best Trials : \", study_hsbc_pipe1.best_trial.params)\n",
    "print(\"Best Values : \" , study_hsbc_pipe1.best_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7249b10",
   "metadata": {
    "papermill": {
     "duration": 0.123343,
     "end_time": "2021-12-26T00:14:03.939709",
     "exception": false,
     "start_time": "2021-12-26T00:14:03.816366",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "    Best Trials :  {'learning_rate': 0.055307070793369925, 'iterations': 3935, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.041845472991642346, 'depth': 8, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}\n",
    "    Best Values :  0.8669049000350807"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90a397ab",
   "metadata": {
    "papermill": {
     "duration": 0.119261,
     "end_time": "2021-12-26T00:14:04.180598",
     "exception": false,
     "start_time": "2021-12-26T00:14:04.061337",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 11242.520582,
   "end_time": "2021-12-26T00:14:05.434234",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-25T21:06:42.913652",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}