credit-part2-datap-reparation-4-tuning-impute-estimator2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e35478df",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:49.018566Z",
     "iopub.status.busy": "2021-12-22T10:41:49.017006Z",
     "iopub.status.idle": "2021-12-22T10:41:52.342673Z",
     "shell.execute_reply": "2021-12-22T10:41:52.342048Z",
     "shell.execute_reply.started": "2021-12-22T10:39:37.888576Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.345912,
     "end_time": "2021-12-22T10:41:52.342833",
     "exception": false,
     "start_time": "2021-12-22T10:41:48.996921",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import mean_squared_error,mean_absolute_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans \n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "#from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb8a08c1",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.015195,
     "end_time": "2021-12-22T10:41:52.376112",
     "exception": false,
     "start_time": "2021-12-22T10:41:52.360917",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7fcfc85f",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:52.411390Z",
     "iopub.status.busy": "2021-12-22T10:41:52.410877Z",
     "iopub.status.idle": "2021-12-22T10:41:52.809959Z",
     "shell.execute_reply": "2021-12-22T10:41:52.810414Z",
     "shell.execute_reply.started": "2021-12-22T10:39:37.906621Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.419133,
     "end_time": "2021-12-22T10:41:52.810559",
     "exception": false,
     "start_time": "2021-12-22T10:41:52.391426",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 199 ms, sys: 58.5 ms, total: 258 ms\n",
      "Wall time: 381 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35fface9",
   "metadata": {
    "papermill": {
     "duration": 0.017555,
     "end_time": "2021-12-22T10:41:52.844716",
     "exception": false,
     "start_time": "2021-12-22T10:41:52.827161",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69d41f2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:52.901628Z",
     "iopub.status.busy": "2021-12-22T10:41:52.900243Z",
     "iopub.status.idle": "2021-12-22T10:41:52.953372Z",
     "shell.execute_reply": "2021-12-22T10:41:52.952439Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.156349Z"
    },
    "papermill": {
     "duration": 0.092251,
     "end_time": "2021-12-22T10:41:52.953536",
     "exception": false,
     "start_time": "2021-12-22T10:41:52.861285",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b877675",
   "metadata": {
    "papermill": {
     "duration": 0.015867,
     "end_time": "2021-12-22T10:41:52.985759",
     "exception": false,
     "start_time": "2021-12-22T10:41:52.969892",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "19bfeb13",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.034188Z",
     "iopub.status.busy": "2021-12-22T10:41:53.032858Z",
     "iopub.status.idle": "2021-12-22T10:41:53.072311Z",
     "shell.execute_reply": "2021-12-22T10:41:53.072772Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.226808Z"
    },
    "papermill": {
     "duration": 0.070835,
     "end_time": "2021-12-22T10:41:53.072917",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.002082",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "993aa287",
   "metadata": {
    "papermill": {
     "duration": 0.016318,
     "end_time": "2021-12-22T10:41:53.105708",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.089390",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# let's predict monthly income: \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "78e1bb91",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.143882Z",
     "iopub.status.busy": "2021-12-22T10:41:53.143115Z",
     "iopub.status.idle": "2021-12-22T10:41:53.152634Z",
     "shell.execute_reply": "2021-12-22T10:41:53.152008Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.290619Z"
    },
    "papermill": {
     "duration": 0.030598,
     "end_time": "2021-12-22T10:41:53.152771",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.122173",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.33 ms, sys: 1.04 ms, total: 6.37 ms\n",
      "Wall time: 6.16 ms\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8fefd6bc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.192185Z",
     "iopub.status.busy": "2021-12-22T10:41:53.191369Z",
     "iopub.status.idle": "2021-12-22T10:41:53.209396Z",
     "shell.execute_reply": "2021-12-22T10:41:53.209921Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.304596Z"
    },
    "papermill": {
     "duration": 0.039932,
     "end_time": "2021-12-22T10:41:53.210078",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.170146",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_final= pd.concat( [train, test], axis=0) \n",
    "testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "X_test=testdf_income.drop([\"MonthlyIncome\"],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f6204834",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.250553Z",
     "iopub.status.busy": "2021-12-22T10:41:53.248033Z",
     "iopub.status.idle": "2021-12-22T10:41:53.253065Z",
     "shell.execute_reply": "2021-12-22T10:41:53.253451Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.332540Z"
    },
    "papermill": {
     "duration": 0.026565,
     "end_time": "2021-12-22T10:41:53.253600",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.227035",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([], dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8837bba0",
   "metadata": {
    "papermill": {
     "duration": 0.01678,
     "end_time": "2021-12-22T10:41:53.287146",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.270366",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fa87f436",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.328224Z",
     "iopub.status.busy": "2021-12-22T10:41:53.327364Z",
     "iopub.status.idle": "2021-12-22T10:41:53.332680Z",
     "shell.execute_reply": "2021-12-22T10:41:53.332245Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.343003Z"
    },
    "papermill": {
     "duration": 0.02835,
     "end_time": "2021-12-22T10:41:53.332789",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.304439",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'age',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'DebtRatio',\n",
       "       'NumberOfOpenCreditLinesAndLoans', 'NumberOfTimes90DaysLate',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberOfDependents'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "155584fc",
   "metadata": {
    "papermill": {
     "duration": 0.016908,
     "end_time": "2021-12-22T10:41:53.366866",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.349958",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8806e004",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.407141Z",
     "iopub.status.busy": "2021-12-22T10:41:53.406379Z",
     "iopub.status.idle": "2021-12-22T10:41:53.408308Z",
     "shell.execute_reply": "2021-12-22T10:41:53.408703Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.359030Z"
    },
    "papermill": {
     "duration": 0.024479,
     "end_time": "2021-12-22T10:41:53.408823",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.384344",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Define vnum pipeline\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0731fafa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:53.447778Z",
     "iopub.status.busy": "2021-12-22T10:41:53.447046Z",
     "iopub.status.idle": "2021-12-22T10:41:55.934381Z",
     "shell.execute_reply": "2021-12-22T10:41:55.933480Z",
     "shell.execute_reply.started": "2021-12-22T10:39:38.369348Z"
    },
    "papermill": {
     "duration": 2.508678,
     "end_time": "2021-12-22T10:41:55.934532",
     "exception": false,
     "start_time": "2021-12-22T10:41:53.425854",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "numeric_transformer1.fit(X)\n",
    "X_pre = numeric_transformer1.transform(X)\n",
    "#X_test_pre = numeric_transformer1.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ab7b163",
   "metadata": {
    "papermill": {
     "duration": 0.017026,
     "end_time": "2021-12-22T10:41:55.969162",
     "exception": false,
     "start_time": "2021-12-22T10:41:55.952136",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "78ea36f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:56.009788Z",
     "iopub.status.busy": "2021-12-22T10:41:56.009061Z",
     "iopub.status.idle": "2021-12-22T10:41:56.011443Z",
     "shell.execute_reply": "2021-12-22T10:41:56.011050Z",
     "shell.execute_reply.started": "2021-12-22T10:39:40.357305Z"
    },
    "papermill": {
     "duration": 0.025231,
     "end_time": "2021-12-22T10:41:56.011548",
     "exception": false,
     "start_time": "2021-12-22T10:41:55.986317",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cross_validation_design = StratifiedKFold( n_splits=5,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design_regression=KFold( n_splits=7,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40104a28",
   "metadata": {
    "papermill": {
     "duration": 0.016805,
     "end_time": "2021-12-22T10:41:56.045383",
     "exception": false,
     "start_time": "2021-12-22T10:41:56.028578",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "87e86cbe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:56.094058Z",
     "iopub.status.busy": "2021-12-22T10:41:56.093178Z",
     "iopub.status.idle": "2021-12-22T10:41:56.095190Z",
     "shell.execute_reply": "2021-12-22T10:41:56.095558Z",
     "shell.execute_reply.started": "2021-12-22T10:40:21.490663Z"
    },
    "papermill": {
     "duration": 0.033341,
     "end_time": "2021-12-22T10:41:56.095692",
     "exception": false,
     "start_time": "2021-12-22T10:41:56.062351",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    " def objective_xgbr_income(trial,data=X_pre,target=y_income):\n",
    "    # Optuna+cv :\n",
    "    #https://www.kaggle.com/hamidrezabakhtaki/xgboost-catboost-lighgbm-optuna-final-submission\n",
    "    # https://www.kaggle.com/prashant111/lightgbm-classifier-in-python\n",
    "    # https://www.kaggle.com/tunguz/tps-09-21-histgradientboosting-with-optuna\n",
    "    X_train,X_test, y_train,  y_test = train_test_split(data, target, \n",
    "                                                        test_size=0.25,\n",
    "                                                       random_state=42,shuffle=False)\n",
    "    param = {   \"verbosity\": 0,\n",
    "                #\"objective\": \"binary:logistic\",\n",
    "                #\"eval_metric\": \"auc\",\n",
    "                'random_state': 48,\n",
    "                # regression\n",
    "                'objective':'reg:squarederror', \n",
    "                 'eval_metric': 'mae',\n",
    "                #early_stopping_rounds=100 ,\n",
    "                'gpu_id':0, \n",
    "                'predictor':\"gpu_predictor\",\n",
    "                # use exact for small dataset.\n",
    "                \"tree_method\": \"exact\",\n",
    "                # big data :\n",
    "                # 'tree_method': 'gpu_hist',\n",
    "                # defines booster, gblinear for linear functions.\n",
    "                \"booster\": trial.suggest_categorical(\"booster\", [\"gbtree\", \"gblinear\", \"dart\"]),\n",
    "                # L2 regularization weight.\n",
    "                \"lambda\": trial.suggest_float(\"lambda\", 1e-8, 1.0, log=True),\n",
    "                # L1 regularization weight.\n",
    "                \"alpha\": trial.suggest_float(\"alpha\", 1e-8, 1.0, log=True),\n",
    "                # sampling ratio for training data.\n",
    "                \"subsample\": trial.suggest_float(\"subsample\", 0.2, 1.0),\n",
    "                # sampling according to each tree.\n",
    "                \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n",
    "                \"learning_rate\" : trial.suggest_float(\"learning_rate\" , 0.005 , 0.2),\n",
    "                \"n_estimators\" : trial.suggest_int(\"n_estimators\" , 1000 ,10000),\n",
    "             }\n",
    "    if param[\"booster\"] in [\"gbtree\", \"dart\"]:\n",
    "        # maximum depth of the tree, signifies complexity of the tree.\n",
    "        param[\"max_depth\"] = trial.suggest_int(\"max_depth\", 3, 9, step=2)\n",
    "        # minimum child weight, larger the term more conservative the tree.\n",
    "        param[\"min_child_weight\"] = trial.suggest_int(\"min_child_weight\", 2, 10)\n",
    "        param[\"eta\"] = trial.suggest_float(\"eta\", 1e-8, 1.0, log=True)\n",
    "        # defines how selective algorithm is.\n",
    "        param[\"gamma\"] = trial.suggest_float(\"gamma\", 1e-8, 1.0, log=True)\n",
    "        param[\"grow_policy\"] = trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"])\n",
    "\n",
    "    if param[\"booster\"] == \"dart\":\n",
    "        param[\"sample_type\"] = trial.suggest_categorical(\"sample_type\", [\"uniform\", \"weighted\"])\n",
    "        param[\"normalize_type\"] = trial.suggest_categorical(\"normalize_type\", [\"tree\", \"forest\"])\n",
    "        param[\"rate_drop\"] = trial.suggest_float(\"rate_drop\", 1e-8, 1.0, log=True)\n",
    "        param[\"skip_drop\"] = trial.suggest_float(\"skip_drop\", 1e-8, 1.0, log=True)\n",
    "             \n",
    "    pipe_xgbr = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "    \n",
    "    #mae =cross_val_score(pipe_xgbr, data, target,cv=cross_validation_design_regression,scoring='neg_mean_absolute_error').mean()\n",
    "    model  = XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param)\n",
    "    \n",
    "    model.fit( X_train, y_train,\n",
    "              eval_set=[(X_test, y_test)],\n",
    "              early_stopping_rounds=50,\n",
    "              verbose=False)\n",
    "             \n",
    "    #pipeline_model_lgbm.fit(X_train,y_train)\n",
    "    #preds = model.predict_proba(X_test)[:,1]\n",
    "    y_pred= model.predict(X_test)\n",
    "    mae = mean_absolute_error(y_test, y_pred)\n",
    "    #oof[test_idx] = preds\n",
    "    #average_precesion = average_precision_score(y_true= y_test, y_score= preds)\n",
    "    return mae                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "37a72630",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T10:41:56.134550Z",
     "iopub.status.busy": "2021-12-22T10:41:56.133761Z",
     "iopub.status.idle": "2021-12-22T13:53:41.922038Z",
     "shell.execute_reply": "2021-12-22T13:53:41.923069Z",
     "shell.execute_reply.started": "2021-12-22T10:40:21.508729Z"
    },
    "papermill": {
     "duration": 11505.81059,
     "end_time": "2021-12-22T13:53:41.923296",
     "exception": false,
     "start_time": "2021-12-22T10:41:56.112706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-22 10:41:56,132]\u001b[0m A new study created in memory with name: no-name-1d22cd0f-e189-483e-bbe6-0e42f95e2c21\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 10:42:15,309]\u001b[0m Trial 0 finished with value: 2754.7587890625 and parameters: {'booster': 'dart', 'lambda': 0.0007527088670920512, 'alpha': 0.1834056576067222, 'subsample': 0.8311062875898771, 'colsample_bytree': 0.5150049018484844, 'learning_rate': 0.027533870285480763, 'n_estimators': 3693, 'max_depth': 7, 'min_child_weight': 4, 'eta': 3.1873408342432446e-06, 'gamma': 0.025461567916966247, 'grow_policy': 'depthwise', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 1.6889342547319356e-07, 'skip_drop': 2.0339021470056713e-05}. Best is trial 0 with value: 2754.7587890625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 10:42:30,789]\u001b[0m Trial 1 finished with value: 2758.1337890625 and parameters: {'booster': 'gbtree', 'lambda': 6.513877718871243e-07, 'alpha': 0.08814980745407636, 'subsample': 0.37792993415288473, 'colsample_bytree': 0.5147710138281751, 'learning_rate': 0.012997054875334584, 'n_estimators': 9118, 'max_depth': 7, 'min_child_weight': 5, 'eta': 5.187296070891566e-07, 'gamma': 0.000456025641050529, 'grow_policy': 'lossguide'}. Best is trial 0 with value: 2754.7587890625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 10:42:35,622]\u001b[0m Trial 2 finished with value: 2728.722900390625 and parameters: {'booster': 'gbtree', 'lambda': 0.0002947202711012878, 'alpha': 1.3443979024375865e-06, 'subsample': 0.24862156066638957, 'colsample_bytree': 0.7346410171229443, 'learning_rate': 0.10044895978452256, 'n_estimators': 5465, 'max_depth': 7, 'min_child_weight': 8, 'eta': 8.250302957266397e-05, 'gamma': 1.1028761487545636e-07, 'grow_policy': 'lossguide'}. Best is trial 2 with value: 2728.722900390625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:21:36,431]\u001b[0m Trial 3 finished with value: 5507.21728515625 and parameters: {'booster': 'dart', 'lambda': 1.6791137913255436e-07, 'alpha': 1.599469544434474e-06, 'subsample': 0.5542667575670102, 'colsample_bytree': 0.6526411864043433, 'learning_rate': 0.008598941608217756, 'n_estimators': 5678, 'max_depth': 5, 'min_child_weight': 10, 'eta': 0.012779675039779058, 'gamma': 0.025826420307039615, 'grow_policy': 'depthwise', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 0.406246021081594, 'skip_drop': 1.4692334474791904e-05}. Best is trial 2 with value: 2728.722900390625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:21:44,887]\u001b[0m Trial 4 finished with value: 2701.507568359375 and parameters: {'booster': 'gbtree', 'lambda': 0.00019168431526235056, 'alpha': 1.195611593943182e-05, 'subsample': 0.6248317523490843, 'colsample_bytree': 0.7746604104445283, 'learning_rate': 0.030615339551486563, 'n_estimators': 1231, 'max_depth': 5, 'min_child_weight': 4, 'eta': 0.00036783489840279485, 'gamma': 0.8362473956888942, 'grow_policy': 'depthwise'}. Best is trial 4 with value: 2701.507568359375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:21:46,711]\u001b[0m Trial 5 finished with value: 2854.872802734375 and parameters: {'booster': 'gblinear', 'lambda': 9.689025706170533e-06, 'alpha': 0.0006212178000025195, 'subsample': 0.6416464000246493, 'colsample_bytree': 0.6037394801928746, 'learning_rate': 0.036952249805787436, 'n_estimators': 2891}. Best is trial 4 with value: 2701.507568359375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:21:59,322]\u001b[0m Trial 6 finished with value: 2617.435791015625 and parameters: {'booster': 'gbtree', 'lambda': 4.331351547746808e-06, 'alpha': 0.00047123513884389105, 'subsample': 0.5401486506024009, 'colsample_bytree': 0.8100314233910042, 'learning_rate': 0.0458978070126128, 'n_estimators': 4985, 'max_depth': 9, 'min_child_weight': 9, 'eta': 7.128572852481928e-05, 'gamma': 0.01152104098626648, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:01,063]\u001b[0m Trial 7 finished with value: 2774.964111328125 and parameters: {'booster': 'gblinear', 'lambda': 8.982098652054373e-07, 'alpha': 7.232186692485383e-08, 'subsample': 0.2674076178407903, 'colsample_bytree': 0.8035083365068136, 'learning_rate': 0.01701156010204654, 'n_estimators': 1728}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:12,994]\u001b[0m Trial 8 finished with value: 2769.96240234375 and parameters: {'booster': 'dart', 'lambda': 0.008990025309589299, 'alpha': 3.509145315337033e-08, 'subsample': 0.3226835694281935, 'colsample_bytree': 0.5734644988281966, 'learning_rate': 0.07723701389220045, 'n_estimators': 5514, 'max_depth': 7, 'min_child_weight': 8, 'eta': 0.41486360612781503, 'gamma': 0.0073573598438310605, 'grow_policy': 'lossguide', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 0.10038535257298484, 'skip_drop': 1.5945831011872124e-06}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:20,104]\u001b[0m Trial 9 finished with value: 2652.81787109375 and parameters: {'booster': 'gbtree', 'lambda': 0.001347892131890677, 'alpha': 1.4531821709826876e-07, 'subsample': 0.6750787461214423, 'colsample_bytree': 0.7133096934885625, 'learning_rate': 0.10747593375590597, 'n_estimators': 8022, 'max_depth': 7, 'min_child_weight': 5, 'eta': 0.006458381033488703, 'gamma': 2.3810675772299661e-07, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:27,991]\u001b[0m Trial 10 finished with value: 2674.2587890625 and parameters: {'booster': 'gbtree', 'lambda': 0.7051056331739114, 'alpha': 0.0014935809607318853, 'subsample': 0.4641368956547503, 'colsample_bytree': 0.9654198999337931, 'learning_rate': 0.18954771087460964, 'n_estimators': 7138, 'max_depth': 9, 'min_child_weight': 10, 'eta': 3.394016440615113e-08, 'gamma': 1.443388689651677e-05, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:32,122]\u001b[0m Trial 11 finished with value: 2960.009521484375 and parameters: {'booster': 'gbtree', 'lambda': 0.03637737845462336, 'alpha': 0.00015386611741745603, 'subsample': 0.7927014515851145, 'colsample_bytree': 0.2481784345923621, 'learning_rate': 0.1514558804676303, 'n_estimators': 9920, 'max_depth': 9, 'min_child_weight': 7, 'eta': 0.0011523560036483312, 'gamma': 2.2289744874753253e-08, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:43,951]\u001b[0m Trial 12 finished with value: 2687.826171875 and parameters: {'booster': 'gbtree', 'lambda': 1.0196952170951073e-08, 'alpha': 0.015005313293226458, 'subsample': 0.9644176058005225, 'colsample_bytree': 0.9446076596767151, 'learning_rate': 0.07201354452039688, 'n_estimators': 7805, 'max_depth': 3, 'min_child_weight': 2, 'eta': 0.03220511410625349, 'gamma': 1.9001626239420638e-05, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:22:53,424]\u001b[0m Trial 13 finished with value: 2625.386474609375 and parameters: {'booster': 'gbtree', 'lambda': 1.71196400004189e-05, 'alpha': 1.1010860992031536e-05, 'subsample': 0.7394605423564059, 'colsample_bytree': 0.8606655772896421, 'learning_rate': 0.13864526151556267, 'n_estimators': 7430, 'max_depth': 9, 'min_child_weight': 6, 'eta': 2.573539170004751e-05, 'gamma': 9.740508655115514e-07, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:02,058]\u001b[0m Trial 14 finished with value: 2646.283203125 and parameters: {'booster': 'gbtree', 'lambda': 9.214640953278205e-06, 'alpha': 3.184856289665557e-05, 'subsample': 0.48749745258017063, 'colsample_bytree': 0.8674247977589798, 'learning_rate': 0.14068786631415117, 'n_estimators': 4230, 'max_depth': 9, 'min_child_weight': 7, 'eta': 2.557567787805827e-05, 'gamma': 2.1555472766789316e-06, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:02,812]\u001b[0m Trial 15 finished with value: 2977.798828125 and parameters: {'booster': 'gblinear', 'lambda': 1.007878215150939e-05, 'alpha': 0.005284438994371007, 'subsample': 0.7528889574979037, 'colsample_bytree': 0.8885347849534329, 'learning_rate': 0.14406895142978124, 'n_estimators': 6576}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:05,665]\u001b[0m Trial 16 finished with value: 3151.086669921875 and parameters: {'booster': 'gbtree', 'lambda': 3.908790760182174e-05, 'alpha': 4.116233869776541e-06, 'subsample': 0.9456665357305556, 'colsample_bytree': 0.21923011908140133, 'learning_rate': 0.05747918686944713, 'n_estimators': 4388, 'max_depth': 9, 'min_child_weight': 9, 'eta': 4.50228640025518e-06, 'gamma': 0.0006973895741376607, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:10,631]\u001b[0m Trial 17 finished with value: 2934.1904296875 and parameters: {'booster': 'gbtree', 'lambda': 3.624249679799563e-08, 'alpha': 0.00016132183020611698, 'subsample': 0.5157926045995261, 'colsample_bytree': 0.3968247873091528, 'learning_rate': 0.18266117915989702, 'n_estimators': 8444, 'max_depth': 9, 'min_child_weight': 6, 'eta': 1.5581159219436764e-07, 'gamma': 0.2459187854397049, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:16,282]\u001b[0m Trial 18 finished with value: 2768.106201171875 and parameters: {'booster': 'dart', 'lambda': 8.089984405325782e-07, 'alpha': 3.6678266437730885e-07, 'subsample': 0.7224927577389683, 'colsample_bytree': 0.8557563483038179, 'learning_rate': 0.10893751446875508, 'n_estimators': 6440, 'max_depth': 3, 'min_child_weight': 7, 'eta': 1.6930313417901104e-05, 'gamma': 1.4344793409181143e-06, 'grow_policy': 'depthwise', 'sample_type': 'weighted', 'normalize_type': 'forest', 'rate_drop': 1.0788428495232139e-08, 'skip_drop': 0.3563334000376484}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:17,015]\u001b[0m Trial 19 finished with value: 2969.8701171875 and parameters: {'booster': 'gblinear', 'lambda': 3.844054588684893e-05, 'alpha': 4.245881714813161e-05, 'subsample': 0.8648452014111293, 'colsample_bytree': 0.9705388442816669, 'learning_rate': 0.12137418347024397, 'n_estimators': 2869}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:25,629]\u001b[0m Trial 20 finished with value: 2661.42333984375 and parameters: {'booster': 'gbtree', 'lambda': 3.3148486987876346e-06, 'alpha': 0.0010708080717470076, 'subsample': 0.41459981713916255, 'colsample_bytree': 0.8281401731309858, 'learning_rate': 0.1661703524805374, 'n_estimators': 4878, 'max_depth': 9, 'min_child_weight': 2, 'eta': 0.0008597790885453458, 'gamma': 0.00014352397173318621, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:34,610]\u001b[0m Trial 21 finished with value: 2651.221435546875 and parameters: {'booster': 'gbtree', 'lambda': 5.622538657696121e-06, 'alpha': 1.3998770197192611e-05, 'subsample': 0.5423569773691698, 'colsample_bytree': 0.8998571981835853, 'learning_rate': 0.13467656763301325, 'n_estimators': 3916, 'max_depth': 9, 'min_child_weight': 7, 'eta': 2.842185070163463e-05, 'gamma': 2.6374739049135233e-06, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:42,745]\u001b[0m Trial 22 finished with value: 2666.19482421875 and parameters: {'booster': 'gbtree', 'lambda': 4.667789542677563e-05, 'alpha': 5.680155671090299e-05, 'subsample': 0.460449280005586, 'colsample_bytree': 0.7022475450211408, 'learning_rate': 0.08831493421232633, 'n_estimators': 2866, 'max_depth': 9, 'min_child_weight': 8, 'eta': 2.676081951714089e-06, 'gamma': 2.122885723416083e-06, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:51,222]\u001b[0m Trial 23 finished with value: 2693.31298828125 and parameters: {'booster': 'gbtree', 'lambda': 1.0490800297262678e-07, 'alpha': 1.1090358572950295e-05, 'subsample': 0.5752703381713171, 'colsample_bytree': 0.9929382159657687, 'learning_rate': 0.16377935414259862, 'n_estimators': 6244, 'max_depth': 9, 'min_child_weight': 6, 'eta': 0.00013001223714926983, 'gamma': 1.924042616980404e-05, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:23:58,145]\u001b[0m Trial 24 finished with value: 2652.200927734375 and parameters: {'booster': 'gbtree', 'lambda': 2.3463837496377768e-06, 'alpha': 0.0003148339721005714, 'subsample': 0.6758397322425449, 'colsample_bytree': 0.774457106724644, 'learning_rate': 0.12949047182303514, 'n_estimators': 4807, 'max_depth': 7, 'min_child_weight': 9, 'eta': 2.169202715924956e-05, 'gamma': 9.165743899644621e-08, 'grow_policy': 'lossguide'}. Best is trial 6 with value: 2617.435791015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:24:10,201]\u001b[0m Trial 25 finished with value: 2593.284423828125 and parameters: {'booster': 'gbtree', 'lambda': 1.7306915171903822e-05, 'alpha': 0.006381712966685966, 'subsample': 0.481056551946391, 'colsample_bytree': 0.8895039406383476, 'learning_rate': 0.05875116320592379, 'n_estimators': 7081, 'max_depth': 9, 'min_child_weight': 9, 'eta': 8.421534090566318e-07, 'gamma': 5.188510607419209e-07, 'grow_policy': 'lossguide'}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:24:15,847]\u001b[0m Trial 26 finished with value: 2695.39599609375 and parameters: {'booster': 'gbtree', 'lambda': 0.001838201274234097, 'alpha': 0.00995731201350005, 'subsample': 0.3438013509237383, 'colsample_bytree': 0.9118723195090962, 'learning_rate': 0.056429222778533246, 'n_estimators': 7111, 'max_depth': 5, 'min_child_weight': 9, 'eta': 4.266374155590719e-07, 'gamma': 0.0024006537948425485, 'grow_policy': 'lossguide'}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:24:27,549]\u001b[0m Trial 27 finished with value: 2610.424560546875 and parameters: {'booster': 'gbtree', 'lambda': 8.900584826552674e-05, 'alpha': 0.9214288240345685, 'subsample': 0.5918190013126592, 'colsample_bytree': 0.8228949578879398, 'learning_rate': 0.04950926439632478, 'n_estimators': 7169, 'max_depth': 9, 'min_child_weight': 10, 'eta': 6.994153134927983e-08, 'gamma': 1.0778136848523237e-08, 'grow_policy': 'depthwise'}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:24:41,820]\u001b[0m Trial 28 finished with value: 2777.364501953125 and parameters: {'booster': 'dart', 'lambda': 0.00013435919007489025, 'alpha': 0.3785321315005459, 'subsample': 0.43231050788537795, 'colsample_bytree': 0.6572427518888158, 'learning_rate': 0.047291370746872445, 'n_estimators': 9131, 'max_depth': 7, 'min_child_weight': 10, 'eta': 1.7884244453490844e-08, 'gamma': 2.4259476080371752e-08, 'grow_policy': 'depthwise', 'sample_type': 'weighted', 'normalize_type': 'forest', 'rate_drop': 7.892057986469642e-05, 'skip_drop': 0.3138166587302818}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:24:42,659]\u001b[0m Trial 29 finished with value: 2924.44677734375 and parameters: {'booster': 'gblinear', 'lambda': 0.0004324818552208607, 'alpha': 0.8836274079159815, 'subsample': 0.5981231482026536, 'colsample_bytree': 0.7691222356761215, 'learning_rate': 0.06900237961637577, 'n_estimators': 6309}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:24:55,543]\u001b[0m Trial 30 finished with value: 2821.937255859375 and parameters: {'booster': 'dart', 'lambda': 0.005151283528283732, 'alpha': 0.06796447792948, 'subsample': 0.5068577865479295, 'colsample_bytree': 0.4251665385802703, 'learning_rate': 0.0386603143438558, 'n_estimators': 8400, 'max_depth': 9, 'min_child_weight': 9, 'eta': 5.848114371981687e-08, 'gamma': 1.2821373576231631e-08, 'grow_policy': 'depthwise', 'sample_type': 'weighted', 'normalize_type': 'forest', 'rate_drop': 0.0001329384281752889, 'skip_drop': 2.8059025592513028e-08}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:25:05,971]\u001b[0m Trial 31 finished with value: 2631.354248046875 and parameters: {'booster': 'gbtree', 'lambda': 2.7301839042671097e-05, 'alpha': 1.0598549635217272e-08, 'subsample': 0.8599762452794747, 'colsample_bytree': 0.8230771419836207, 'learning_rate': 0.0891219816171366, 'n_estimators': 6937, 'max_depth': 9, 'min_child_weight': 10, 'eta': 4.736580545767343e-07, 'gamma': 4.6696377252182875e-07, 'grow_policy': 'depthwise'}. Best is trial 25 with value: 2593.284423828125.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:25:18,298]\u001b[0m Trial 32 finished with value: 2592.543701171875 and parameters: {'booster': 'gbtree', 'lambda': 1.8507348611258262e-06, 'alpha': 0.04238618209025097, 'subsample': 0.718247777487934, 'colsample_bytree': 0.9205402343619136, 'learning_rate': 0.05552738442191385, 'n_estimators': 7849, 'max_depth': 9, 'min_child_weight': 8, 'eta': 2.2849638679235976e-06, 'gamma': 5.350781368747221e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:25:31,094]\u001b[0m Trial 33 finished with value: 2598.656494140625 and parameters: {'booster': 'gbtree', 'lambda': 2.1483625255908685e-06, 'alpha': 0.05566566177111777, 'subsample': 0.6725485254544776, 'colsample_bytree': 0.9197516451088154, 'learning_rate': 0.053423798092285514, 'n_estimators': 9230, 'max_depth': 9, 'min_child_weight': 9, 'eta': 1.1972084670621278e-07, 'gamma': 5.908154241099185e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:25:52,706]\u001b[0m Trial 34 finished with value: 2601.907470703125 and parameters: {'booster': 'gbtree', 'lambda': 2.2880069041641008e-07, 'alpha': 0.06833411111373003, 'subsample': 0.6402877968421363, 'colsample_bytree': 0.9205217023046074, 'learning_rate': 0.02416393872623522, 'n_estimators': 9189, 'max_depth': 9, 'min_child_weight': 8, 'eta': 1.363049748005484e-07, 'gamma': 4.94985654621319e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:26:08,361]\u001b[0m Trial 35 finished with value: 2619.484130859375 and parameters: {'booster': 'gbtree', 'lambda': 2.540758954644427e-07, 'alpha': 0.07081818982914605, 'subsample': 0.6943693005799441, 'colsample_bytree': 0.9989550088307972, 'learning_rate': 0.02267960695924995, 'n_estimators': 9975, 'max_depth': 7, 'min_child_weight': 8, 'eta': 1.4295244028131366e-06, 'gamma': 7.141708326078164e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:26:57,240]\u001b[0m Trial 36 finished with value: 2619.431884765625 and parameters: {'booster': 'gbtree', 'lambda': 4.018904393731594e-07, 'alpha': 0.04614722892201621, 'subsample': 0.798133796534918, 'colsample_bytree': 0.9304415363057194, 'learning_rate': 0.005672517491658431, 'n_estimators': 9055, 'max_depth': 7, 'min_child_weight': 8, 'eta': 1.2779347257157972e-07, 'gamma': 3.304960359282773e-07, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:27:18,219]\u001b[0m Trial 37 finished with value: 2596.175537109375 and parameters: {'booster': 'gbtree', 'lambda': 7.854379662586266e-08, 'alpha': 0.2412650099132782, 'subsample': 0.784002694706466, 'colsample_bytree': 0.9156414415752494, 'learning_rate': 0.02615005357822016, 'n_estimators': 9137, 'max_depth': 9, 'min_child_weight': 9, 'eta': 1.2449351066882756e-08, 'gamma': 5.0377766488186434e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:27:27,089]\u001b[0m Trial 38 finished with value: 2624.527587890625 and parameters: {'booster': 'gbtree', 'lambda': 1.5674736668474393e-06, 'alpha': 0.18029468469819418, 'subsample': 0.8843150950292147, 'colsample_bytree': 0.9450486635266455, 'learning_rate': 0.06179699701175943, 'n_estimators': 8509, 'max_depth': 7, 'min_child_weight': 9, 'eta': 1.2332191947017504e-08, 'gamma': 2.5659615954423455e-07, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:27:44,965]\u001b[0m Trial 39 finished with value: 2656.368896484375 and parameters: {'booster': 'dart', 'lambda': 2.0471157229824383e-08, 'alpha': 0.0027071734147792316, 'subsample': 0.8014218762243502, 'colsample_bytree': 0.6790362553894231, 'learning_rate': 0.03560075620224723, 'n_estimators': 9403, 'max_depth': 9, 'min_child_weight': 9, 'eta': 1.3872144648209066e-06, 'gamma': 5.185762899567511e-06, 'grow_policy': 'depthwise', 'sample_type': 'weighted', 'normalize_type': 'tree', 'rate_drop': 0.00047801597359203315, 'skip_drop': 0.0024674686457228294}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:27:45,763]\u001b[0m Trial 40 finished with value: 2945.353759765625 and parameters: {'booster': 'gblinear', 'lambda': 5.6928243467917934e-08, 'alpha': 0.016736191773043915, 'subsample': 0.9225532986249805, 'colsample_bytree': 0.7349319503207129, 'learning_rate': 0.08640314282227735, 'n_estimators': 7791}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:28:05,374]\u001b[0m Trial 41 finished with value: 2602.9345703125 and parameters: {'booster': 'gbtree', 'lambda': 1.3905167529625825e-07, 'alpha': 0.22403394088264558, 'subsample': 0.6162147139618632, 'colsample_bytree': 0.9134570838099708, 'learning_rate': 0.026123134297813363, 'n_estimators': 9534, 'max_depth': 9, 'min_child_weight': 8, 'eta': 2.8415197268936085e-07, 'gamma': 3.9238529675227385e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:28:28,714]\u001b[0m Trial 42 finished with value: 2621.546142578125 and parameters: {'booster': 'gbtree', 'lambda': 4.952860359656553e-07, 'alpha': 0.041501684885267315, 'subsample': 0.6375994928651542, 'colsample_bytree': 0.8770391814945492, 'learning_rate': 0.018297953971269647, 'n_estimators': 8831, 'max_depth': 9, 'min_child_weight': 8, 'eta': 3.357802014965894e-08, 'gamma': 7.081126109330346e-08, 'grow_policy': 'depthwise'}. Best is trial 32 with value: 2592.543701171875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:28:46,228]\u001b[0m Trial 43 finished with value: 2588.592041015625 and parameters: {'booster': 'gbtree', 'lambda': 7.982717348686855e-08, 'alpha': 0.027388551770942213, 'subsample': 0.7686980188795852, 'colsample_bytree': 0.9589579211866373, 'learning_rate': 0.03032354529501959, 'n_estimators': 8186, 'max_depth': 9, 'min_child_weight': 9, 'eta': 5.354511775953184e-06, 'gamma': 1.4642007269970655e-07, 'grow_policy': 'depthwise'}. Best is trial 43 with value: 2588.592041015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:29:00,634]\u001b[0m Trial 44 finished with value: 2591.789794921875 and parameters: {'booster': 'gbtree', 'lambda': 1.4161330807438987e-06, 'alpha': 0.019801074148557342, 'subsample': 0.7726780696226244, 'colsample_bytree': 0.9738626330549432, 'learning_rate': 0.04220088990183707, 'n_estimators': 8156, 'max_depth': 9, 'min_child_weight': 10, 'eta': 6.860286374025488e-06, 'gamma': 1.5685097075414179e-07, 'grow_policy': 'depthwise'}. Best is trial 43 with value: 2588.592041015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:29:11,506]\u001b[0m Trial 45 finished with value: 2619.169677734375 and parameters: {'booster': 'gbtree', 'lambda': 7.01062653124751e-08, 'alpha': 0.02267616747830141, 'subsample': 0.7760352461481002, 'colsample_bytree': 0.9629371721651229, 'learning_rate': 0.041787657681817324, 'n_estimators': 8159, 'max_depth': 7, 'min_child_weight': 10, 'eta': 6.107685649503617e-06, 'gamma': 1.6081782010740202e-07, 'grow_policy': 'depthwise'}. Best is trial 43 with value: 2588.592041015625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:29:28,102]\u001b[0m Trial 46 finished with value: 2580.17333984375 and parameters: {'booster': 'gbtree', 'lambda': 1.9691442008429506e-08, 'alpha': 0.006114234781784277, 'subsample': 0.8355560837217457, 'colsample_bytree': 0.9969448217796413, 'learning_rate': 0.032762637801037095, 'n_estimators': 7676, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.2153504740994073e-06, 'gamma': 5.466161876945856e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:30:01,084]\u001b[0m Trial 47 finished with value: 2589.302734375 and parameters: {'booster': 'gbtree', 'lambda': 1.164864663353675e-08, 'alpha': 0.0048783701387345784, 'subsample': 0.83276643337533, 'colsample_bytree': 0.9781171494545127, 'learning_rate': 0.012394415007647843, 'n_estimators': 7684, 'max_depth': 9, 'min_child_weight': 10, 'eta': 9.587015427447028e-07, 'gamma': 9.027037129716188e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:30:17,907]\u001b[0m Trial 48 finished with value: 2669.496337890625 and parameters: {'booster': 'gbtree', 'lambda': 1.518230448418954e-08, 'alpha': 0.0027240548918400355, 'subsample': 0.8263498208718114, 'colsample_bytree': 0.970548037612955, 'learning_rate': 0.01243456450762367, 'n_estimators': 7649, 'max_depth': 5, 'min_child_weight': 10, 'eta': 8.94804124946633e-06, 'gamma': 5.856888624191721e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:30:28,074]\u001b[0m Trial 49 finished with value: 2725.785888671875 and parameters: {'booster': 'gbtree', 'lambda': 2.983281664192745e-08, 'alpha': 0.004677267607021558, 'subsample': 0.8969286230885507, 'colsample_bytree': 0.5778740107577953, 'learning_rate': 0.033916384929234024, 'n_estimators': 5753, 'max_depth': 7, 'min_child_weight': 10, 'eta': 2.4520159277637055e-06, 'gamma': 7.696327387246634e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:30:33,328]\u001b[0m Trial 50 finished with value: 2726.77978515625 and parameters: {'booster': 'gblinear', 'lambda': 3.6275925672937825e-08, 'alpha': 0.0012500068345616285, 'subsample': 0.8347113367692838, 'colsample_bytree': 0.9927229620140663, 'learning_rate': 0.005292578065360587, 'n_estimators': 8150}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:31:01,609]\u001b[0m Trial 51 finished with value: 2584.7763671875 and parameters: {'booster': 'gbtree', 'lambda': 1.2955039692547894e-06, 'alpha': 0.007265353704804746, 'subsample': 0.990820476638865, 'colsample_bytree': 0.9491164407031709, 'learning_rate': 0.014424567305447347, 'n_estimators': 7483, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.0487738064452982e-06, 'gamma': 4.90468948273967e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:31:28,492]\u001b[0m Trial 52 finished with value: 2586.602783203125 and parameters: {'booster': 'gbtree', 'lambda': 1.2288413571779966e-08, 'alpha': 0.012487240887403965, 'subsample': 0.9829169154761588, 'colsample_bytree': 0.9460944201296111, 'learning_rate': 0.016017199545970064, 'n_estimators': 8722, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.2128625153848358e-06, 'gamma': 1.3616185428540007e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:31:55,993]\u001b[0m Trial 53 finished with value: 2587.74658203125 and parameters: {'booster': 'gbtree', 'lambda': 1.0687906734403116e-08, 'alpha': 0.02000656235418223, 'subsample': 0.9806224063120477, 'colsample_bytree': 0.9560570551187876, 'learning_rate': 0.015103139324704211, 'n_estimators': 8707, 'max_depth': 9, 'min_child_weight': 10, 'eta': 8.251356866447539e-06, 'gamma': 1.6856522178800037e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:32:25,288]\u001b[0m Trial 54 finished with value: 2587.178466796875 and parameters: {'booster': 'gbtree', 'lambda': 1.1618071341632781e-08, 'alpha': 0.003147158184366636, 'subsample': 0.9826307994894954, 'colsample_bytree': 0.9498042174262098, 'learning_rate': 0.014228677439753807, 'n_estimators': 8761, 'max_depth': 9, 'min_child_weight': 10, 'eta': 7.465241492974672e-05, 'gamma': 8.00936567342069e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:32:52,481]\u001b[0m Trial 55 finished with value: 2617.58935546875 and parameters: {'booster': 'gbtree', 'lambda': 1.0871058377096974e-08, 'alpha': 0.0007453888036491846, 'subsample': 0.9853885492209717, 'colsample_bytree': 0.8501394446553637, 'learning_rate': 0.015166251156182536, 'n_estimators': 8615, 'max_depth': 9, 'min_child_weight': 10, 'eta': 6.256085988834844e-05, 'gamma': 1.1319705515401825e-05, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:33:16,728]\u001b[0m Trial 56 finished with value: 2601.652099609375 and parameters: {'booster': 'dart', 'lambda': 2.7394102930178317e-08, 'alpha': 0.002640286227163723, 'subsample': 0.9998828257449857, 'colsample_bytree': 0.8672117166264461, 'learning_rate': 0.03192932647983711, 'n_estimators': 8795, 'max_depth': 9, 'min_child_weight': 3, 'eta': 0.0002035955028586303, 'gamma': 4.612280340715516e-05, 'grow_policy': 'depthwise', 'sample_type': 'uniform', 'normalize_type': 'forest', 'rate_drop': 1.9650748289902546e-06, 'skip_drop': 1.0602210441619153e-08}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:33:41,531]\u001b[0m Trial 57 finished with value: 2593.15771484375 and parameters: {'booster': 'gbtree', 'lambda': 4.705736799341885e-08, 'alpha': 0.00036730869095987374, 'subsample': 0.9527848463521758, 'colsample_bytree': 0.949090966795769, 'learning_rate': 0.019074590425499253, 'n_estimators': 9695, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.0462183500535517e-05, 'gamma': 3.2233563376758253e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:33:54,232]\u001b[0m Trial 58 finished with value: 2742.65185546875 and parameters: {'booster': 'gbtree', 'lambda': 1.6382231822010473e-07, 'alpha': 0.009852389730529841, 'subsample': 0.9221082667456295, 'colsample_bytree': 0.45773219497504614, 'learning_rate': 0.030691035108909005, 'n_estimators': 7341, 'max_depth': 9, 'min_child_weight': 10, 'eta': 5.2644254655001655e-05, 'gamma': 3.6716570050084463e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:34:19,147]\u001b[0m Trial 59 finished with value: 2737.811767578125 and parameters: {'booster': 'gbtree', 'lambda': 2.6209141182405526e-08, 'alpha': 0.12982277685427146, 'subsample': 0.9770261492126021, 'colsample_bytree': 0.5099126238460371, 'learning_rate': 0.011074355356074646, 'n_estimators': 6802, 'max_depth': 9, 'min_child_weight': 10, 'eta': 0.0003868767827848489, 'gamma': 1.4068802820437216e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:34:32,281]\u001b[0m Trial 60 finished with value: 2660.080078125 and parameters: {'booster': 'gbtree', 'lambda': 9.936030342979702e-08, 'alpha': 0.03172860484089047, 'subsample': 0.937083806935261, 'colsample_bytree': 0.946127934968749, 'learning_rate': 0.021351172115592655, 'n_estimators': 8732, 'max_depth': 5, 'min_child_weight': 5, 'eta': 4.813794996133209e-06, 'gamma': 0.00010134504981485058, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:35:03,344]\u001b[0m Trial 61 finished with value: 2586.786865234375 and parameters: {'booster': 'gbtree', 'lambda': 1.1139506623926795e-08, 'alpha': 0.008127373071272875, 'subsample': 0.8957257678105387, 'colsample_bytree': 0.9990973005219637, 'learning_rate': 0.013388540889411875, 'n_estimators': 7639, 'max_depth': 9, 'min_child_weight': 10, 'eta': 8.610692564504657e-07, 'gamma': 7.646633838134891e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:35:33,624]\u001b[0m Trial 62 finished with value: 2585.8857421875 and parameters: {'booster': 'gbtree', 'lambda': 1.6170047655402192e-08, 'alpha': 0.008618064731415338, 'subsample': 0.8996830162245703, 'colsample_bytree': 0.8904372315668374, 'learning_rate': 0.01444882057245225, 'n_estimators': 7408, 'max_depth': 9, 'min_child_weight': 10, 'eta': 2.8303815931526573e-07, 'gamma': 1.5118567815728635e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:36:06,591]\u001b[0m Trial 63 finished with value: 2948.52978515625 and parameters: {'booster': 'gbtree', 'lambda': 1.0951058074116541e-08, 'alpha': 0.010575782614741227, 'subsample': 0.897117427414872, 'colsample_bytree': 0.3190285313202529, 'learning_rate': 0.005181307424123595, 'n_estimators': 7451, 'max_depth': 9, 'min_child_weight': 10, 'eta': 2.569348324896366e-07, 'gamma': 6.870013974296644e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:36:31,631]\u001b[0m Trial 64 finished with value: 2586.9599609375 and parameters: {'booster': 'gbtree', 'lambda': 2.135259062566528e-08, 'alpha': 0.0028449242440868588, 'subsample': 0.9632923795075028, 'colsample_bytree': 0.8993224822490145, 'learning_rate': 0.016691738241088855, 'n_estimators': 5751, 'max_depth': 9, 'min_child_weight': 10, 'eta': 5.774639792210682e-07, 'gamma': 9.067833204579731e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:36:56,413]\u001b[0m Trial 65 finished with value: 2590.15185546875 and parameters: {'booster': 'gbtree', 'lambda': 1.898300059897173e-08, 'alpha': 0.00022003082689017048, 'subsample': 0.9191203525372306, 'colsample_bytree': 0.8894346092411041, 'learning_rate': 0.018695074457049973, 'n_estimators': 5880, 'max_depth': 9, 'min_child_weight': 9, 'eta': 0.5335141069568368, 'gamma': 4.4252312481975777e-05, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:36:59,035]\u001b[0m Trial 66 finished with value: 2878.684814453125 and parameters: {'booster': 'gblinear', 'lambda': 0.5966917599170334, 'alpha': 0.0019072649796605479, 'subsample': 0.9527066918276981, 'colsample_bytree': 0.7986440467639434, 'learning_rate': 0.012143475744571829, 'n_estimators': 6579}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:37:04,370]\u001b[0m Trial 67 finished with value: 2727.469482421875 and parameters: {'booster': 'gbtree', 'lambda': 4.520403262207075e-08, 'alpha': 0.007668403464345669, 'subsample': 0.9630999865920666, 'colsample_bytree': 0.8415534239085812, 'learning_rate': 0.028215759692444185, 'n_estimators': 6097, 'max_depth': 3, 'min_child_weight': 10, 'eta': 0.054176712237923405, 'gamma': 8.753332797313315e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:37:21,154]\u001b[0m Trial 68 finished with value: 2591.468994140625 and parameters: {'booster': 'gbtree', 'lambda': 2.915237590977722e-07, 'alpha': 0.0008257998894319243, 'subsample': 0.8723383030602869, 'colsample_bytree': 0.8949746314315348, 'learning_rate': 0.038248080244338206, 'n_estimators': 5080, 'max_depth': 9, 'min_child_weight': 4, 'eta': 7.201678765684428e-07, 'gamma': 3.113098815549765e-05, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:38:31,276]\u001b[0m Trial 69 finished with value: 2598.779541015625 and parameters: {'booster': 'dart', 'lambda': 2.1885007714381228e-08, 'alpha': 0.00405510282052836, 'subsample': 0.9998938431819124, 'colsample_bytree': 0.9886337066913655, 'learning_rate': 0.009554303241592996, 'n_estimators': 3350, 'max_depth': 9, 'min_child_weight': 9, 'eta': 2.918643106707706e-07, 'gamma': 1.1195571118648569e-06, 'grow_policy': 'depthwise', 'sample_type': 'weighted', 'normalize_type': 'forest', 'rate_drop': 0.021888488969108697, 'skip_drop': 0.006236147589561319}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:38:53,589]\u001b[0m Trial 70 finished with value: 2622.1708984375 and parameters: {'booster': 'gbtree', 'lambda': 6.461058557260219e-07, 'alpha': 0.0015991488807597562, 'subsample': 0.9016485864372392, 'colsample_bytree': 0.7938466582033404, 'learning_rate': 0.021364551736375033, 'n_estimators': 5325, 'max_depth': 9, 'min_child_weight': 10, 'eta': 0.0052731575624274584, 'gamma': 2.3981908852722563e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:39:25,035]\u001b[0m Trial 71 finished with value: 2585.574951171875 and parameters: {'booster': 'gbtree', 'lambda': 1.0062670853353627e-08, 'alpha': 0.016276782756644792, 'subsample': 0.9744974851483097, 'colsample_bytree': 0.941657984194292, 'learning_rate': 0.013278628870347734, 'n_estimators': 6692, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.4774133564582594e-06, 'gamma': 4.5245388810337934e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:39:50,302]\u001b[0m Trial 72 finished with value: 2587.451416015625 and parameters: {'booster': 'gbtree', 'lambda': 4.420823985999235e-08, 'alpha': 0.009086958768433625, 'subsample': 0.9344961613164915, 'colsample_bytree': 0.9988234661780674, 'learning_rate': 0.01730631652235394, 'n_estimators': 6721, 'max_depth': 9, 'min_child_weight': 10, 'eta': 5.658898965215104e-07, 'gamma': 5.949884733581502e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:40:10,260]\u001b[0m Trial 73 finished with value: 2591.08740234375 and parameters: {'booster': 'gbtree', 'lambda': 1.742984945503642e-08, 'alpha': 0.00047308599249660637, 'subsample': 0.8548277060598591, 'colsample_bytree': 0.9332788174530655, 'learning_rate': 0.026593794802621563, 'n_estimators': 7437, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.4168504268571617e-06, 'gamma': 3.801183456391986e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:40:45,783]\u001b[0m Trial 74 finished with value: 2627.203369140625 and parameters: {'booster': 'gbtree', 'lambda': 1.1334737042733238e-07, 'alpha': 0.014761210571295376, 'subsample': 0.9636572353750382, 'colsample_bytree': 0.8774654315541579, 'learning_rate': 0.010050771050582812, 'n_estimators': 7968, 'max_depth': 9, 'min_child_weight': 9, 'eta': 1.9134268143295456e-06, 'gamma': 1.6279883864566193e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:40:55,942]\u001b[0m Trial 75 finished with value: 2624.81884765625 and parameters: {'booster': 'gbtree', 'lambda': 2.119188419206297e-08, 'alpha': 0.11461824985489107, 'subsample': 0.21377323214250155, 'colsample_bytree': 0.9389968949718861, 'learning_rate': 0.04725345360807329, 'n_estimators': 6053, 'max_depth': 9, 'min_child_weight': 9, 'eta': 5.744371249196856e-08, 'gamma': 0.00039946648661630857, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:41:17,668]\u001b[0m Trial 76 finished with value: 2594.971435546875 and parameters: {'booster': 'gbtree', 'lambda': 3.7099327725520766e-08, 'alpha': 0.004291665379983457, 'subsample': 0.9027855623090156, 'colsample_bytree': 0.9035499517342359, 'learning_rate': 0.02304326835824016, 'n_estimators': 6888, 'max_depth': 9, 'min_child_weight': 10, 'eta': 3.72904660581451e-07, 'gamma': 2.3541956482635845e-08, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:41:34,888]\u001b[0m Trial 77 finished with value: 2595.000244140625 and parameters: {'booster': 'gbtree', 'lambda': 6.352668934325197e-08, 'alpha': 0.002640450583510863, 'subsample': 0.9750505943074145, 'colsample_bytree': 0.9735118649178192, 'learning_rate': 0.03579098334632447, 'n_estimators': 7204, 'max_depth': 9, 'min_child_weight': 10, 'eta': 3.0586473501384406e-06, 'gamma': 3.18002554965244e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:42:19,802]\u001b[0m Trial 78 finished with value: 2628.013916015625 and parameters: {'booster': 'gbtree', 'lambda': 1.8362990963570934e-07, 'alpha': 7.20744348950351e-05, 'subsample': 0.9339830315143123, 'colsample_bytree': 0.8430308978860528, 'learning_rate': 0.005799061744370629, 'n_estimators': 8375, 'max_depth': 7, 'min_child_weight': 9, 'eta': 9.680838389273873e-07, 'gamma': 6.25570567414944e-06, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:42:21,656]\u001b[0m Trial 79 finished with value: 2766.893310546875 and parameters: {'booster': 'gblinear', 'lambda': 1.562672518305901e-08, 'alpha': 0.01149869128813657, 'subsample': 0.8834473227182761, 'colsample_bytree': 0.7435292940060954, 'learning_rate': 0.015240681831528543, 'n_estimators': 6479}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:42:33,094]\u001b[0m Trial 80 finished with value: 2723.0810546875 and parameters: {'booster': 'gbtree', 'lambda': 2.9225915090765356e-08, 'alpha': 0.006341237142233533, 'subsample': 0.9445043590271274, 'colsample_bytree': 0.6248843884761475, 'learning_rate': 0.04222991920549522, 'n_estimators': 7614, 'max_depth': 9, 'min_child_weight': 10, 'eta': 8.48447424796003e-08, 'gamma': 2.4711853806789853e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:42:56,066]\u001b[0m Trial 81 finished with value: 2593.391357421875 and parameters: {'booster': 'gbtree', 'lambda': 4.936107791987229e-08, 'alpha': 0.007660018494766602, 'subsample': 0.9234275117286334, 'colsample_bytree': 0.9963555458277292, 'learning_rate': 0.01972887913708005, 'n_estimators': 6579, 'max_depth': 9, 'min_child_weight': 10, 'eta': 6.791466822726918e-07, 'gamma': 4.6649828882023657e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:43:21,793]\u001b[0m Trial 82 finished with value: 2586.089599609375 and parameters: {'booster': 'gbtree', 'lambda': 1.4627880486425282e-08, 'alpha': 0.028643817001187774, 'subsample': 0.9985214192245133, 'colsample_bytree': 0.9999002172151105, 'learning_rate': 0.017454639366827934, 'n_estimators': 6849, 'max_depth': 9, 'min_child_weight': 10, 'eta': 1.7999867815960376e-07, 'gamma': 5.107201521456082e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:43:40,046]\u001b[0m Trial 83 finished with value: 2590.256103515625 and parameters: {'booster': 'gbtree', 'lambda': 1.0044333970316545e-08, 'alpha': 0.032374853566776995, 'subsample': 0.9949955338225979, 'colsample_bytree': 0.934743303536836, 'learning_rate': 0.02757836667108235, 'n_estimators': 5477, 'max_depth': 9, 'min_child_weight': 10, 'eta': 2.184511169086749e-07, 'gamma': 9.89822106254857e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:44:22,135]\u001b[0m Trial 84 finished with value: 2598.173583984375 and parameters: {'booster': 'gbtree', 'lambda': 1.7157714583077985e-08, 'alpha': 0.014553334370900672, 'subsample': 0.9647004300473181, 'colsample_bytree': 0.9617938458408122, 'learning_rate': 0.009225780970984032, 'n_estimators': 1413, 'max_depth': 9, 'min_child_weight': 9, 'eta': 1.9510657926296226e-07, 'gamma': 8.214801890021438e-08, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:44:49,581]\u001b[0m Trial 85 finished with value: 2583.8427734375 and parameters: {'booster': 'gbtree', 'lambda': 9.610174762388367e-08, 'alpha': 0.0017716943427067981, 'subsample': 0.9092407872426185, 'colsample_bytree': 0.9060198605807529, 'learning_rate': 0.015613871741587446, 'n_estimators': 7208, 'max_depth': 9, 'min_child_weight': 10, 'eta': 4.1615364011184144e-07, 'gamma': 1.0764928111280895e-07, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:45:10,935]\u001b[0m Trial 86 finished with value: 2590.525390625 and parameters: {'booster': 'gbtree', 'lambda': 9.661443443981701e-08, 'alpha': 0.0018795382715150312, 'subsample': 0.9091334008753587, 'colsample_bytree': 0.9010773032611396, 'learning_rate': 0.023685310746543585, 'n_estimators': 7079, 'max_depth': 9, 'min_child_weight': 10, 'eta': 5.166234426039787e-07, 'gamma': 2.912329604865437e-08, 'grow_policy': 'depthwise'}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:45:34,812]\u001b[0m Trial 87 finished with value: 2626.871337890625 and parameters: {'booster': 'dart', 'lambda': 6.331976425347971e-08, 'alpha': 1.769638915279272e-06, 'subsample': 0.8438590106878328, 'colsample_bytree': 0.9261141383030829, 'learning_rate': 0.03250327658547643, 'n_estimators': 7296, 'max_depth': 9, 'min_child_weight': 9, 'eta': 3.1853831738074564e-08, 'gamma': 9.951994065752461e-08, 'grow_policy': 'depthwise', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 0.004044712692929144, 'skip_drop': 7.008079350083586e-07}. Best is trial 46 with value: 2580.17333984375.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:46:02,883]\u001b[0m Trial 88 finished with value: 2568.92138671875 and parameters: {'booster': 'gbtree', 'lambda': 3.1808957948248396e-07, 'alpha': 0.43323872963149074, 'subsample': 0.8695619977650685, 'colsample_bytree': 0.9733178919388802, 'learning_rate': 0.017721644999776395, 'n_estimators': 6244, 'max_depth': 9, 'min_child_weight': 3, 'eta': 1.1147164107354344e-06, 'gamma': 1.8791466279302114e-07, 'grow_policy': 'depthwise'}. Best is trial 88 with value: 2568.92138671875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:46:13,592]\u001b[0m Trial 89 finished with value: 2592.968505859375 and parameters: {'booster': 'gbtree', 'lambda': 3.659801525250525e-07, 'alpha': 0.5566607902970329, 'subsample': 0.8216633342322566, 'colsample_bytree': 0.9706340368756912, 'learning_rate': 0.09940087491938338, 'n_estimators': 6355, 'max_depth': 9, 'min_child_weight': 3, 'eta': 1.2140495549903485e-06, 'gamma': 1.4662979973181361e-08, 'grow_policy': 'depthwise'}. Best is trial 88 with value: 2568.92138671875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:47:00,973]\u001b[0m Trial 90 finished with value: 2576.181884765625 and parameters: {'booster': 'gbtree', 'lambda': 8.975459489806879e-07, 'alpha': 0.3735104962693831, 'subsample': 0.8773423863178772, 'colsample_bytree': 0.9796191803676411, 'learning_rate': 0.008591485294654675, 'n_estimators': 7504, 'max_depth': 9, 'min_child_weight': 6, 'eta': 1.380654831699079e-05, 'gamma': 2.0890044375691925e-07, 'grow_policy': 'lossguide'}. Best is trial 88 with value: 2568.92138671875.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:47:50,777]\u001b[0m Trial 91 finished with value: 2557.924072265625 and parameters: {'booster': 'gbtree', 'lambda': 8.544792472633987e-07, 'alpha': 0.31141671752487043, 'subsample': 0.8779467596981366, 'colsample_bytree': 0.9759532762677546, 'learning_rate': 0.008686087328805853, 'n_estimators': 6988, 'max_depth': 9, 'min_child_weight': 2, 'eta': 3.7603213457541647e-06, 'gamma': 2.1478058456847449e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:48:44,873]\u001b[0m Trial 92 finished with value: 2558.794677734375 and parameters: {'booster': 'gbtree', 'lambda': 6.708021432907738e-06, 'alpha': 0.5012165448790593, 'subsample': 0.8716536907459954, 'colsample_bytree': 0.9808364402916253, 'learning_rate': 0.008130549513657593, 'n_estimators': 6951, 'max_depth': 9, 'min_child_weight': 2, 'eta': 1.3227379305072605e-05, 'gamma': 1.9483772094880706e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:48:54,123]\u001b[0m Trial 93 finished with value: 2621.61474609375 and parameters: {'booster': 'gbtree', 'lambda': 1.1185730370915769e-06, 'alpha': 0.5009685557109057, 'subsample': 0.8076919033387168, 'colsample_bytree': 0.9787181719636496, 'learning_rate': 0.19709108041559736, 'n_estimators': 6977, 'max_depth': 9, 'min_child_weight': 2, 'eta': 1.548961147584967e-05, 'gamma': 1.8755368650182214e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:49:42,155]\u001b[0m Trial 94 finished with value: 2562.32763671875 and parameters: {'booster': 'gbtree', 'lambda': 6.958517366882396e-07, 'alpha': 0.31124950663093176, 'subsample': 0.859928356921421, 'colsample_bytree': 0.9822492733924449, 'learning_rate': 0.008823520296746596, 'n_estimators': 7874, 'max_depth': 9, 'min_child_weight': 3, 'eta': 3.5525308389230076e-05, 'gamma': 2.3423391146078386e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:50:35,810]\u001b[0m Trial 95 finished with value: 2567.789794921875 and parameters: {'booster': 'gbtree', 'lambda': 4.919452188561132e-06, 'alpha': 0.34970844539409757, 'subsample': 0.8608229344852731, 'colsample_bytree': 0.9173028148691831, 'learning_rate': 0.008106382397602937, 'n_estimators': 7906, 'max_depth': 9, 'min_child_weight': 3, 'eta': 1.6527045709553438e-05, 'gamma': 2.841437623171489e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:51:55,393]\u001b[0m Trial 96 finished with value: 2566.376708984375 and parameters: {'booster': 'gbtree', 'lambda': 3.041578050521905e-06, 'alpha': 0.36041342355997735, 'subsample': 0.870607944804775, 'colsample_bytree': 0.9239854328150887, 'learning_rate': 0.005024618380474211, 'n_estimators': 6200, 'max_depth': 9, 'min_child_weight': 3, 'eta': 3.480104847972646e-05, 'gamma': 2.725149947539707e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:52:42,771]\u001b[0m Trial 97 finished with value: 2598.159912109375 and parameters: {'booster': 'gbtree', 'lambda': 3.927753487642491e-06, 'alpha': 0.31328611669867346, 'subsample': 0.8710416789873848, 'colsample_bytree': 0.9740986996051499, 'learning_rate': 0.006355329559824199, 'n_estimators': 7949, 'max_depth': 7, 'min_child_weight': 3, 'eta': 4.0840745695991706e-05, 'gamma': 2.7165586519720427e-07, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:53:38,294]\u001b[0m Trial 98 finished with value: 2566.851318359375 and parameters: {'booster': 'gbtree', 'lambda': 5.9965292187552264e-06, 'alpha': 0.7447132228312693, 'subsample': 0.8459760057598786, 'colsample_bytree': 0.9198470793234116, 'learning_rate': 0.007508001129364852, 'n_estimators': 7830, 'max_depth': 9, 'min_child_weight': 3, 'eta': 1.5925693341472015e-05, 'gamma': 0.03493191181626064, 'grow_policy': 'lossguide'}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n",
      "\u001b[32m[I 2021-12-22 13:53:41,915]\u001b[0m Trial 99 finished with value: 2738.994384765625 and parameters: {'booster': 'gblinear', 'lambda': 9.09136836214634e-06, 'alpha': 0.6640268611428268, 'subsample': 0.8508677174614252, 'colsample_bytree': 0.9270116074521553, 'learning_rate': 0.008692759548739637, 'n_estimators': 6135}. Best is trial 91 with value: 2557.924072265625.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3h 45min 1s, sys: 14.9 s, total: 3h 45min 16s\n",
      "Wall time: 3h 11min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_xgbr_income = optuna.create_study(direction=\"minimize\")\n",
    "study_xgbr_income.optimize(objective_xgbr_income ,n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "41cd2b3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T13:53:42.042543Z",
     "iopub.status.busy": "2021-12-22T13:53:42.027366Z",
     "iopub.status.idle": "2021-12-22T13:53:42.050433Z",
     "shell.execute_reply": "2021-12-22T13:53:42.049983Z",
     "shell.execute_reply.started": "2021-12-22T10:40:39.533863Z"
    },
    "papermill": {
     "duration": 0.080574,
     "end_time": "2021-12-22T13:53:42.050556",
     "exception": false,
     "start_time": "2021-12-22T13:53:41.969982",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of finished trials :  100\n",
      "Best Trials :  {'booster': 'gbtree', 'lambda': 8.544792472633987e-07, 'alpha': 0.31141671752487043, 'subsample': 0.8779467596981366, 'colsample_bytree': 0.9759532762677546, 'learning_rate': 0.008686087328805853, 'n_estimators': 6988, 'max_depth': 9, 'min_child_weight': 2, 'eta': 3.7603213457541647e-06, 'gamma': 2.1478058456847449e-07, 'grow_policy': 'lossguide'}\n",
      "Best Values :  2557.924072265625\n"
     ]
    }
   ],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_xgbr_income.trials))\n",
    "print(\"Best Trials : \", study_xgbr_income.best_trial.params)\n",
    "print(\"Best Values : \" , study_xgbr_income.best_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fbc0687e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T13:53:42.143647Z",
     "iopub.status.busy": "2021-12-22T13:53:42.142731Z",
     "iopub.status.idle": "2021-12-22T13:53:42.145129Z",
     "shell.execute_reply": "2021-12-22T13:53:42.144599Z",
     "shell.execute_reply.started": "2021-12-22T10:40:39.545298Z"
    },
    "papermill": {
     "duration": 0.050346,
     "end_time": "2021-12-22T13:53:42.145243",
     "exception": false,
     "start_time": "2021-12-22T13:53:42.094897",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#testdf_NumberOfDependents= train_final[train_final['NumberOfDependents'].isnull()==True]\n",
    "#traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "#y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "#traindf_NumberOfDependents.drop(\"NumberOfDependents\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c88c12cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-22T13:53:42.236911Z",
     "iopub.status.busy": "2021-12-22T13:53:42.236165Z",
     "iopub.status.idle": "2021-12-22T13:53:42.238703Z",
     "shell.execute_reply": "2021-12-22T13:53:42.238262Z",
     "shell.execute_reply.started": "2021-12-22T10:40:39.554221Z"
    },
    "papermill": {
     "duration": 0.049882,
     "end_time": "2021-12-22T13:53:42.238821",
     "exception": false,
     "start_time": "2021-12-22T13:53:42.188939",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save the predictions to a CSV file\n",
    "#sub = pd.read_csv('../input/GiveMeSomeCredit/sampleEntry.csv')\n",
    "#sub['Probability']=predictions\n",
    "#sub.to_csv('pipe_lgbm_catboost_qauntile_new.csv', index=False)\n",
    "#sub"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 11522.551448,
   "end_time": "2021-12-22T13:53:43.514486",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-22T10:41:40.963038",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}