credit-part3-modeling-1-pipe1-tune-catboost.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "50162cc1",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:41.194087Z",
     "iopub.status.busy": "2021-12-25T21:27:41.192701Z",
     "iopub.status.idle": "2021-12-25T21:27:44.877369Z",
     "shell.execute_reply": "2021-12-25T21:27:44.876729Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.719104,
     "end_time": "2021-12-25T21:27:44.877541",
     "exception": false,
     "start_time": "2021-12-25T21:27:41.158437",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1eb9d18",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.028771,
     "end_time": "2021-12-25T21:27:44.935489",
     "exception": false,
     "start_time": "2021-12-25T21:27:44.906718",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d9bcf5c6",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:44.999219Z",
     "iopub.status.busy": "2021-12-25T21:27:44.998466Z",
     "iopub.status.idle": "2021-12-25T21:27:45.444914Z",
     "shell.execute_reply": "2021-12-25T21:27:45.445409Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.481372,
     "end_time": "2021-12-25T21:27:45.445576",
     "exception": false,
     "start_time": "2021-12-25T21:27:44.964204",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 228 ms, sys: 97 ms, total: 325 ms\n",
      "Wall time: 428 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2cb9791",
   "metadata": {
    "papermill": {
     "duration": 0.029143,
     "end_time": "2021-12-25T21:27:45.504514",
     "exception": false,
     "start_time": "2021-12-25T21:27:45.475371",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2001b27b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:45.590987Z",
     "iopub.status.busy": "2021-12-25T21:27:45.572403Z",
     "iopub.status.idle": "2021-12-25T21:27:45.659144Z",
     "shell.execute_reply": "2021-12-25T21:27:45.658601Z"
    },
    "papermill": {
     "duration": 0.124578,
     "end_time": "2021-12-25T21:27:45.659291",
     "exception": false,
     "start_time": "2021-12-25T21:27:45.534713",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "371bff0c",
   "metadata": {
    "papermill": {
     "duration": 0.029085,
     "end_time": "2021-12-25T21:27:45.717962",
     "exception": false,
     "start_time": "2021-12-25T21:27:45.688877",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4614707a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:45.787523Z",
     "iopub.status.busy": "2021-12-25T21:27:45.786543Z",
     "iopub.status.idle": "2021-12-25T21:27:45.842407Z",
     "shell.execute_reply": "2021-12-25T21:27:45.841713Z"
    },
    "papermill": {
     "duration": 0.093896,
     "end_time": "2021-12-25T21:27:45.842541",
     "exception": false,
     "start_time": "2021-12-25T21:27:45.748645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e76d55c6",
   "metadata": {
    "papermill": {
     "duration": 0.030204,
     "end_time": "2021-12-25T21:27:45.902529",
     "exception": false,
     "start_time": "2021-12-25T21:27:45.872325",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# X and Y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "708dd033",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:45.969089Z",
     "iopub.status.busy": "2021-12-25T21:27:45.965586Z",
     "iopub.status.idle": "2021-12-25T21:27:45.971848Z",
     "shell.execute_reply": "2021-12-25T21:27:45.971294Z"
    },
    "papermill": {
     "duration": 0.039111,
     "end_time": "2021-12-25T21:27:45.972023",
     "exception": false,
     "start_time": "2021-12-25T21:27:45.932912",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0620620",
   "metadata": {
    "papermill": {
     "duration": 0.029487,
     "end_time": "2021-12-25T21:27:46.031268",
     "exception": false,
     "start_time": "2021-12-25T21:27:46.001781",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3573bf88",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:46.096587Z",
     "iopub.status.busy": "2021-12-25T21:27:46.095638Z",
     "iopub.status.idle": "2021-12-25T21:27:46.098427Z",
     "shell.execute_reply": "2021-12-25T21:27:46.097910Z"
    },
    "papermill": {
     "duration": 0.037799,
     "end_time": "2021-12-25T21:27:46.098564",
     "exception": false,
     "start_time": "2021-12-25T21:27:46.060765",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1286e97",
   "metadata": {
    "papermill": {
     "duration": 0.03086,
     "end_time": "2021-12-25T21:27:46.159610",
     "exception": false,
     "start_time": "2021-12-25T21:27:46.128750",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Impute NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f532e6f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:46.229342Z",
     "iopub.status.busy": "2021-12-25T21:27:46.223827Z",
     "iopub.status.idle": "2021-12-25T21:27:46.233639Z",
     "shell.execute_reply": "2021-12-25T21:27:46.234237Z"
    },
    "papermill": {
     "duration": 0.04452,
     "end_time": "2021-12-25T21:27:46.234397",
     "exception": false,
     "start_time": "2021-12-25T21:27:46.189877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 305 µs, sys: 51 µs, total: 356 µs\n",
      "Wall time: 360 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            'early_stopping_rounds':100 ,\n",
    "           # 'gpu_id':0, \n",
    "           # 'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "            # 'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "466e019c",
   "metadata": {
    "papermill": {
     "duration": 0.030101,
     "end_time": "2021-12-25T21:27:46.294744",
     "exception": false,
     "start_time": "2021-12-25T21:27:46.264643",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# XGBR regression imputer \n",
    "## Predict income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "207a41ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:27:46.359117Z",
     "iopub.status.busy": "2021-12-25T21:27:46.358462Z",
     "iopub.status.idle": "2021-12-25T21:46:14.982093Z",
     "shell.execute_reply": "2021-12-25T21:46:14.982620Z"
    },
    "papermill": {
     "duration": 1108.658012,
     "end_time": "2021-12-25T21:46:14.982805",
     "exception": false,
     "start_time": "2021-12-25T21:27:46.324793",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 12min 5s, sys: 5.63 s, total: 1h 12min 11s\n",
      "Wall time: 18min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "64ae982f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:46:15.050864Z",
     "iopub.status.busy": "2021-12-25T21:46:15.050207Z",
     "iopub.status.idle": "2021-12-25T21:46:15.066590Z",
     "shell.execute_reply": "2021-12-25T21:46:15.065594Z"
    },
    "papermill": {
     "duration": 0.052713,
     "end_time": "2021-12-25T21:46:15.066833",
     "exception": false,
     "start_time": "2021-12-25T21:46:15.014120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    146076 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c632a2c6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:46:15.134942Z",
     "iopub.status.busy": "2021-12-25T21:46:15.134355Z",
     "iopub.status.idle": "2021-12-25T21:46:15.148859Z",
     "shell.execute_reply": "2021-12-25T21:46:15.148391Z"
    },
    "papermill": {
     "duration": 0.048744,
     "end_time": "2021-12-25T21:46:15.148997",
     "exception": false,
     "start_time": "2021-12-25T21:46:15.100253",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    98877 non-null   float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9eab025a",
   "metadata": {
    "papermill": {
     "duration": 0.031637,
     "end_time": "2021-12-25T21:46:15.212781",
     "exception": false,
     "start_time": "2021-12-25T21:46:15.181144",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict number of dependents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "50ac0038",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T21:46:15.283727Z",
     "iopub.status.busy": "2021-12-25T21:46:15.283133Z",
     "iopub.status.idle": "2021-12-25T22:08:35.545214Z",
     "shell.execute_reply": "2021-12-25T22:08:35.545738Z"
    },
    "papermill": {
     "duration": 1340.301295,
     "end_time": "2021-12-25T22:08:35.545920",
     "exception": false,
     "start_time": "2021-12-25T21:46:15.244625",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 27min 15s, sys: 5.41 s, total: 1h 27min 21s\n",
      "Wall time: 22min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b440eb94",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:35.615719Z",
     "iopub.status.busy": "2021-12-25T22:08:35.615106Z",
     "iopub.status.idle": "2021-12-25T22:08:35.627780Z",
     "shell.execute_reply": "2021-12-25T22:08:35.629156Z"
    },
    "papermill": {
     "duration": 0.05093,
     "end_time": "2021-12-25T22:08:35.629399",
     "exception": false,
     "start_time": "2021-12-25T22:08:35.578469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    150000 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5cebe49a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:35.701306Z",
     "iopub.status.busy": "2021-12-25T22:08:35.700626Z",
     "iopub.status.idle": "2021-12-25T22:08:35.713137Z",
     "shell.execute_reply": "2021-12-25T22:08:35.714098Z"
    },
    "papermill": {
     "duration": 0.05122,
     "end_time": "2021-12-25T22:08:35.714332",
     "exception": false,
     "start_time": "2021-12-25T22:08:35.663112",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    101503 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67fea140",
   "metadata": {
    "papermill": {
     "duration": 0.033095,
     "end_time": "2021-12-25T22:08:35.782702",
     "exception": false,
     "start_time": "2021-12-25T22:08:35.749607",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Add bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d4d465ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:35.852797Z",
     "iopub.status.busy": "2021-12-25T22:08:35.852187Z",
     "iopub.status.idle": "2021-12-25T22:08:36.533954Z",
     "shell.execute_reply": "2021-12-25T22:08:36.533420Z"
    },
    "papermill": {
     "duration": 0.718009,
     "end_time": "2021-12-25T22:08:36.534119",
     "exception": false,
     "start_time": "2021-12-25T22:08:35.816110",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) \n",
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff0c235d",
   "metadata": {
    "papermill": {
     "duration": 0.033546,
     "end_time": "2021-12-25T22:08:36.601147",
     "exception": false,
     "start_time": "2021-12-25T22:08:36.567601",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c498fab4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:36.671383Z",
     "iopub.status.busy": "2021-12-25T22:08:36.670730Z",
     "iopub.status.idle": "2021-12-25T22:08:36.673734Z",
     "shell.execute_reply": "2021-12-25T22:08:36.674211Z"
    },
    "papermill": {
     "duration": 0.03956,
     "end_time": "2021-12-25T22:08:36.674361",
     "exception": false,
     "start_time": "2021-12-25T22:08:36.634801",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b8a6d441",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:36.747031Z",
     "iopub.status.busy": "2021-12-25T22:08:36.746382Z",
     "iopub.status.idle": "2021-12-25T22:08:36.766722Z",
     "shell.execute_reply": "2021-12-25T22:08:36.767488Z"
    },
    "papermill": {
     "duration": 0.05946,
     "end_time": "2021-12-25T22:08:36.767712",
     "exception": false,
     "start_time": "2021-12-25T22:08:36.708252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c9ffb712",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:36.841247Z",
     "iopub.status.busy": "2021-12-25T22:08:36.840605Z",
     "iopub.status.idle": "2021-12-25T22:08:36.862340Z",
     "shell.execute_reply": "2021-12-25T22:08:36.862836Z"
    },
    "papermill": {
     "duration": 0.05954,
     "end_time": "2021-12-25T22:08:36.863026",
     "exception": false,
     "start_time": "2021-12-25T22:08:36.803486",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bd095288",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:36.935836Z",
     "iopub.status.busy": "2021-12-25T22:08:36.935256Z",
     "iopub.status.idle": "2021-12-25T22:08:36.938717Z",
     "shell.execute_reply": "2021-12-25T22:08:36.939295Z"
    },
    "papermill": {
     "duration": 0.041701,
     "end_time": "2021-12-25T22:08:36.939561",
     "exception": false,
     "start_time": "2021-12-25T22:08:36.897860",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfa0b04f",
   "metadata": {
    "papermill": {
     "duration": 0.034429,
     "end_time": "2021-12-25T22:08:37.009044",
     "exception": false,
     "start_time": "2021-12-25T22:08:36.974615",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extract final cat  and num Features \n",
    "## Cat Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d371a76c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.082094Z",
     "iopub.status.busy": "2021-12-25T22:08:37.081439Z",
     "iopub.status.idle": "2021-12-25T22:08:37.091064Z",
     "shell.execute_reply": "2021-12-25T22:08:37.091564Z"
    },
    "papermill": {
     "duration": 0.047585,
     "end_time": "2021-12-25T22:08:37.091716",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.044131",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28379148",
   "metadata": {
    "papermill": {
     "duration": 0.034978,
     "end_time": "2021-12-25T22:08:37.161929",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.126951",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2bf2d785",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.235287Z",
     "iopub.status.busy": "2021-12-25T22:08:37.234670Z",
     "iopub.status.idle": "2021-12-25T22:08:37.245212Z",
     "shell.execute_reply": "2021-12-25T22:08:37.245762Z"
    },
    "papermill": {
     "duration": 0.048943,
     "end_time": "2021-12-25T22:08:37.245923",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.196980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d771336",
   "metadata": {
    "papermill": {
     "duration": 0.035394,
     "end_time": "2021-12-25T22:08:37.316647",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.281253",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2bfe04c6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.391107Z",
     "iopub.status.busy": "2021-12-25T22:08:37.390464Z",
     "iopub.status.idle": "2021-12-25T22:08:37.402864Z",
     "shell.execute_reply": "2021-12-25T22:08:37.403380Z"
    },
    "papermill": {
     "duration": 0.051084,
     "end_time": "2021-12-25T22:08:37.403538",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.352454",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeaturesEngineer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X=X.copy()\n",
    "        # Calculate some metrics across rows\n",
    "        X[\"num_missing\"]  = X.isnull().sum(axis=1)\n",
    "        X[\"std_row\"] = X.std(axis=1)\n",
    "        X[\"sem_row\"] = X.sem(axis=1)\n",
    "        X[\"abs_sum_row\"] = X.abs().sum(axis=1)\n",
    "        X[\"mean_row\"] = X.mean(axis=1)\n",
    "        X[\"max_row\"]= X.max(axis=1)\n",
    "        X[\"min_row\"]= X.min(axis=1)\n",
    "        X['Weighted_Sum_PastDue'] = 2 * X['NumberOfTime30-59DaysPastDueNotWorse'] + 3 * X['NumberOfTime60-89DaysPastDueNotWorse'] + 6 * X['NumberOfTimes90DaysLate']\n",
    "        X['90days_out_of_TotalPastDue'] = X['NumberOfTimes90DaysLate'] / (X['Weighted_Sum_PastDue'])\n",
    "        X.loc[X['Weighted_Sum_PastDue']==0, '90days_out_of_TotalPastDue'] = 0\n",
    "        X['RemainingLines'] = X['NumberOfOpenCreditLinesAndLoans'] - X['NumberRealEstateLoansOrLines']\n",
    "        X['Loans_vs_Other_Lines'] = X['RemainingLines'] / (1 + X['NumberRealEstateLoansOrLines'])\n",
    "        X['Debt_per_Real_Estate_Loan'] = X['DebtRatio'] / X['NumberRealEstateLoansOrLines']\n",
    "        X.loc[X['NumberRealEstateLoansOrLines']==0, 'Debt_per_Real_Estate_Loan'] = 0\n",
    "        X['Disposable_Income_per_person'] = (X['MonthlyIncome'] - X['DebtRatio']) / (X['NumberOfDependents'] + 1)\n",
    "        X.loc[X['NumberOfDependents']==0, 'Disposable_Income_per_person'] = 0\n",
    "        X['RemainingLines_per_person'] = X['RemainingLines'] / (X['NumberOfDependents'] + 1)\n",
    "        X['NumberRE_X_DebtRatio_X_age'] = X['NumberRealEstateLoansOrLines'] * X['DebtRatio'] / X['age']\n",
    "        X['RevolvingUtilization_per_age'] = X['RevolvingUtilizationOfUnsecuredLines'] / X['age']\n",
    "        X[X==np.inf]=np.nan\n",
    "        X=reduce_mem_usage(X)\n",
    "        return X  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6cf4661a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.479844Z",
     "iopub.status.busy": "2021-12-25T22:08:37.479237Z",
     "iopub.status.idle": "2021-12-25T22:08:37.487391Z",
     "shell.execute_reply": "2021-12-25T22:08:37.487883Z"
    },
    "papermill": {
     "duration": 0.047823,
     "end_time": "2021-12-25T22:08:37.488062",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.440239",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8cd51628",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.571305Z",
     "iopub.status.busy": "2021-12-25T22:08:37.570642Z",
     "iopub.status.idle": "2021-12-25T22:08:37.572417Z",
     "shell.execute_reply": "2021-12-25T22:08:37.572988Z"
    },
    "papermill": {
     "duration": 0.047557,
     "end_time": "2021-12-25T22:08:37.573162",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.525605",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "               # ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "deac448f",
   "metadata": {
    "papermill": {
     "duration": 0.035784,
     "end_time": "2021-12-25T22:08:37.646450",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.610666",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# First pipe to tune : \n",
    "imputer + transformer + kmeans fe+ploy fe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f1be8088",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.721506Z",
     "iopub.status.busy": "2021-12-25T22:08:37.720903Z",
     "iopub.status.idle": "2021-12-25T22:08:37.796184Z",
     "shell.execute_reply": "2021-12-25T22:08:37.796647Z"
    },
    "papermill": {
     "duration": 0.114263,
     "end_time": "2021-12-25T22:08:37.796810",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.682547",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4ed4ceaf-7de7-4bc5-a960-8ebdc2a6e502\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4ed4ceaf-7de7-4bc5-a960-8ebdc2a6e502\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"6f420f9f-df49-46e0-960a-2358c1a769ef\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"6f420f9f-df49-46e0-960a-2358c1a769ef\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"e62bf044-2231-48c7-9613-769618280b12\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"e62bf044-2231-48c7-9613-769618280b12\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ffb93935-876e-4813-83a1-a5dc641e17b3\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ffb93935-876e-4813-83a1-a5dc641e17b3\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ff53a8a2-9b8d-4fa9-911b-05926b5d8430\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ff53a8a2-9b8d-4fa9-911b-05926b5d8430\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"e03a885d-8b26-4833-9c2f-808f9c20cdfd\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"e03a885d-8b26-4833-9c2f-808f9c20cdfd\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"13a81c9d-3494-40ce-9860-fe422c6e5a59\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"13a81c9d-3494-40ce-9860-fe422c6e5a59\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "11eb99c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:37.873709Z",
     "iopub.status.busy": "2021-12-25T22:08:37.873113Z",
     "iopub.status.idle": "2021-12-25T22:08:42.159886Z",
     "shell.execute_reply": "2021-12-25T22:08:42.159302Z"
    },
    "papermill": {
     "duration": 4.326528,
     "end_time": "2021-12-25T22:08:42.160060",
     "exception": false,
     "start_time": "2021-12-25T22:08:37.833532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 78)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer.fit(X,y)\n",
    "X_pre=preprocessor_woe_powertransformer.transform(X)\n",
    "X_pre.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "519ed055",
   "metadata": {
    "papermill": {
     "duration": 0.03718,
     "end_time": "2021-12-25T22:08:42.235542",
     "exception": false,
     "start_time": "2021-12-25T22:08:42.198362",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7a4cb719",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:42.314281Z",
     "iopub.status.busy": "2021-12-25T22:08:42.313463Z",
     "iopub.status.idle": "2021-12-25T22:08:42.318521Z",
     "shell.execute_reply": "2021-12-25T22:08:42.319046Z"
    },
    "papermill": {
     "duration": 0.046105,
     "end_time": "2021-12-25T22:08:42.319207",
     "exception": false,
     "start_time": "2021-12-25T22:08:42.273102",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StratifiedKFold(n_splits=5, random_state=1, shuffle=True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N_FOLD = 5\n",
    "cross_validation_design = StratifiedKFold(n_splits=N_FOLD,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a64d20c9",
   "metadata": {
    "papermill": {
     "duration": 0.037252,
     "end_time": "2021-12-25T22:08:42.394167",
     "exception": false,
     "start_time": "2021-12-25T22:08:42.356915",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cf55ea5d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:42.472936Z",
     "iopub.status.busy": "2021-12-25T22:08:42.472293Z",
     "iopub.status.idle": "2021-12-25T22:08:42.482766Z",
     "shell.execute_reply": "2021-12-25T22:08:42.483348Z"
    },
    "papermill": {
     "duration": 0.051558,
     "end_time": "2021-12-25T22:08:42.483523",
     "exception": false,
     "start_time": "2021-12-25T22:08:42.431965",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective_cat_pipe1(trial,X=X_pre,y=y):\n",
    "    # https://www.kaggle.com/hamidrezabakhtaki/xgboost-catboost-lighgbm-optuna-final-submission\n",
    "    # https://www.kaggle.com/prashant111/lightgbm-classifier-in-python\n",
    "    # https://www.kaggle.com/tunguz/tps-09-21-histgradientboosting-with-optuna\n",
    "    # https://github.com/solegalli/optuna-examples/tree/main/catboost\n",
    "    X_train,X_test, y_train,  y_test = train_test_split(X, y,stratify=y, \n",
    "                                                       test_size=0.25,\n",
    "                                                      random_state=42,\n",
    "                                                        shuffle=True)\n",
    "    param = {\n",
    "        \"loss_function\":\"Logloss\",\n",
    "        \"eval_metric\":\"AUC\",\n",
    "        #\"task_type\" : \"GPU\",\n",
    "        #\"min_data_in_leaf\":trial.suggest_int(\"min_data_in_leaf\" , 1,300),\n",
    "        #\"l2_leaf_reg\":trial.suggest_float(\"l2_leaf_reg\" , 0.0001 , 1.0 ,log =True),\n",
    "        # Use on cpu \n",
    "        #\"grow_policy\":trial.suggest_categorical(\"grow_policy\" ,['SymmetricTree', 'Depthwise', 'Lossguide']),\n",
    "        #\"leaf_estimation_method\":trial.suggest_categorical(\"leaf_estimation_method\" ,[\"Newton\" , \"Gradient\"]),\n",
    "\n",
    "        \"learning_rate\":trial.suggest_float(\"learning_rate\" ,0.005 ,0.2),\n",
    "        \"iterations\":trial.suggest_int(\"iterations\" ,1000, 15000),\n",
    "        \"objective\": trial.suggest_categorical(\"objective\", [\"Logloss\", \"CrossEntropy\"]),\n",
    "        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.01, 0.1),\n",
    "        \"depth\": trial.suggest_int(\"depth\", 1, 12),\n",
    "        \"boosting_type\": trial.suggest_categorical(\"boosting_type\", [\"Ordered\", \"Plain\"]),\n",
    "        \"bootstrap_type\": trial.suggest_categorical(\n",
    "        \"bootstrap_type\", [\"Bayesian\", \"Bernoulli\", \"MVS\"]\n",
    "        ),\n",
    "    }\n",
    "    if  param[\"bootstrap_type\"] == \"Bayesian\":\n",
    "        param[\"bagging_temperature\"] = trial.suggest_float(\"bagging_temperature\", 0, 10)\n",
    "    elif param[\"bootstrap_type\"] == \"Bernoulli\":\n",
    "         param[\"subsample\"] = trial.suggest_float(\"subsample\", 0.1, 1)\n",
    "    model= CatBoostClassifier(**param,\n",
    "                              # use it  only on gpu\n",
    "                            # rsm=np.nan\n",
    "                              #cat_features=cat_features\n",
    "                             )\n",
    "    model.fit( X_train, y_train,\n",
    "              eval_set=[(X_test, y_test)],\n",
    "              early_stopping_rounds=100,\n",
    "              verbose=False)\n",
    "    preds = model.predict_proba(X_test)[:,1]\n",
    "    final_auc= roc_auc_score(y_true=y_test, y_score=preds)   \n",
    "    return final_auc               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "44497574",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T22:08:42.562972Z",
     "iopub.status.busy": "2021-12-25T22:08:42.562390Z",
     "iopub.status.idle": "2021-12-26T05:35:26.777276Z",
     "shell.execute_reply": "2021-12-26T05:35:26.776743Z"
    },
    "papermill": {
     "duration": 26804.255908,
     "end_time": "2021-12-26T05:35:26.777415",
     "exception": false,
     "start_time": "2021-12-25T22:08:42.521507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 22:08:42,563]\u001b[0m A new study created in memory with name: no-name-58216271-be96-4acf-9e03-27e5c09a0f4f\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:09:17,192]\u001b[0m Trial 0 finished with value: 0.8654240510321664 and parameters: {'learning_rate': 0.18408518224212925, 'iterations': 9013, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.024158038801009633, 'depth': 10, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.5091831778649507}. Best is trial 0 with value: 0.8654240510321664.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:10:22,123]\u001b[0m Trial 1 finished with value: 0.8687228892641999 and parameters: {'learning_rate': 0.1159158548267501, 'iterations': 9737, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06534132209656504, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:11:06,208]\u001b[0m Trial 2 finished with value: 0.8682562604165046 and parameters: {'learning_rate': 0.02131717315187868, 'iterations': 8733, 'objective': 'Logloss', 'colsample_bylevel': 0.03004024037101575, 'depth': 6, 'boosting_type': 'Plain', 'bootstrap_type': 'Bernoulli', 'subsample': 0.540639232798019}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:12:34,473]\u001b[0m Trial 3 finished with value: 0.868166734180996 and parameters: {'learning_rate': 0.029317819786365665, 'iterations': 14860, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.053020116482643674, 'depth': 7, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:13:26,529]\u001b[0m Trial 4 finished with value: 0.8672680622800644 and parameters: {'learning_rate': 0.11356671604685735, 'iterations': 6802, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.019438120811486458, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.6744685789420748}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:16:55,801]\u001b[0m Trial 5 finished with value: 0.8681049974545858 and parameters: {'learning_rate': 0.017130324984380375, 'iterations': 9953, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.01880348656348333, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.8232259024357655}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:17:25,205]\u001b[0m Trial 6 finished with value: 0.8665766599778751 and parameters: {'learning_rate': 0.1351996345596711, 'iterations': 11638, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09201879689369549, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:17:45,766]\u001b[0m Trial 7 finished with value: 0.8675201577139595 and parameters: {'learning_rate': 0.058672625895474464, 'iterations': 10189, 'objective': 'Logloss', 'colsample_bylevel': 0.02061774821113266, 'depth': 9, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:25:52,383]\u001b[0m Trial 8 finished with value: 0.8676785191450674 and parameters: {'learning_rate': 0.010445718375872133, 'iterations': 14205, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06449041242792933, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.4707414356146511}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:26:05,570]\u001b[0m Trial 9 finished with value: 0.8652182353367521 and parameters: {'learning_rate': 0.14736552799568375, 'iterations': 8799, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0672095478133527, 'depth': 11, 'boosting_type': 'Plain', 'bootstrap_type': 'Bernoulli', 'subsample': 0.5000508660171161}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:28:11,375]\u001b[0m Trial 10 finished with value: 0.8638173738231993 and parameters: {'learning_rate': 0.08017879258815448, 'iterations': 2948, 'objective': 'Logloss', 'colsample_bylevel': 0.09750673921989744, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 6.500491490626909}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:28:40,487]\u001b[0m Trial 11 finished with value: 0.8683828868439926 and parameters: {'learning_rate': 0.0797062929822725, 'iterations': 5653, 'objective': 'Logloss', 'colsample_bylevel': 0.04431497402718053, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.12855735959732595}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:29:01,344]\u001b[0m Trial 12 finished with value: 0.8685319718017104 and parameters: {'learning_rate': 0.08901565604191083, 'iterations': 5001, 'objective': 'Logloss', 'colsample_bylevel': 0.04507017550393903, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.4544784456999067}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:29:17,332]\u001b[0m Trial 13 finished with value: 0.8679325816246415 and parameters: {'learning_rate': 0.11526489230665267, 'iterations': 3608, 'objective': 'Logloss', 'colsample_bylevel': 0.07634661670449991, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.29510349855358853}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:29:30,167]\u001b[0m Trial 14 finished with value: 0.8667038166524591 and parameters: {'learning_rate': 0.16550156195125007, 'iterations': 1252, 'objective': 'Logloss', 'colsample_bylevel': 0.045620107234904984, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:30:28,621]\u001b[0m Trial 15 finished with value: 0.8651246666798336 and parameters: {'learning_rate': 0.0823669897057854, 'iterations': 5332, 'objective': 'Logloss', 'colsample_bylevel': 0.03727875377700883, 'depth': 1, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 9.659081094974589}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:31:23,230]\u001b[0m Trial 16 finished with value: 0.8681233679507526 and parameters: {'learning_rate': 0.05516334453840623, 'iterations': 12111, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0803208352089185, 'depth': 8, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:31:43,117]\u001b[0m Trial 17 finished with value: 0.8678833712731784 and parameters: {'learning_rate': 0.13671198056311235, 'iterations': 7112, 'objective': 'Logloss', 'colsample_bylevel': 0.05861776413763771, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 3.3404077196373105}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:32:11,637]\u001b[0m Trial 18 finished with value: 0.8658442461986758 and parameters: {'learning_rate': 0.09814131907035772, 'iterations': 3962, 'objective': 'Logloss', 'colsample_bylevel': 0.074834924272057, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 3.3280639081035734}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:32:34,711]\u001b[0m Trial 19 finished with value: 0.8671995862841109 and parameters: {'learning_rate': 0.19679786324130805, 'iterations': 12076, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04691405541377716, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:35:15,001]\u001b[0m Trial 20 finished with value: 0.8677551826123107 and parameters: {'learning_rate': 0.055544463475758206, 'iterations': 1030, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03560136398418443, 'depth': 12, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:35:37,673]\u001b[0m Trial 21 finished with value: 0.8682438993874267 and parameters: {'learning_rate': 0.08854151768768789, 'iterations': 5353, 'objective': 'Logloss', 'colsample_bylevel': 0.04511296210605498, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.016530924870599734}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:36:03,194]\u001b[0m Trial 22 finished with value: 0.8675381348009904 and parameters: {'learning_rate': 0.11342905422175983, 'iterations': 5885, 'objective': 'Logloss', 'colsample_bylevel': 0.054608677313902024, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 1.6541832235173008}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:36:38,616]\u001b[0m Trial 23 finished with value: 0.8680990906159675 and parameters: {'learning_rate': 0.07074766412106966, 'iterations': 7395, 'objective': 'Logloss', 'colsample_bylevel': 0.06471895731485744, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 2.0307769308786248}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:36:49,524]\u001b[0m Trial 24 finished with value: 0.8569945704065742 and parameters: {'learning_rate': 0.10488634011115475, 'iterations': 4566, 'objective': 'Logloss', 'colsample_bylevel': 0.03830861586384192, 'depth': 1, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 5.975559678865535}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:37:22,110]\u001b[0m Trial 25 finished with value: 0.868057925196252 and parameters: {'learning_rate': 0.036431024806846526, 'iterations': 2526, 'objective': 'Logloss', 'colsample_bylevel': 0.0498967685532712, 'depth': 5, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 1.7279880749420795}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:37:40,926]\u001b[0m Trial 26 finished with value: 0.8679689805220742 and parameters: {'learning_rate': 0.12494849040056442, 'iterations': 6040, 'objective': 'Logloss', 'colsample_bylevel': 0.05970069873707622, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.008127164846691376}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:37:53,950]\u001b[0m Trial 27 finished with value: 0.8672001450391154 and parameters: {'learning_rate': 0.15796000127000154, 'iterations': 10702, 'objective': 'Logloss', 'colsample_bylevel': 0.031447991925838366, 'depth': 5, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:40:25,751]\u001b[0m Trial 28 finished with value: 0.8660592357390102 and parameters: {'learning_rate': 0.04323292017531397, 'iterations': 7749, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0712784595505292, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 4.236641852680137}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:41:13,372]\u001b[0m Trial 29 finished with value: 0.8684315213357063 and parameters: {'learning_rate': 0.09540751547014495, 'iterations': 6429, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08394636825729232, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:42:05,191]\u001b[0m Trial 30 finished with value: 0.8682654342614246 and parameters: {'learning_rate': 0.09412771213650417, 'iterations': 8223, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08531584343631461, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:43:08,037]\u001b[0m Trial 31 finished with value: 0.8680846713159037 and parameters: {'learning_rate': 0.06886432245340353, 'iterations': 6349, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0880123924670378, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:44:17,237]\u001b[0m Trial 32 finished with value: 0.8679806858692592 and parameters: {'learning_rate': 0.10235314130807423, 'iterations': 4621, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04101817226755847, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:45:04,974]\u001b[0m Trial 33 finished with value: 0.8681261617257747 and parameters: {'learning_rate': 0.07155236861545544, 'iterations': 8263, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.024938443251872743, 'depth': 7, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:45:45,194]\u001b[0m Trial 34 finished with value: 0.8679166970180866 and parameters: {'learning_rate': 0.12058930925270231, 'iterations': 5194, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.05812142416406188, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:47:48,470]\u001b[0m Trial 35 finished with value: 0.8663891520612289 and parameters: {'learning_rate': 0.08507421071429151, 'iterations': 9336, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02817262954221938, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.16420882109166762}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:48:04,167]\u001b[0m Trial 36 finished with value: 0.866773450069493 and parameters: {'learning_rate': 0.1310703159052045, 'iterations': 6909, 'objective': 'Logloss', 'colsample_bylevel': 0.08228873438604875, 'depth': 6, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 8.665050294621611}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:48:40,011]\u001b[0m Trial 37 finished with value: 0.8678338131252326 and parameters: {'learning_rate': 0.10604490786626962, 'iterations': 11016, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09833094344884485, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:49:52,432]\u001b[0m Trial 38 finished with value: 0.8677015649382102 and parameters: {'learning_rate': 0.14465743727181782, 'iterations': 13446, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.012340165376699297, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.9633809590618185}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:50:11,394]\u001b[0m Trial 39 finished with value: 0.8676767630579105 and parameters: {'learning_rate': 0.09280516448216755, 'iterations': 9271, 'objective': 'Logloss', 'colsample_bylevel': 0.049822419717535205, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:52:06,404]\u001b[0m Trial 40 finished with value: 0.8682030247483766 and parameters: {'learning_rate': 0.06385211386410758, 'iterations': 2718, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0709815675488109, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.16024488070628412}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:52:57,701]\u001b[0m Trial 41 finished with value: 0.8680481469836739 and parameters: {'learning_rate': 0.09322281501147464, 'iterations': 8335, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0889066084696973, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:53:46,994]\u001b[0m Trial 42 finished with value: 0.8681804065738599 and parameters: {'learning_rate': 0.07866489236761948, 'iterations': 6508, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08140807757864606, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:54:25,819]\u001b[0m Trial 43 finished with value: 0.8667984914162232 and parameters: {'learning_rate': 0.11073884906274735, 'iterations': 7980, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09107053222946901, 'depth': 7, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:55:20,657]\u001b[0m Trial 44 finished with value: 0.8680835252979864 and parameters: {'learning_rate': 0.09616208908864007, 'iterations': 9962, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08453580939917127, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:56:12,117]\u001b[0m Trial 45 finished with value: 0.8686112865044338 and parameters: {'learning_rate': 0.07783294698141768, 'iterations': 8823, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07645471496390228, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:56:31,832]\u001b[0m Trial 46 finished with value: 0.8668171184835655 and parameters: {'learning_rate': 0.04882077825094894, 'iterations': 8902, 'objective': 'Logloss', 'colsample_bylevel': 0.06859147379624297, 'depth': 8, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 2.3463936835762436}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:57:12,247]\u001b[0m Trial 47 finished with value: 0.8681274445816525 and parameters: {'learning_rate': 0.07791863999690864, 'iterations': 3721, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0631689382272666, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:57:35,198]\u001b[0m Trial 48 finished with value: 0.8670973968356952 and parameters: {'learning_rate': 0.02837924906636379, 'iterations': 5696, 'objective': 'Logloss', 'colsample_bylevel': 0.07618286379898626, 'depth': 9, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.9930551081309773}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:59:34,823]\u001b[0m Trial 49 finished with value: 0.8678817349192367 and parameters: {'learning_rate': 0.06288229286277716, 'iterations': 4461, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09495440835477532, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.2997073793982839}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 22:59:50,133]\u001b[0m Trial 50 finished with value: 0.8679525485636779 and parameters: {'learning_rate': 0.12359845463590038, 'iterations': 7417, 'objective': 'Logloss', 'colsample_bylevel': 0.0738998534892848, 'depth': 5, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:00:48,987]\u001b[0m Trial 51 finished with value: 0.8684220966211925 and parameters: {'learning_rate': 0.08755044169570542, 'iterations': 8680, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07842373024591871, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:01:44,910]\u001b[0m Trial 52 finished with value: 0.868356756495162 and parameters: {'learning_rate': 0.08620114677933118, 'iterations': 9487, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07984464060467002, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:02:28,947]\u001b[0m Trial 53 finished with value: 0.8683827100949605 and parameters: {'learning_rate': 0.0735599035638548, 'iterations': 10685, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07737754615822323, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:03:13,194]\u001b[0m Trial 54 finished with value: 0.8678756342268411 and parameters: {'learning_rate': 0.10086764379158317, 'iterations': 8725, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04202854613541583, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:05:23,344]\u001b[0m Trial 55 finished with value: 0.8647230586695946 and parameters: {'learning_rate': 0.1123539763990106, 'iterations': 11382, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.062007568706341734, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 5.0557356431435085}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:06:06,471]\u001b[0m Trial 56 finished with value: 0.868293354906902 and parameters: {'learning_rate': 0.0872281674268875, 'iterations': 10159, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.050272562267934875, 'depth': 7, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:06:58,682]\u001b[0m Trial 57 finished with value: 0.8672955439037525 and parameters: {'learning_rate': 0.05124197795433015, 'iterations': 6739, 'objective': 'Logloss', 'colsample_bylevel': 0.05503568852070647, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 7.354763391225898}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:07:21,871]\u001b[0m Trial 58 finished with value: 0.8678550971296367 and parameters: {'learning_rate': 0.11852795896470349, 'iterations': 4957, 'objective': 'Logloss', 'colsample_bylevel': 0.06620418136864516, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:09:26,719]\u001b[0m Trial 59 finished with value: 0.8677131391490167 and parameters: {'learning_rate': 0.06376261369197021, 'iterations': 1757, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07305035186011527, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 3.144313256076008}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:09:45,091]\u001b[0m Trial 60 finished with value: 0.8677817462813487 and parameters: {'learning_rate': 0.07937802869430002, 'iterations': 7479, 'objective': 'Logloss', 'colsample_bylevel': 0.034199095788352486, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:10:47,950]\u001b[0m Trial 61 finished with value: 0.8682171589693566 and parameters: {'learning_rate': 0.07462872235233343, 'iterations': 10641, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0792368738402912, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:11:34,477]\u001b[0m Trial 62 finished with value: 0.8683030190878464 and parameters: {'learning_rate': 0.10669232678837397, 'iterations': 9654, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07700369193221984, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:12:30,151]\u001b[0m Trial 63 finished with value: 0.8681723274326221 and parameters: {'learning_rate': 0.08900705497732118, 'iterations': 12553, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06847172258429088, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:13:11,905]\u001b[0m Trial 64 finished with value: 0.8675654168693199 and parameters: {'learning_rate': 0.0677054958626609, 'iterations': 10453, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0847966175123954, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:13:46,898]\u001b[0m Trial 65 finished with value: 0.8677973401072381 and parameters: {'learning_rate': 0.0985791757404496, 'iterations': 8843, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04395586140373414, 'depth': 8, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:15:23,469]\u001b[0m Trial 66 finished with value: 0.8678216060388599 and parameters: {'learning_rate': 0.08400751457262016, 'iterations': 6067, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07832569156528152, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.979083080579442}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:16:32,827]\u001b[0m Trial 67 finished with value: 0.8683677320398924 and parameters: {'learning_rate': 0.07374943173038352, 'iterations': 4153, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09418546942097357, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:17:16,293]\u001b[0m Trial 68 finished with value: 0.8665209498232989 and parameters: {'learning_rate': 0.058444722745716826, 'iterations': 11762, 'objective': 'Logloss', 'colsample_bylevel': 0.08744197038461805, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 4.6336249238008715}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:22:49,358]\u001b[0m Trial 69 finished with value: 0.8681690832326473 and parameters: {'learning_rate': 0.007650824905251416, 'iterations': 5650, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07181630086700504, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.7455807935509211}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:23:05,290]\u001b[0m Trial 70 finished with value: 0.867268831993591 and parameters: {'learning_rate': 0.13210140815829535, 'iterations': 10990, 'objective': 'Logloss', 'colsample_bylevel': 0.05669455064948216, 'depth': 5, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:24:22,416]\u001b[0m Trial 71 finished with value: 0.8670553875818913 and parameters: {'learning_rate': 0.09249625929953181, 'iterations': 4273, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0945893359042888, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:25:42,128]\u001b[0m Trial 72 finished with value: 0.868191650092929 and parameters: {'learning_rate': 0.0706257330164021, 'iterations': 4911, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09365024903732808, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:26:56,771]\u001b[0m Trial 73 finished with value: 0.8685031617094912 and parameters: {'learning_rate': 0.07552696687817083, 'iterations': 3520, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08229553731275953, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 1 with value: 0.8687228892641999.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:28:22,387]\u001b[0m Trial 74 finished with value: 0.8687371774278851 and parameters: {'learning_rate': 0.07837018035940185, 'iterations': 3263, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0823703485388484, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 74 with value: 0.8687371774278851.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:29:28,505]\u001b[0m Trial 75 finished with value: 0.8682518930049391 and parameters: {'learning_rate': 0.07996050542083878, 'iterations': 3220, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08147331480081321, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 74 with value: 0.8687371774278851.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:30:46,260]\u001b[0m Trial 76 finished with value: 0.8690178605923141 and parameters: {'learning_rate': 0.08926084391040985, 'iterations': 2073, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08311513863516848, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:32:39,474]\u001b[0m Trial 77 finished with value: 0.8667419260243954 and parameters: {'learning_rate': 0.10592184116062929, 'iterations': 1807, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09042190448346649, 'depth': 11, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:33:46,818]\u001b[0m Trial 78 finished with value: 0.8681726182132876 and parameters: {'learning_rate': 0.09050609470457997, 'iterations': 2109, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08305547696169925, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:34:58,645]\u001b[0m Trial 79 finished with value: 0.8682629483718128 and parameters: {'learning_rate': 0.09594515563146097, 'iterations': 3428, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08598595208774881, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:36:43,088]\u001b[0m Trial 80 finished with value: 0.8672972372735109 and parameters: {'learning_rate': 0.08285592597447897, 'iterations': 2409, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0754924417988068, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:38:43,353]\u001b[0m Trial 81 finished with value: 0.8680218569905566 and parameters: {'learning_rate': 0.06597570845157694, 'iterations': 3193, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08288229397923844, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 0.9430854681164872}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:39:19,582]\u001b[0m Trial 82 finished with value: 0.8674034919496632 and parameters: {'learning_rate': 0.1758516660910787, 'iterations': 2877, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08734831309747368, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:39:36,252]\u001b[0m Trial 83 finished with value: 0.8681200553317976 and parameters: {'learning_rate': 0.10023974367090557, 'iterations': 3747, 'objective': 'Logloss', 'colsample_bylevel': 0.0698965817140643, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:40:37,850]\u001b[0m Trial 84 finished with value: 0.8680389104213557 and parameters: {'learning_rate': 0.07666054894241857, 'iterations': 1097, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0473010433646454, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:41:55,363]\u001b[0m Trial 85 finished with value: 0.8682274560258673 and parameters: {'learning_rate': 0.10912180265916659, 'iterations': 8434, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08974609899552659, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 2.8427332287167504}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:42:19,805]\u001b[0m Trial 86 finished with value: 0.8673371654500024 and parameters: {'learning_rate': 0.08396392766506651, 'iterations': 1481, 'objective': 'Logloss', 'colsample_bylevel': 0.03844287922932832, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bernoulli', 'subsample': 0.337952616796677}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:43:14,929]\u001b[0m Trial 87 finished with value: 0.8677894092071239 and parameters: {'learning_rate': 0.11598923132434402, 'iterations': 7873, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07863158900295973, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:44:26,709]\u001b[0m Trial 88 finished with value: 0.868286148107661 and parameters: {'learning_rate': 0.0902254841754601, 'iterations': 2324, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06090571192404565, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:44:54,336]\u001b[0m Trial 89 finished with value: 0.8680341724069809 and parameters: {'learning_rate': 0.1019834978267329, 'iterations': 6316, 'objective': 'Logloss', 'colsample_bylevel': 0.07448430923751896, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 4.036196531375203}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:45:50,386]\u001b[0m Trial 90 finished with value: 0.8678335736588022 and parameters: {'learning_rate': 0.05708736634012863, 'iterations': 7089, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08109144365718009, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:46:28,113]\u001b[0m Trial 91 finished with value: 0.8679184017910082 and parameters: {'learning_rate': 0.0820057828307171, 'iterations': 9649, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.05272638609625428, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:47:34,162]\u001b[0m Trial 92 finished with value: 0.8679370402615139 and parameters: {'learning_rate': 0.07239081094889083, 'iterations': 5402, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.083636100195101, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:48:38,503]\u001b[0m Trial 93 finished with value: 0.8682503079652327 and parameters: {'learning_rate': 0.06198405934193908, 'iterations': 9135, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07675973925504835, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:49:30,725]\u001b[0m Trial 94 finished with value: 0.8685030533794391 and parameters: {'learning_rate': 0.07649354768038849, 'iterations': 9917, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06528380590088353, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:50:27,738]\u001b[0m Trial 95 finished with value: 0.868202984837305 and parameters: {'learning_rate': 0.08757832631609265, 'iterations': 9862, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06426569267760487, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:51:06,545]\u001b[0m Trial 96 finished with value: 0.8684341725711867 and parameters: {'learning_rate': 0.0976890348248023, 'iterations': 2716, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06655511023323629, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:51:56,267]\u001b[0m Trial 97 finished with value: 0.8681941701920305 and parameters: {'learning_rate': 0.09682752234653502, 'iterations': 2678, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06648484929786228, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:52:51,500]\u001b[0m Trial 98 finished with value: 0.8674956010016721 and parameters: {'learning_rate': 0.07719948073586173, 'iterations': 8501, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07236851608730739, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:53:29,123]\u001b[0m Trial 99 finished with value: 0.8675830974741034 and parameters: {'learning_rate': 0.09304090014378272, 'iterations': 3929, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06884557378978097, 'depth': 9, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:54:21,913]\u001b[0m Trial 100 finished with value: 0.8680275471690713 and parameters: {'learning_rate': 0.10207942632010397, 'iterations': 1993, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07434985126072881, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:55:39,079]\u001b[0m Trial 101 finished with value: 0.8682922602032198 and parameters: {'learning_rate': 0.08576817444925318, 'iterations': 3003, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.041168364809792184, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:56:15,031]\u001b[0m Trial 102 finished with value: 0.8681403130514997 and parameters: {'learning_rate': 0.06939445518840232, 'iterations': 4717, 'objective': 'Logloss', 'colsample_bylevel': 0.08624727010345593, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 5.488532094714669}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:57:18,932]\u001b[0m Trial 103 finished with value: 0.8678194223330772 and parameters: {'learning_rate': 0.08011680811369837, 'iterations': 9097, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.059626240228936735, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:58:14,103]\u001b[0m Trial 104 finished with value: 0.8673528048885453 and parameters: {'learning_rate': 0.11044614494838713, 'iterations': 3289, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.05238088089451344, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.9811151711702626}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:59:07,759]\u001b[0m Trial 105 finished with value: 0.8680428160048049 and parameters: {'learning_rate': 0.12514583264764703, 'iterations': 10150, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04727238854378273, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-25 23:59:32,127]\u001b[0m Trial 106 finished with value: 0.867841367720956 and parameters: {'learning_rate': 0.09674032207639205, 'iterations': 5085, 'objective': 'Logloss', 'colsample_bylevel': 0.05673612866442593, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 7.452107694708674}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:00:08,220]\u001b[0m Trial 107 finished with value: 0.8671069469850058 and parameters: {'learning_rate': 0.07509520603771432, 'iterations': 1527, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07897544675841521, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:00:50,409]\u001b[0m Trial 108 finished with value: 0.8676865298673251 and parameters: {'learning_rate': 0.08679815843747236, 'iterations': 4320, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.06307649188820286, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:02:01,372]\u001b[0m Trial 109 finished with value: 0.8685204717114657 and parameters: {'learning_rate': 0.09308029389675557, 'iterations': 8128, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0800024053213698, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:03:21,691]\u001b[0m Trial 110 finished with value: 0.8683333515023736 and parameters: {'learning_rate': 0.1061490504771096, 'iterations': 8198, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08061842209677889, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:04:34,648]\u001b[0m Trial 111 finished with value: 0.8685296911690391 and parameters: {'learning_rate': 0.09089145730088526, 'iterations': 8597, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09209275626425752, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:05:44,635]\u001b[0m Trial 112 finished with value: 0.8684655198672525 and parameters: {'learning_rate': 0.09216279292412262, 'iterations': 7656, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09998075373446556, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:06:34,546]\u001b[0m Trial 113 finished with value: 0.8679950424519247 and parameters: {'learning_rate': 0.09108872496916859, 'iterations': 7369, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0959165629934919, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:07:17,003]\u001b[0m Trial 114 finished with value: 0.8676817177323889 and parameters: {'learning_rate': 0.09584043326815336, 'iterations': 8022, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09243331570263955, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:07:50,927]\u001b[0m Trial 115 finished with value: 0.867828225575188 and parameters: {'learning_rate': 0.11501577422123303, 'iterations': 6588, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0995704038258419, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:08:54,461]\u001b[0m Trial 116 finished with value: 0.8689586496665874 and parameters: {'learning_rate': 0.08274847490155929, 'iterations': 9406, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09710326490540848, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:10:00,798]\u001b[0m Trial 117 finished with value: 0.8684948887144762 and parameters: {'learning_rate': 0.08033115745033087, 'iterations': 9505, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09704924561787277, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:10:36,355]\u001b[0m Trial 118 finished with value: 0.8680733821841812 and parameters: {'learning_rate': 0.08229073832840507, 'iterations': 9423, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09833357711130192, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:11:18,208]\u001b[0m Trial 119 finished with value: 0.8671935083980421 and parameters: {'learning_rate': 0.076493281975849, 'iterations': 9866, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09689199977920385, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:12:30,914]\u001b[0m Trial 120 finished with value: 0.8686222221380923 and parameters: {'learning_rate': 0.06624698217499463, 'iterations': 10400, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09980984981369748, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:14:02,543]\u001b[0m Trial 121 finished with value: 0.8685652633371284 and parameters: {'learning_rate': 0.06748593073373739, 'iterations': 10474, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09230723155325767, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:15:21,578]\u001b[0m Trial 122 finished with value: 0.8677793117059721 and parameters: {'learning_rate': 0.06106239219368069, 'iterations': 10371, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09198167069275609, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:16:42,615]\u001b[0m Trial 123 finished with value: 0.8681891185906639 and parameters: {'learning_rate': 0.06637467394610791, 'iterations': 11269, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09747466920812106, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:18:04,580]\u001b[0m Trial 124 finished with value: 0.8684637637800958 and parameters: {'learning_rate': 0.05323803105999425, 'iterations': 10955, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0952102251177282, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:19:23,212]\u001b[0m Trial 125 finished with value: 0.8684745853821206 and parameters: {'learning_rate': 0.07020805285982386, 'iterations': 8871, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08861923657697869, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:21:15,862]\u001b[0m Trial 126 finished with value: 0.8685844434578933 and parameters: {'learning_rate': 0.07354170595856492, 'iterations': 9505, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09236475047024165, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:22:50,773]\u001b[0m Trial 127 finished with value: 0.8682449997926904 and parameters: {'learning_rate': 0.06616166220746678, 'iterations': 10147, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.092808531297367, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:26:28,837]\u001b[0m Trial 128 finished with value: 0.8677597153697446 and parameters: {'learning_rate': 0.059394421444445226, 'iterations': 10456, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0903494963213813, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:27:56,566]\u001b[0m Trial 129 finished with value: 0.8684770484654057 and parameters: {'learning_rate': 0.07288376081506885, 'iterations': 9191, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08669657343678076, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:30:07,395]\u001b[0m Trial 130 finished with value: 0.8677060235750823 and parameters: {'learning_rate': 0.06905664479856351, 'iterations': 8606, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08453006954898946, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.6643424377957188}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:31:29,814]\u001b[0m Trial 131 finished with value: 0.8683219084279457 and parameters: {'learning_rate': 0.0453920536302848, 'iterations': 9848, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09664509671233323, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:32:43,085]\u001b[0m Trial 132 finished with value: 0.8681911027410878 and parameters: {'learning_rate': 0.08119692897020181, 'iterations': 9412, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09131128875977591, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:34:12,723]\u001b[0m Trial 133 finished with value: 0.8684892042375433 and parameters: {'learning_rate': 0.07945304621798298, 'iterations': 8961, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09404947157135729, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:35:40,159]\u001b[0m Trial 134 finished with value: 0.8685468985425434 and parameters: {'learning_rate': 0.07607925281709503, 'iterations': 9626, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08975426801692324, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:39:09,970]\u001b[0m Trial 135 finished with value: 0.8677011259164209 and parameters: {'learning_rate': 0.07303274848937105, 'iterations': 9706, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0822673545179003, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:40:33,757]\u001b[0m Trial 136 finished with value: 0.868417484041615 and parameters: {'learning_rate': 0.08469270769776466, 'iterations': 10649, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0901512194545057, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:42:01,127]\u001b[0m Trial 137 finished with value: 0.8684770484654055 and parameters: {'learning_rate': 0.07688136953876448, 'iterations': 10193, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08844368407809333, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:42:44,516]\u001b[0m Trial 138 finished with value: 0.8677166969359837 and parameters: {'learning_rate': 0.06541558608634616, 'iterations': 9263, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0857003254350287, 'depth': 7, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:43:45,255]\u001b[0m Trial 139 finished with value: 0.8682266749091773 and parameters: {'learning_rate': 0.08848695795399045, 'iterations': 8171, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09196170183144514, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:46:22,059]\u001b[0m Trial 140 finished with value: 0.8674152429095017 and parameters: {'learning_rate': 0.05643303464905983, 'iterations': 8602, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09366489532258769, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:47:36,913]\u001b[0m Trial 141 finished with value: 0.8682278893460746 and parameters: {'learning_rate': 0.07843232020575, 'iterations': 9515, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0988617732992423, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:48:48,742]\u001b[0m Trial 142 finished with value: 0.8683062974973113 and parameters: {'learning_rate': 0.08276583807337444, 'iterations': 10042, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09769529529627043, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:49:41,655]\u001b[0m Trial 143 finished with value: 0.8679008523226033 and parameters: {'learning_rate': 0.0742069769191502, 'iterations': 9037, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09647117083927306, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:51:11,307]\u001b[0m Trial 144 finished with value: 0.8685537917547922 and parameters: {'learning_rate': 0.06834355870817331, 'iterations': 9619, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09562529676660635, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:52:38,352]\u001b[0m Trial 145 finished with value: 0.8682862336313862 and parameters: {'learning_rate': 0.06225574850741919, 'iterations': 10773, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09004836387940905, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:53:38,197]\u001b[0m Trial 146 finished with value: 0.8680620360366417 and parameters: {'learning_rate': 0.06981652541661176, 'iterations': 10361, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08756581003845106, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:56:16,771]\u001b[0m Trial 147 finished with value: 0.8681416586247758 and parameters: {'learning_rate': 0.017231168525468188, 'iterations': 14765, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09510467722223484, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:58:20,924]\u001b[0m Trial 148 finished with value: 0.868474950283348 and parameters: {'learning_rate': 0.07502982058394149, 'iterations': 9882, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08436144826090154, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 00:59:20,681]\u001b[0m Trial 149 finished with value: 0.8679325873262232 and parameters: {'learning_rate': 0.0856384717242232, 'iterations': 11323, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09335381643662122, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:00:24,216]\u001b[0m Trial 150 finished with value: 0.8685618309849583 and parameters: {'learning_rate': 0.06735309825294729, 'iterations': 8811, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08028061239370113, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:01:38,747]\u001b[0m Trial 151 finished with value: 0.8683901563606322 and parameters: {'learning_rate': 0.06567274520746388, 'iterations': 9657, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08062706915561854, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:02:33,397]\u001b[0m Trial 152 finished with value: 0.8678689405699511 and parameters: {'learning_rate': 0.07084295864514957, 'iterations': 8709, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08210302440045505, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:03:31,235]\u001b[0m Trial 153 finished with value: 0.8682283511741904 and parameters: {'learning_rate': 0.1412979173930774, 'iterations': 9208, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0774303665423276, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:04:15,040]\u001b[0m Trial 154 finished with value: 0.8678763298198059 and parameters: {'learning_rate': 0.08981867396487625, 'iterations': 8310, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0798928197109257, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:05:22,130]\u001b[0m Trial 155 finished with value: 0.8680807771356175 and parameters: {'learning_rate': 0.06113330871735609, 'iterations': 8835, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08862336627285915, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:06:24,421]\u001b[0m Trial 156 finished with value: 0.867416371822674 and parameters: {'learning_rate': 0.07731005220567365, 'iterations': 7821, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07607486060084707, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.3669658439662432}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:08:00,451]\u001b[0m Trial 157 finished with value: 0.8683256201576182 and parameters: {'learning_rate': 0.06759141996748406, 'iterations': 9286, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.014334436332418245, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:13:34,973]\u001b[0m Trial 158 finished with value: 0.8671426787973822 and parameters: {'learning_rate': 0.05180836611222366, 'iterations': 8461, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08310696361616914, 'depth': 12, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:14:04,281]\u001b[0m Trial 159 finished with value: 0.8676927160834459 and parameters: {'learning_rate': 0.07271846874704344, 'iterations': 9701, 'objective': 'Logloss', 'colsample_bylevel': 0.08496385194988769, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:15:25,290]\u001b[0m Trial 160 finished with value: 0.8668971116745086 and parameters: {'learning_rate': 0.0834366107194824, 'iterations': 3525, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09974052613193865, 'depth': 10, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 6.48581094739695}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:16:37,144]\u001b[0m Trial 161 finished with value: 0.8688829839761381 and parameters: {'learning_rate': 0.08095721683822364, 'iterations': 9428, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09603067440859182, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:17:40,969]\u001b[0m Trial 162 finished with value: 0.8681702463553095 and parameters: {'learning_rate': 0.07876060254702655, 'iterations': 10321, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09528558012172876, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:18:12,428]\u001b[0m Trial 163 finished with value: 0.8675201805202862 and parameters: {'learning_rate': 0.1923964277705415, 'iterations': 9938, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09366994831290472, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:18:47,741]\u001b[0m Trial 164 finished with value: 0.8679182706546297 and parameters: {'learning_rate': 0.15504376738091077, 'iterations': 9445, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09561461587556072, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:19:44,771]\u001b[0m Trial 165 finished with value: 0.8685943699115949 and parameters: {'learning_rate': 0.09448184064846242, 'iterations': 8923, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09140224477566873, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:20:39,072]\u001b[0m Trial 166 finished with value: 0.8683541223644268 and parameters: {'learning_rate': 0.09443830974045216, 'iterations': 9098, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09123673805269317, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:21:46,288]\u001b[0m Trial 167 finished with value: 0.8680530275375905 and parameters: {'learning_rate': 0.08730012337572372, 'iterations': 8743, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09199895226032691, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:22:55,183]\u001b[0m Trial 168 finished with value: 0.8678562032364823 and parameters: {'learning_rate': 0.09979296618252326, 'iterations': 8939, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09777921560959207, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:23:42,926]\u001b[0m Trial 169 finished with value: 0.8684042449689585 and parameters: {'learning_rate': 0.09332661972937478, 'iterations': 8412, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08596730136800838, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:24:34,079]\u001b[0m Trial 170 finished with value: 0.867922563945633 and parameters: {'learning_rate': 0.0888365663600565, 'iterations': 4005, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08875566934074944, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:25:32,623]\u001b[0m Trial 171 finished with value: 0.8682720366930078 and parameters: {'learning_rate': 0.08253151418284177, 'iterations': 9652, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07943822361501042, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:26:22,987]\u001b[0m Trial 172 finished with value: 0.8676521151203164 and parameters: {'learning_rate': 0.06893998251309218, 'iterations': 10077, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09313290343519123, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:27:28,644]\u001b[0m Trial 173 finished with value: 0.867895350296284 and parameters: {'learning_rate': 0.07628589389770978, 'iterations': 2261, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09514231314542305, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:28:20,506]\u001b[0m Trial 174 finished with value: 0.8676935257080441 and parameters: {'learning_rate': 0.06419632577184577, 'iterations': 10545, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09068981032054689, 'depth': 8, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:29:11,316]\u001b[0m Trial 175 finished with value: 0.8681438537337217 and parameters: {'learning_rate': 0.08021098672704265, 'iterations': 9334, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07343216507420058, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:29:33,175]\u001b[0m Trial 176 finished with value: 0.8681489737540689 and parameters: {'learning_rate': 0.07375220227154786, 'iterations': 10838, 'objective': 'Logloss', 'colsample_bylevel': 0.09824232863129692, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:31:28,550]\u001b[0m Trial 177 finished with value: 0.8688323368260918 and parameters: {'learning_rate': 0.08530792302715572, 'iterations': 8127, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08707884609958927, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:32:51,757]\u001b[0m Trial 178 finished with value: 0.862224226467554 and parameters: {'learning_rate': 0.10393714665345224, 'iterations': 7562, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08355265439900338, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 9.937941806627833}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:34:13,543]\u001b[0m Trial 179 finished with value: 0.868572567063258 and parameters: {'learning_rate': 0.08589160408311963, 'iterations': 7983, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08932851595255167, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:35:15,750]\u001b[0m Trial 180 finished with value: 0.8683066852048654 and parameters: {'learning_rate': 0.09218223404680026, 'iterations': 8096, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0868003288159094, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:36:33,827]\u001b[0m Trial 181 finished with value: 0.8683579937383862 and parameters: {'learning_rate': 0.08459387250223784, 'iterations': 8005, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08929797666837969, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:37:49,514]\u001b[0m Trial 182 finished with value: 0.8679769228253518 and parameters: {'learning_rate': 0.08873640830372086, 'iterations': 13109, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0875481519689683, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:38:43,274]\u001b[0m Trial 183 finished with value: 0.8678184302578653 and parameters: {'learning_rate': 0.08496865409629625, 'iterations': 8951, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09247279320557325, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:39:55,083]\u001b[0m Trial 184 finished with value: 0.8681464821628754 and parameters: {'learning_rate': 0.09574978007593939, 'iterations': 7263, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.031919396951350376, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:40:57,837]\u001b[0m Trial 185 finished with value: 0.8679788385567955 and parameters: {'learning_rate': 0.08042387506586592, 'iterations': 8611, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08107005146551209, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:41:58,489]\u001b[0m Trial 186 finished with value: 0.8679677603835951 and parameters: {'learning_rate': 0.07162520545571485, 'iterations': 9537, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09064363847067054, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:43:45,886]\u001b[0m Trial 187 finished with value: 0.8672042216700152 and parameters: {'learning_rate': 0.08791530239647652, 'iterations': 8350, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08234855482089781, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:44:36,364]\u001b[0m Trial 188 finished with value: 0.8678698072103661 and parameters: {'learning_rate': 0.09119249402865874, 'iterations': 7739, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0856362491719793, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:46:59,925]\u001b[0m Trial 189 finished with value: 0.8682601945078625 and parameters: {'learning_rate': 0.09822315879400197, 'iterations': 9100, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09423886496352743, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.11301710752305472}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:47:26,485]\u001b[0m Trial 190 finished with value: 0.8681644763546513 and parameters: {'learning_rate': 0.08345648430675802, 'iterations': 1494, 'objective': 'Logloss', 'colsample_bylevel': 0.08828964408068382, 'depth': 2, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:48:26,218]\u001b[0m Trial 191 finished with value: 0.868178490842416 and parameters: {'learning_rate': 0.07662147676758693, 'iterations': 9744, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07774143335787914, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:49:31,310]\u001b[0m Trial 192 finished with value: 0.868340763558555 and parameters: {'learning_rate': 0.06739421309572172, 'iterations': 10047, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.09643221856958234, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:50:54,066]\u001b[0m Trial 193 finished with value: 0.8684311393297339 and parameters: {'learning_rate': 0.0782495389001179, 'iterations': 9265, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08362309453375424, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:52:13,693]\u001b[0m Trial 194 finished with value: 0.8686530448886438 and parameters: {'learning_rate': 0.0742735367326748, 'iterations': 8671, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0544649735990822, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:53:26,990]\u001b[0m Trial 195 finished with value: 0.868508760662699 and parameters: {'learning_rate': 0.07371248257226032, 'iterations': 8790, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.043378462000478554, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:54:41,291]\u001b[0m Trial 196 finished with value: 0.8682813416743064 and parameters: {'learning_rate': 0.07057837694898379, 'iterations': 8736, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03662826219394923, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:56:29,557]\u001b[0m Trial 197 finished with value: 0.868410682054673 and parameters: {'learning_rate': 0.05731592958970172, 'iterations': 8376, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.045198005478542774, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:58:04,842]\u001b[0m Trial 198 finished with value: 0.866881745911886 and parameters: {'learning_rate': 0.06425574444610499, 'iterations': 8828, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.043050740516757474, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 7.339503339744185}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 01:59:12,646]\u001b[0m Trial 199 finished with value: 0.8681759821464776 and parameters: {'learning_rate': 0.08107035785781688, 'iterations': 7986, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.05445220309976727, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:00:42,391]\u001b[0m Trial 200 finished with value: 0.8687334828029579 and parameters: {'learning_rate': 0.07302627080161482, 'iterations': 8540, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04034695903392598, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:01:44,424]\u001b[0m Trial 201 finished with value: 0.8682671960501631 and parameters: {'learning_rate': 0.07441314949613333, 'iterations': 8672, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03797998837047331, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:02:53,967]\u001b[0m Trial 202 finished with value: 0.8687276443833195 and parameters: {'learning_rate': 0.06884486355385033, 'iterations': 9083, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03460024405953682, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:04:27,360]\u001b[0m Trial 203 finished with value: 0.868613795200372 and parameters: {'learning_rate': 0.06676115733888285, 'iterations': 9134, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02753720039515164, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:05:31,835]\u001b[0m Trial 204 finished with value: 0.8681378214603065 and parameters: {'learning_rate': 0.060837982796894154, 'iterations': 9062, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0292913190956602, 'depth': 9, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:07:14,319]\u001b[0m Trial 205 finished with value: 0.8687046556059935 and parameters: {'learning_rate': 0.06781005334933932, 'iterations': 9343, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.032631960313069495, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:08:31,781]\u001b[0m Trial 206 finished with value: 0.8681327755605215 and parameters: {'learning_rate': 0.06793574468162736, 'iterations': 9444, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.026361162989511407, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:11:16,025]\u001b[0m Trial 207 finished with value: 0.8684336138161822 and parameters: {'learning_rate': 0.06388410504451991, 'iterations': 9428, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03259665186833625, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:12:29,016]\u001b[0m Trial 208 finished with value: 0.8683872827634663 and parameters: {'learning_rate': 0.07050106951769945, 'iterations': 9271, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03446722095743964, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:14:49,868]\u001b[0m Trial 209 finished with value: 0.8686624867079026 and parameters: {'learning_rate': 0.05977006914758296, 'iterations': 9766, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.039695479780396996, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:17:07,876]\u001b[0m Trial 210 finished with value: 0.868102996199417 and parameters: {'learning_rate': 0.058233671980622306, 'iterations': 9760, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.022558269719880578, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:18:33,460]\u001b[0m Trial 211 finished with value: 0.8654273294416315 and parameters: {'learning_rate': 0.054301625447087366, 'iterations': 9088, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.022851770603293388, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:20:08,650]\u001b[0m Trial 212 finished with value: 0.867990076374283 and parameters: {'learning_rate': 0.06237051028703283, 'iterations': 9627, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04127050391182131, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:21:16,793]\u001b[0m Trial 213 finished with value: 0.8673433915771949 and parameters: {'learning_rate': 0.06707578413012372, 'iterations': 10210, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03917738812763858, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:22:44,002]\u001b[0m Trial 214 finished with value: 0.8684871288618123 and parameters: {'learning_rate': 0.0698090717722592, 'iterations': 8973, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02710898342803852, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:24:02,816]\u001b[0m Trial 215 finished with value: 0.8686997294394236 and parameters: {'learning_rate': 0.06529393859785007, 'iterations': 9906, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030568616668308724, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:25:25,887]\u001b[0m Trial 216 finished with value: 0.8683131450969066 and parameters: {'learning_rate': 0.06378773237525368, 'iterations': 9873, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030268051897324275, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:26:46,470]\u001b[0m Trial 217 finished with value: 0.8678056986259781 and parameters: {'learning_rate': 0.05846460922371387, 'iterations': 9482, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03373238298663273, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:28:30,946]\u001b[0m Trial 218 finished with value: 0.8686914849523173 and parameters: {'learning_rate': 0.06731395965902388, 'iterations': 10137, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02504046642432424, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:29:51,913]\u001b[0m Trial 219 finished with value: 0.8683170620835196 and parameters: {'learning_rate': 0.0655855480019938, 'iterations': 10433, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.028604437769799512, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:31:22,383]\u001b[0m Trial 220 finished with value: 0.8684111381812072 and parameters: {'learning_rate': 0.06876524853385117, 'iterations': 10141, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02601830498647413, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:32:40,689]\u001b[0m Trial 221 finished with value: 0.8684992618276232 and parameters: {'learning_rate': 0.07430445542635539, 'iterations': 9932, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030793489283342086, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:34:36,950]\u001b[0m Trial 222 finished with value: 0.8683736901927458 and parameters: {'learning_rate': 0.060686391816195645, 'iterations': 9224, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.023094980383659168, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:35:56,649]\u001b[0m Trial 223 finished with value: 0.8679677432788501 and parameters: {'learning_rate': 0.07180847416933829, 'iterations': 9599, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03244320676163612, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:37:11,532]\u001b[0m Trial 224 finished with value: 0.8679087832227174 and parameters: {'learning_rate': 0.06747915983326565, 'iterations': 10253, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02792373048376383, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:38:24,576]\u001b[0m Trial 225 finished with value: 0.8689565742908567 and parameters: {'learning_rate': 0.07250051367083894, 'iterations': 10648, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03518449867318291, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:39:30,529]\u001b[0m Trial 226 finished with value: 0.8678754460746457 and parameters: {'learning_rate': 0.07162164576037316, 'iterations': 10802, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03454796795757217, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:40:38,513]\u001b[0m Trial 227 finished with value: 0.8684478335608872 and parameters: {'learning_rate': 0.047458276431802345, 'iterations': 11126, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.040155500591190635, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:42:11,077]\u001b[0m Trial 228 finished with value: 0.8680193197867098 and parameters: {'learning_rate': 0.06418153035175114, 'iterations': 10490, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02968439746821716, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:43:47,213]\u001b[0m Trial 229 finished with value: 0.8680667398415262 and parameters: {'learning_rate': 0.0787028969549051, 'iterations': 11564, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03325229775703199, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:45:52,610]\u001b[0m Trial 230 finished with value: 0.8684528281464371 and parameters: {'learning_rate': 0.0602300933137356, 'iterations': 10654, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.01857787666843072, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:47:28,458]\u001b[0m Trial 231 finished with value: 0.8688927963982059 and parameters: {'learning_rate': 0.07563951013570332, 'iterations': 9936, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.036482331027352474, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:48:49,694]\u001b[0m Trial 232 finished with value: 0.8687026144397529 and parameters: {'learning_rate': 0.07179759793401946, 'iterations': 9951, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.036256687952898874, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:49:43,398]\u001b[0m Trial 233 finished with value: 0.8678092108002917 and parameters: {'learning_rate': 0.07351200086549678, 'iterations': 9996, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.036448805668350244, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:50:31,201]\u001b[0m Trial 234 finished with value: 0.8679861707908337 and parameters: {'learning_rate': 0.07718441657108725, 'iterations': 10447, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03609794569245635, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:51:30,555]\u001b[0m Trial 235 finished with value: 0.8686139263367506 and parameters: {'learning_rate': 0.0701386499812193, 'iterations': 9292, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03807723118032798, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:52:38,683]\u001b[0m Trial 236 finished with value: 0.8686771796838869 and parameters: {'learning_rate': 0.07238219484996568, 'iterations': 9831, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.038571234947844396, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:53:45,978]\u001b[0m Trial 237 finished with value: 0.8673010516316536 and parameters: {'learning_rate': 0.08084140619024305, 'iterations': 9877, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.037862486585320536, 'depth': 11, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:55:04,653]\u001b[0m Trial 238 finished with value: 0.8686118908720916 and parameters: {'learning_rate': 0.0720181749036018, 'iterations': 9314, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.035297660657353916, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:55:59,404]\u001b[0m Trial 239 finished with value: 0.8678860282102402 and parameters: {'learning_rate': 0.07169906220117711, 'iterations': 9306, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.035447352227813526, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.8722063628149346}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:57:21,107]\u001b[0m Trial 240 finished with value: 0.8684629997681509 and parameters: {'learning_rate': 0.0741182743000871, 'iterations': 9761, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03918101007653841, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:58:28,813]\u001b[0m Trial 241 finished with value: 0.8686107334510109 and parameters: {'learning_rate': 0.07469126128966357, 'iterations': 9442, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03796615722799062, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 02:59:17,774]\u001b[0m Trial 242 finished with value: 0.8680318803711465 and parameters: {'learning_rate': 0.07595794300808524, 'iterations': 9409, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03702534599013375, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:00:35,951]\u001b[0m Trial 243 finished with value: 0.8683069246712957 and parameters: {'learning_rate': 0.07147792836877666, 'iterations': 8975, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.034842441318690846, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:01:27,395]\u001b[0m Trial 244 finished with value: 0.8680412423682619 and parameters: {'learning_rate': 0.07800996469654967, 'iterations': 9194, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04003903800078795, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:02:27,193]\u001b[0m Trial 245 finished with value: 0.8684627032859036 and parameters: {'learning_rate': 0.06897779869098969, 'iterations': 10109, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03793025706280808, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:03:40,250]\u001b[0m Trial 246 finished with value: 0.868340233311459 and parameters: {'learning_rate': 0.07404035803405386, 'iterations': 9429, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03583524348717228, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:05:12,314]\u001b[0m Trial 247 finished with value: 0.8687075406063225 and parameters: {'learning_rate': 0.06414821000363104, 'iterations': 9700, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03144227925090641, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:06:50,322]\u001b[0m Trial 248 finished with value: 0.8686590999683859 and parameters: {'learning_rate': 0.0644166927197333, 'iterations': 9788, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03127336358256298, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:08:45,310]\u001b[0m Trial 249 finished with value: 0.8688206029709984 and parameters: {'learning_rate': 0.06493441805554645, 'iterations': 9880, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.031553346297031, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:10:30,935]\u001b[0m Trial 250 finished with value: 0.8687545444456766 and parameters: {'learning_rate': 0.06275876557968724, 'iterations': 9922, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030967407280939396, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:12:01,006]\u001b[0m Trial 251 finished with value: 0.8683139034072698 and parameters: {'learning_rate': 0.05623060902304945, 'iterations': 10080, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03163054965725314, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:13:24,729]\u001b[0m Trial 252 finished with value: 0.8684184248025917 and parameters: {'learning_rate': 0.06185275544190957, 'iterations': 9836, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.031575554246940626, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:14:58,796]\u001b[0m Trial 253 finished with value: 0.8685508953512997 and parameters: {'learning_rate': 0.06290648500530274, 'iterations': 10320, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03293851361171137, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:16:03,291]\u001b[0m Trial 254 finished with value: 0.8684410144692004 and parameters: {'learning_rate': 0.06516808250692416, 'iterations': 10061, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03007614320313559, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:17:35,677]\u001b[0m Trial 255 finished with value: 0.8685367440255748 and parameters: {'learning_rate': 0.05404737293525184, 'iterations': 9770, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03423868768080971, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:18:58,743]\u001b[0m Trial 256 finished with value: 0.8681501083688227 and parameters: {'learning_rate': 0.06526859111988366, 'iterations': 9733, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02842690857472914, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:20:38,543]\u001b[0m Trial 257 finished with value: 0.8684276442601652 and parameters: {'learning_rate': 0.05953464857627346, 'iterations': 10236, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.033311468813871474, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:22:07,081]\u001b[0m Trial 258 finished with value: 0.8682893010823289 and parameters: {'learning_rate': 0.06796513068203383, 'iterations': 9777, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02560945341966391, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:23:52,532]\u001b[0m Trial 259 finished with value: 0.8686754749109651 and parameters: {'learning_rate': 0.06350970334568795, 'iterations': 10021, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03081858406993832, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:24:27,579]\u001b[0m Trial 260 finished with value: 0.868417130543551 and parameters: {'learning_rate': 0.1276605431471775, 'iterations': 10211, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030698961925138362, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:26:00,275]\u001b[0m Trial 261 finished with value: 0.868868940980465 and parameters: {'learning_rate': 0.058094330278740364, 'iterations': 10050, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029279547852639316, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:28:11,564]\u001b[0m Trial 262 finished with value: 0.8687711246451962 and parameters: {'learning_rate': 0.05557814929494216, 'iterations': 10440, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03049991795915497, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:29:22,047]\u001b[0m Trial 263 finished with value: 0.8683238811752063 and parameters: {'learning_rate': 0.05220642437615422, 'iterations': 10501, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030210766938551202, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:30:49,800]\u001b[0m Trial 264 finished with value: 0.8680864787172957 and parameters: {'learning_rate': 0.04950477581185751, 'iterations': 10768, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02895428990707865, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:32:58,372]\u001b[0m Trial 265 finished with value: 0.8682000884338125 and parameters: {'learning_rate': 0.05774892020231248, 'iterations': 10976, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03241867526540918, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.26663607484466034}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:34:35,738]\u001b[0m Trial 266 finished with value: 0.8686763757608704 and parameters: {'learning_rate': 0.055681708194833576, 'iterations': 10005, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.031098262804429195, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:35:45,187]\u001b[0m Trial 267 finished with value: 0.8681292120719725 and parameters: {'learning_rate': 0.05070115668699208, 'iterations': 10051, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03145732938217553, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:37:44,181]\u001b[0m Trial 268 finished with value: 0.8685030818873476 and parameters: {'learning_rate': 0.05846925242617281, 'iterations': 9909, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03318173817940977, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:40:07,261]\u001b[0m Trial 269 finished with value: 0.8687870890738949 and parameters: {'learning_rate': 0.04336890769155444, 'iterations': 10298, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030462501344013057, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:41:10,835]\u001b[0m Trial 270 finished with value: 0.868291655835562 and parameters: {'learning_rate': 0.05646922801730564, 'iterations': 10642, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03048560825908042, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:43:17,474]\u001b[0m Trial 271 finished with value: 0.8689364020948798 and parameters: {'learning_rate': 0.04030004540501472, 'iterations': 10337, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.027781093573416148, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:45:02,510]\u001b[0m Trial 272 finished with value: 0.8687951397072242 and parameters: {'learning_rate': 0.04362752997859637, 'iterations': 10447, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02780284621613862, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:47:05,910]\u001b[0m Trial 273 finished with value: 0.8684247763645812 and parameters: {'learning_rate': 0.039355419881523954, 'iterations': 10291, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02691093785355507, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:49:12,599]\u001b[0m Trial 274 finished with value: 0.8685799164020409 and parameters: {'learning_rate': 0.03600564286106223, 'iterations': 10696, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.028401113073216196, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:51:00,749]\u001b[0m Trial 275 finished with value: 0.8681097297673787 and parameters: {'learning_rate': 0.04214165928642104, 'iterations': 10408, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.023855260737211382, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:53:33,720]\u001b[0m Trial 276 finished with value: 0.8682935373575158 and parameters: {'learning_rate': 0.030055816823239274, 'iterations': 11050, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.025788896553911117, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:55:48,629]\u001b[0m Trial 277 finished with value: 0.8683697332950612 and parameters: {'learning_rate': 0.035692930348510434, 'iterations': 10198, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.020729543806341854, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:57:19,116]\u001b[0m Trial 278 finished with value: 0.8684564657555477 and parameters: {'learning_rate': 0.04388033693191735, 'iterations': 10041, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.028740215397725765, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 03:58:50,578]\u001b[0m Trial 279 finished with value: 0.8682514026689149 and parameters: {'learning_rate': 0.046656334316570174, 'iterations': 10531, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029012156817982113, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:00:32,147]\u001b[0m Trial 280 finished with value: 0.8681137037698082 and parameters: {'learning_rate': 0.0420932399993029, 'iterations': 10231, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03356230820692322, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:02:39,236]\u001b[0m Trial 281 finished with value: 0.8686127917219967 and parameters: {'learning_rate': 0.03745959421326387, 'iterations': 10886, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.027134428017090737, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:03:25,877]\u001b[0m Trial 282 finished with value: 0.8678550914280552 and parameters: {'learning_rate': 0.11964364283795194, 'iterations': 10029, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03150600121051823, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:04:30,042]\u001b[0m Trial 283 finished with value: 0.868670964959858 and parameters: {'learning_rate': 0.0492126998568592, 'iterations': 10587, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029815454375532006, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:07:02,636]\u001b[0m Trial 284 finished with value: 0.8682358145446072 and parameters: {'learning_rate': 0.029566978512727167, 'iterations': 10387, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03512079167576, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:08:20,767]\u001b[0m Trial 285 finished with value: 0.8684838903634192 and parameters: {'learning_rate': 0.04526603783365564, 'iterations': 9947, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.032222861589132776, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:09:58,443]\u001b[0m Trial 286 finished with value: 0.8685214295771875 and parameters: {'learning_rate': 0.05108696921951993, 'iterations': 1871, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03440760040179389, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:12:09,031]\u001b[0m Trial 287 finished with value: 0.8679516135042828 and parameters: {'learning_rate': 0.053866588266021795, 'iterations': 9567, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.027472020523585258, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.6234750588059914}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:13:35,070]\u001b[0m Trial 288 finished with value: 0.8679748189417126 and parameters: {'learning_rate': 0.06160492474759202, 'iterations': 10333, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02498872654219481, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:15:35,612]\u001b[0m Trial 289 finished with value: 0.8685131052679376 and parameters: {'learning_rate': 0.03970661730981818, 'iterations': 9980, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02954193022128415, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:17:37,750]\u001b[0m Trial 290 finished with value: 0.868316201144686 and parameters: {'learning_rate': 0.03215379327863955, 'iterations': 9635, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03125755198719675, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:18:54,791]\u001b[0m Trial 291 finished with value: 0.8680856804958607 and parameters: {'learning_rate': 0.04675147248969542, 'iterations': 10734, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03596351909878929, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:20:23,550]\u001b[0m Trial 292 finished with value: 0.8682795342729145 and parameters: {'learning_rate': 0.05626360234604499, 'iterations': 10141, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03338905461327206, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:24:06,265]\u001b[0m Trial 293 finished with value: 0.8684679373378842 and parameters: {'learning_rate': 0.022095824415023196, 'iterations': 9640, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.024651091180708462, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:25:19,623]\u001b[0m Trial 294 finished with value: 0.8684523036009228 and parameters: {'learning_rate': 0.06197262283021997, 'iterations': 10311, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030370820425482652, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:26:58,505]\u001b[0m Trial 295 finished with value: 0.8684323024523962 and parameters: {'learning_rate': 0.04276324277019272, 'iterations': 11171, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02759139635145763, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:28:15,392]\u001b[0m Trial 296 finished with value: 0.8682730002603115 and parameters: {'learning_rate': 0.0540300733780884, 'iterations': 9905, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.031990543541736124, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:29:11,933]\u001b[0m Trial 297 finished with value: 0.868146647508744 and parameters: {'learning_rate': 0.0642762742393428, 'iterations': 10566, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03619517179304327, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:30:42,468]\u001b[0m Trial 298 finished with value: 0.8683470181936559 and parameters: {'learning_rate': 0.0695465963192377, 'iterations': 9674, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029176543860417382, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:31:46,389]\u001b[0m Trial 299 finished with value: 0.8684446292719843 and parameters: {'learning_rate': 0.08325066058491895, 'iterations': 1162, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03388144302030677, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:33:04,503]\u001b[0m Trial 300 finished with value: 0.868703196001084 and parameters: {'learning_rate': 0.062113621678573705, 'iterations': 10017, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03260689953609387, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:34:10,885]\u001b[0m Trial 301 finished with value: 0.8685866214620944 and parameters: {'learning_rate': 0.05926740728806139, 'iterations': 10338, 'objective': 'Logloss', 'colsample_bylevel': 0.03271471596494417, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:35:06,366]\u001b[0m Trial 302 finished with value: 0.868365468511966 and parameters: {'learning_rate': 0.0798474330477295, 'iterations': 9586, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03653792850909008, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:35:32,789]\u001b[0m Trial 303 finished with value: 0.8685194397251818 and parameters: {'learning_rate': 0.05404735905640688, 'iterations': 9852, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.027077387358351076, 'depth': 4, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:36:29,582]\u001b[0m Trial 304 finished with value: 0.8684373540537629 and parameters: {'learning_rate': 0.06832657643010234, 'iterations': 10167, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.05036627984219938, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:37:05,340]\u001b[0m Trial 305 finished with value: 0.8678567163788332 and parameters: {'learning_rate': 0.13396211429820704, 'iterations': 10911, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03469560313473695, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:38:40,815]\u001b[0m Trial 306 finished with value: 0.8687401764598478 and parameters: {'learning_rate': 0.06556099544378799, 'iterations': 10672, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.025448422710549277, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:39:52,002]\u001b[0m Trial 307 finished with value: 0.8683254206022593 and parameters: {'learning_rate': 0.06803869741015933, 'iterations': 11611, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.021638605324493923, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:41:05,106]\u001b[0m Trial 308 finished with value: 0.868102887869365 and parameters: {'learning_rate': 0.0656880140408706, 'iterations': 10641, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02528821999354283, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:42:54,325]\u001b[0m Trial 309 finished with value: 0.8682546126593997 and parameters: {'learning_rate': 0.07847274941006474, 'iterations': 11269, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.026733080680546126, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.4170288482085974}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:44:14,770]\u001b[0m Trial 310 finished with value: 0.8685277241233601 and parameters: {'learning_rate': 0.07064940228732733, 'iterations': 10899, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.024805786658721685, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:46:05,668]\u001b[0m Trial 311 finished with value: 0.868360154637842 and parameters: {'learning_rate': 0.03552947921488607, 'iterations': 10522, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.023920803810423396, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:47:27,341]\u001b[0m Trial 312 finished with value: 0.8688155000553965 and parameters: {'learning_rate': 0.06207198653429466, 'iterations': 2952, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029343710628875256, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:48:42,539]\u001b[0m Trial 313 finished with value: 0.8683092167071303 and parameters: {'learning_rate': 0.05888753083860525, 'iterations': 2588, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.028390650818169272, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:49:41,111]\u001b[0m Trial 314 finished with value: 0.8686918327487997 and parameters: {'learning_rate': 0.061897996868137704, 'iterations': 10381, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.0290435896578519, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:50:11,283]\u001b[0m Trial 315 finished with value: 0.867275770818493 and parameters: {'learning_rate': 0.1730268633966735, 'iterations': 10718, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02857387768637801, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:51:22,453]\u001b[0m Trial 316 finished with value: 0.8682568305746724 and parameters: {'learning_rate': 0.06280370770892643, 'iterations': 2225, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029707252260592707, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:53:14,293]\u001b[0m Trial 317 finished with value: 0.8675315779820606 and parameters: {'learning_rate': 0.061992161136803274, 'iterations': 3311, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03208108806557697, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 8.648267646671119}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:54:28,037]\u001b[0m Trial 318 finished with value: 0.8683019528920726 and parameters: {'learning_rate': 0.06105477557254563, 'iterations': 1558, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030112710978828245, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:54:55,455]\u001b[0m Trial 319 finished with value: 0.8682809254588438 and parameters: {'learning_rate': 0.040625715285698236, 'iterations': 2677, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03390116841830811, 'depth': 5, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:55:57,392]\u001b[0m Trial 320 finished with value: 0.8680399367060575 and parameters: {'learning_rate': 0.05788578758914843, 'iterations': 3117, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03192737986023671, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:58:02,061]\u001b[0m Trial 321 finished with value: 0.867775947772782 and parameters: {'learning_rate': 0.02358742172618031, 'iterations': 10358, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.026971752433470955, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 04:59:54,303]\u001b[0m Trial 322 finished with value: 0.8681750812965725 and parameters: {'learning_rate': 0.04994406705969586, 'iterations': 11108, 'objective': 'Logloss', 'colsample_bylevel': 0.017932104176798344, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:00:52,843]\u001b[0m Trial 323 finished with value: 0.8678495951033174 and parameters: {'learning_rate': 0.0652644175202671, 'iterations': 10602, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.029186760771364388, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:01:43,719]\u001b[0m Trial 324 finished with value: 0.8678133045359366 and parameters: {'learning_rate': 0.11570646062174984, 'iterations': 9371, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03295257958980049, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:03:28,327]\u001b[0m Trial 325 finished with value: 0.868411924999479 and parameters: {'learning_rate': 0.03327212697069648, 'iterations': 9497, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.030979083459658028, 'depth': 6, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:04:36,992]\u001b[0m Trial 326 finished with value: 0.868895185360929 and parameters: {'learning_rate': 0.0641693287318327, 'iterations': 10297, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03513270137649239, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:05:19,979]\u001b[0m Trial 327 finished with value: 0.8680565910261392 and parameters: {'learning_rate': 0.08349957469650046, 'iterations': 9798, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.035385780014116804, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:06:19,909]\u001b[0m Trial 328 finished with value: 0.8686522694735355 and parameters: {'learning_rate': 0.07061207817243581, 'iterations': 10121, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03685652778866367, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:07:17,957]\u001b[0m Trial 329 finished with value: 0.8681832060504636 and parameters: {'learning_rate': 0.07606847647166581, 'iterations': 2866, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03401398298606554, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:08:17,333]\u001b[0m Trial 330 finished with value: 0.8680292918530647 and parameters: {'learning_rate': 0.10965439051742754, 'iterations': 12060, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03279833746488618, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:10:19,537]\u001b[0m Trial 331 finished with value: 0.8685781489117207 and parameters: {'learning_rate': 0.06535114817202094, 'iterations': 10928, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03476561763227012, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:11:06,706]\u001b[0m Trial 332 finished with value: 0.8673572236143458 and parameters: {'learning_rate': 0.13875147050625827, 'iterations': 3652, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.036955156558958216, 'depth': 5, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bernoulli', 'subsample': 0.22837752440868925}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:12:25,485]\u001b[0m Trial 333 finished with value: 0.8683496979370445 and parameters: {'learning_rate': 0.08218330806122877, 'iterations': 9221, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03085829627627782, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:14:49,389]\u001b[0m Trial 334 finished with value: 0.8679497775949825 and parameters: {'learning_rate': 0.06738510560530596, 'iterations': 10273, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03250786400355932, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'Bayesian', 'bagging_temperature': 8.423370382698312}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:16:22,650]\u001b[0m Trial 335 finished with value: 0.8684251526689719 and parameters: {'learning_rate': 0.0583935439204129, 'iterations': 9584, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.027153878637132742, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:17:29,288]\u001b[0m Trial 336 finished with value: 0.8684608616750216 and parameters: {'learning_rate': 0.0767994858849351, 'iterations': 9952, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03491888560672328, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:18:26,000]\u001b[0m Trial 337 finished with value: 0.8682239495531352 and parameters: {'learning_rate': 0.07194715557302071, 'iterations': 10624, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.07053654075831918, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:20:36,106]\u001b[0m Trial 338 finished with value: 0.8685121017895623 and parameters: {'learning_rate': 0.06425882227037932, 'iterations': 10169, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03024419400019015, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:21:07,137]\u001b[0m Trial 339 finished with value: 0.8680655596141187 and parameters: {'learning_rate': 0.04535748559186009, 'iterations': 9583, 'objective': 'Logloss', 'colsample_bylevel': 0.03358359599342361, 'depth': 3, 'boosting_type': 'Plain', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:23:25,316]\u001b[0m Trial 340 finished with value: 0.8684198958106648 and parameters: {'learning_rate': 0.05413060118067993, 'iterations': 13836, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02609946791520609, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:24:14,042]\u001b[0m Trial 341 finished with value: 0.8681988283842617 and parameters: {'learning_rate': 0.08057486094245458, 'iterations': 9072, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.060922183352734326, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:25:35,018]\u001b[0m Trial 342 finished with value: 0.8681857261495655 and parameters: {'learning_rate': 0.060648132530538724, 'iterations': 8372, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.02852636747391732, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:26:35,280]\u001b[0m Trial 343 finished with value: 0.8682579024720279 and parameters: {'learning_rate': 0.06750872724806585, 'iterations': 2071, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.04130462403748693, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:29:06,738]\u001b[0m Trial 344 finished with value: 0.8671885480219823 and parameters: {'learning_rate': 0.07105472261309619, 'iterations': 9835, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.031673390467247604, 'depth': 1, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:30:04,089]\u001b[0m Trial 345 finished with value: 0.8681353412722765 and parameters: {'learning_rate': 0.07579391549849647, 'iterations': 2373, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03663878775044306, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:31:15,461]\u001b[0m Trial 346 finished with value: 0.8678720878430372 and parameters: {'learning_rate': 0.0640783735537818, 'iterations': 10401, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.03027308392738572, 'depth': 2, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:33:01,728]\u001b[0m Trial 347 finished with value: 0.8685005047724291 and parameters: {'learning_rate': 0.05213579559657316, 'iterations': 9471, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.034851045049733895, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:34:14,804]\u001b[0m Trial 348 finished with value: 0.868115938789826 and parameters: {'learning_rate': 0.06909901825313665, 'iterations': 11434, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.039011796952408126, 'depth': 4, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n",
      "\u001b[32m[I 2021-12-26 05:35:26,626]\u001b[0m Trial 349 finished with value: 0.8681960859234744 and parameters: {'learning_rate': 0.08520998217764907, 'iterations': 10084, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.058759393031581786, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}. Best is trial 76 with value: 0.8690178605923141.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1d 33min, sys: 50min 21s, total: 1d 1h 23min 22s\n",
      "Wall time: 7h 26min 44s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_cat_pipe1 = optuna.create_study(direction=\"maximize\")\n",
    "study_cat_pipe1.optimize(objective_cat_pipe1 ,n_trials=350)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "13321360",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-26T05:35:27.068419Z",
     "iopub.status.busy": "2021-12-26T05:35:27.067740Z",
     "iopub.status.idle": "2021-12-26T05:35:27.142343Z",
     "shell.execute_reply": "2021-12-26T05:35:27.142854Z"
    },
    "papermill": {
     "duration": 0.222402,
     "end_time": "2021-12-26T05:35:27.143058",
     "exception": false,
     "start_time": "2021-12-26T05:35:26.920656",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of finished trials :  350\n",
      "Best Trials :  {'learning_rate': 0.08926084391040985, 'iterations': 2073, 'objective': 'CrossEntropy', 'colsample_bylevel': 0.08311513863516848, 'depth': 3, 'boosting_type': 'Ordered', 'bootstrap_type': 'MVS'}\n",
      "Best Values :  0.8690178605923141\n"
     ]
    }
   ],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_cat_pipe1.trials))\n",
    "print(\"Best Trials : \", study_cat_pipe1.best_trial.params)\n",
    "print(\"Best Values : \" , study_cat_pipe1.best_value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 29277.292213,
   "end_time": "2021-12-26T05:35:28.554703",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-25T21:27:31.262490",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}