credit-part3-modeling-1-pipe2-tune-lgbm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b1eca91e",
   "metadata": {
    "_cell_guid": "ddedf45c-cdd2-4013-973f-6ccf6c78347a",
    "_uuid": "20c9b28e-4c93-447d-8210-cad89f404354",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:51.963241Z",
     "iopub.status.busy": "2021-12-25T12:37:51.961666Z",
     "iopub.status.idle": "2021-12-25T12:37:55.534121Z",
     "shell.execute_reply": "2021-12-25T12:37:55.533516Z",
     "shell.execute_reply.started": "2021-12-25T11:33:55.975898Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 3.604815,
     "end_time": "2021-12-25T12:37:55.534300",
     "exception": false,
     "start_time": "2021-12-25T12:37:51.929485",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       ".datatable .frame thead tr.colnames {  background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABwAAAA4CAYAAADuMJi0AAAGR0lEQVR42rVZ21IbRxBtCbQrkIR2dQVjsLmDLBsET3nTQ8ouYRkQVf6e/E9+Im958qMfkgoXAaKSSj6C9Jnd2R2NeiRSRaZqitVOT5+Z6dNnWoKGlN94JFp8Ipofkb/7SOXjGyp8wF+z35K3f0uUp/GW4XfLQ8v2gefj3ZCCzojoNfue+43o1Q3l3xB/yA3JO7jnF2pCLnI+pNyx/qw7L+SQ7T2N9p2f8c60QcfcK6KGXsAd+ZvA4LlZYuSSAoOhMs5vwJkEGDlbPMaJoA+FcQ0IH38QLWkbAFLkOOhoMF5tU6/eBRhNjro0ZgKiPRAt3FLhCO/vqdgmNTm32LkmKpvBmQY4q5uAaAgbwDBG2BVv3bfI8KKAMWj2kfw9+pkZREIbEql4ST1x7hgHIANkbJ//MF8mAH/ilTCJ2tIi4ASr1IC3VNqXHKOxjy34mgoImnOQtx1g81fkqTiMOBVGcTogNhiT5iBHET8R8C+iApJUmgim3SQAXhsLQz7ee2G8gOAQNtJckBEplADiAxtX+G9NmhDl0qJKnTvyWlAMPYZnvIviGXRg6/Dh824DBXhP/tbfREXJEIvQ+aaPGjG7pvw6r3xdx+9hqb4dgZaP2XmdHO2K/B0c1+oUph6k8kShBryl/Ft0DYgjTlOieOACHFFpVyUl72T9V3cM1jUoYvxIC2vpCSys/ck70mDYuYvdvKjlMdKAUThneWVU1aAsyjv6PURDiwNsHGBZzY+JtAAgE2TFxdRHJdyIp/f+zqu09M5cDP2F08Ukkpj4YNSdX950HY2pNCCUK/Hhx5ZMBfjNSEzdsIihVzzAMdn9dz4eDYhnyQb9SSCiAryiJcQk82LiTbJ4x2FZJaUenpKnzP95WyDf4Y+QN9EFHHSeDLGdBjjKNQ5vKHf4XMA7KrY0y0GEObBOO/8e1ywuQExOHXktuQyJALEBpcEqhwtHqgiDuCK5b6i0p2MQpcckIIoh+6hYgTZtO8xlMi6O4tKCF/kOGHEg/W0UUpHW0ZoGNZ1ExZWcn7EErgwt4uj50E/sFBjXXIayWvh7WryjasxarZKssXon0zxvvkc32Q0bqbBCuZiKt9dWFysfQefeL29JYFaeztX6tePaZdz5mYx8+6Zq3Mk0wXECQxlhdzgS2wjBHju3j1RIgKyOMdNUE8X0+RAdbSapS11MRCv1SzUXmO6wGZe2SQYrv2MvCSWEv2VODE6DN7bz8ufypgQKW7uQskFTQHULLKyaEyrnlZbgOGLrV5qrn9U79jjm2HJmgkaVN98AfBub91lGPLZBqdroN5LYgjSu4zYZDDHXZOIPC691HqrWI1900I8qLzgKP4ft8DxEWigprPfrO+KcXno9gZz4jjGewWdUcpGCj0qVFuGPYbl2VturndZ2qRvlL8acDO6lF/DY/VjsFesiUK+ypJ+r/ep+cJkSQxEK4PG4WozgA75TYrDDqStE69K8/mzGEM+JXTeqvmedEElMmwCMm2SLd6bNNF9su02zEtoW6nAQtpMj5Gd7fKa//wqonF7UdtHFsVn+6hf1o7AfriPH7M6EeIUEF5zKVxXbYo7kS/OEtOqDYZKPoBsETIixn0uYrasThmzDkhdKPkz2EnaX0HdQbIgr59vAdGYDqjHrxkjS7WOxkTD8sqEqhiwcJETgBYigrBqF08KyDaje9SZ/I1A7MzaTzMGDEulPtZUkuKcyIRAjxEJPVrnVlb/9wkfij31D/pQt1IN+iL8bGJcstBIO7Y5VI/cwDqURbXhMuJxBqD0KLoK3esWFs0Jz5i5ZvJUAfFJMFb9XmGIOnzGpijpcWYCaMqXSQWp8EnCABepQ0Elyi4wfKfsw78ikIqif1pe1AGPlLmojl1SKxHHXp1L+Ut7AmDQHvhI5xHGi4EooO2BR7k78PEkJOdL7cAxQUZ/Tyclu9gnfwGgOmm2lNHGNmZXsq4Pqgc1EG1ATrvKl8s4R9ywwnqulGUnaRLVhxy8v3ieUwy2hbooT68uscW++DCDH0WSzuoyN2D4LUJ/tLECbcSKznwMIFs0ChF4mRTCnQbIIfk4SHJo6A9BMuTnXTs3Ku/KxsgZWqzuSe+Os8cEUfnMBY6UF5gi3SUbd5K7vDjq5WW0UENJlRsWn4sy21Er/E/AvPQSFHy1p4fgAAAAASUVORK5CYII=');  background-repeat: repeat-x;  background-size: 14px;  height: 28px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "import warnings\n",
    "# Preparation  \n",
    "from sklearn.preprocessing import LabelEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PowerTransformer, StandardScaler,Normalizer,RobustScaler,MaxAbsScaler,MinMaxScaler,QuantileTransformer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import KBinsDiscretizer\n",
    "# Import StandardScaler from scikit-learn\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer,IterativeImputer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.compose import make_column_transformer,ColumnTransformer\n",
    "from sklearn.pipeline import make_pipeline, Pipeline,FeatureUnion\n",
    "from sklearn.manifold import TSNE\n",
    "# Import train_test_split()\n",
    "# Metrics\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from sklearn.metrics import make_scorer,mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import roc_curve,confusion_matrix\n",
    "from datetime import datetime, date\n",
    "from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#import tensorflow as tf \n",
    "#from tensorflow.keras import layers\n",
    "#from tensorflow.keras.callbacks import EarlyStopping\n",
    "#from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "#import smogn\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "# For training random forest model\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from sklearn.neighbors import KNeighborsRegressor \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer\n",
    "# Model selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_regression,f_classif,chi2\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.feature_selection import mutual_info_classif,VarianceThreshold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import lightgbm as lgbm\n",
    "#from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import svm\n",
    "from xgboost import XGBClassifier,XGBRegressor\n",
    "from sklearn import set_config\n",
    "from itertools import combinations\n",
    "# Cluster :\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "#from yellowbrick.cluster import KElbowVisualizer\n",
    "#import smong \n",
    "import category_encoders as ce\n",
    "import warnings\n",
    "import optuna \n",
    "from joblib import Parallel, delayed\n",
    "import joblib \n",
    "from sklearn import set_config\n",
    "from typing import List, Optional, Union\n",
    "set_config(display='diagram')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "169bc4eb",
   "metadata": {
    "_cell_guid": "38c6befb-892d-4ef3-9c63-13be5959b671",
    "_uuid": "2953f9f3-02f8-4fa9-a7fc-cb93565c2183",
    "papermill": {
     "duration": 0.030081,
     "end_time": "2021-12-25T12:37:55.595595",
     "exception": false,
     "start_time": "2021-12-25T12:37:55.565514",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "##  Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "686c046c",
   "metadata": {
    "_cell_guid": "98a5b4a6-90e9-449d-96cc-0d85c9ff1c77",
    "_uuid": "e086592f-9f6a-4592-80b5-c1e7ad227e02",
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:55.657099Z",
     "iopub.status.busy": "2021-12-25T12:37:55.656567Z",
     "iopub.status.idle": "2021-12-25T12:37:56.094960Z",
     "shell.execute_reply": "2021-12-25T12:37:56.095449Z",
     "shell.execute_reply.started": "2021-12-25T11:33:59.624619Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 0.471587,
     "end_time": "2021-12-25T12:37:56.095617",
     "exception": false,
     "start_time": "2021-12-25T12:37:55.624030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 200 ms, sys: 69.8 ms, total: 269 ms\n",
      "Wall time: 422 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>SeriousDlqin2yrs</th>\n",
       "      <th>RevolvingUtilizationOfUnsecuredLines</th>\n",
       "      <th>age</th>\n",
       "      <th>NumberOfTime30-59DaysPastDueNotWorse</th>\n",
       "      <th>DebtRatio</th>\n",
       "      <th>MonthlyIncome</th>\n",
       "      <th>NumberOfOpenCreditLinesAndLoans</th>\n",
       "      <th>NumberOfTimes90DaysLate</th>\n",
       "      <th>NumberRealEstateLoansOrLines</th>\n",
       "      <th>NumberOfTime60-89DaysPastDueNotWorse</th>\n",
       "      <th>NumberOfDependents</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.766127</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.802982</td>\n",
       "      <td>9120.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.957151</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0.121876</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.658180</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085113</td>\n",
       "      <td>3042.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  SeriousDlqin2yrs  RevolvingUtilizationOfUnsecuredLines  age  \\\n",
       "0           1                 1                              0.766127   45   \n",
       "1           2                 0                              0.957151   40   \n",
       "2           3                 0                              0.658180   38   \n",
       "\n",
       "   NumberOfTime30-59DaysPastDueNotWorse  DebtRatio  MonthlyIncome  \\\n",
       "0                                     2   0.802982         9120.0   \n",
       "1                                     0   0.121876         2600.0   \n",
       "2                                     1   0.085113         3042.0   \n",
       "\n",
       "   NumberOfOpenCreditLinesAndLoans  NumberOfTimes90DaysLate  \\\n",
       "0                               13                        0   \n",
       "1                                4                        0   \n",
       "2                                2                        1   \n",
       "\n",
       "   NumberRealEstateLoansOrLines  NumberOfTime60-89DaysPastDueNotWorse  \\\n",
       "0                             6                                     0   \n",
       "1                             0                                     0   \n",
       "2                             0                                     0   \n",
       "\n",
       "   NumberOfDependents  \n",
       "0                 2.0  \n",
       "1                 1.0  \n",
       "2                 0.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "train = pd.read_csv('../input/GiveMeSomeCredit/cs-training.csv')\n",
    "test = pd.read_csv('../input/GiveMeSomeCredit/cs-test.csv')\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6c9e4d9",
   "metadata": {
    "papermill": {
     "duration": 0.029194,
     "end_time": "2021-12-25T12:37:56.154449",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.125255",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Convert Dtypes :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bc19c6e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:56.236375Z",
     "iopub.status.busy": "2021-12-25T12:37:56.214176Z",
     "iopub.status.idle": "2021-12-25T12:37:56.289804Z",
     "shell.execute_reply": "2021-12-25T12:37:56.289255Z",
     "shell.execute_reply.started": "2021-12-25T11:34:00.070928Z"
    },
    "papermill": {
     "duration": 0.107958,
     "end_time": "2021-12-25T12:37:56.289929",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.181971",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert Dtypes :\n",
    "train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = train[train.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "train[train.select_dtypes(['object','category']).columns] = train.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))\n",
    "# Convert Dtypes :\n",
    "test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns] = test[test.select_dtypes(['int64','int16','float16','float32','float64','int8']).columns].apply(pd.to_numeric)\n",
    "test[test.select_dtypes(['object','category']).columns] = test.select_dtypes(['object','category']).apply(lambda x: x.astype('category'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8178ee4",
   "metadata": {
    "papermill": {
     "duration": 0.02726,
     "end_time": "2021-12-25T12:37:56.344893",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.317633",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Reduce memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ab4b3639",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:56.418519Z",
     "iopub.status.busy": "2021-12-25T12:37:56.416842Z",
     "iopub.status.idle": "2021-12-25T12:37:56.460247Z",
     "shell.execute_reply": "2021-12-25T12:37:56.460848Z",
     "shell.execute_reply.started": "2021-12-25T11:34:00.152559Z"
    },
    "papermill": {
     "duration": 0.087748,
     "end_time": "2021-12-25T12:37:56.461040",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.373292",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 13.73 MB\n",
      "Memory usage after optimization is: 3.29 MB\n",
      "Decreased by 76.0%\n",
      "Memory usage of dataframe is 9.29 MB\n",
      "Memory usage after optimization is: 2.90 MB\n",
      "Decreased by 68.7%\n"
     ]
    }
   ],
   "source": [
    "# Author : https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        name =df[col].dtype.name \n",
    "        \n",
    "        if col_type != object and col_type.name != 'category':\n",
    "        #if name != \"category\":    \n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "\n",
    "train= reduce_mem_usage(train)\n",
    "test= reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27f5e3f2",
   "metadata": {
    "papermill": {
     "duration": 0.029249,
     "end_time": "2021-12-25T12:37:56.519067",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.489818",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# X and Y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e10ef23f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:56.580638Z",
     "iopub.status.busy": "2021-12-25T12:37:56.580082Z",
     "iopub.status.idle": "2021-12-25T12:37:56.583802Z",
     "shell.execute_reply": "2021-12-25T12:37:56.583232Z",
     "shell.execute_reply.started": "2021-12-25T11:34:00.215486Z"
    },
    "papermill": {
     "duration": 0.036616,
     "end_time": "2021-12-25T12:37:56.583916",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.547300",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cardinality : \n",
    "# - RevolvingUtilizationOfUnsecuredLines :125728, high Outlier\n",
    "# - DebtRatio :114194 , high Outlier \n",
    "# deal with outlier + bin \n",
    "PERCENTAGE = [\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\"]\n",
    "# MonthlyIncome:13594 , high outlier +bin \n",
    "REAL= [\"MonthlyIncome\"]\n",
    "# Can be considred as cat \n",
    "NUMERIC_DISCRET_low = [\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "Late_Pay_Cols = ['NumberOfTime30-59DaysPastDueNotWorse',\n",
    "                 'NumberOfTimes90DaysLate',\n",
    "                 'NumberOfTime60-89DaysPastDueNotWorse']\n",
    "TARGET = [\"SeriousDlqin2yrs\"]\n",
    "\n",
    "#also change the type for TARGET to categorical\n",
    "#df[TARGET] = df[TARGET].astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "589b2423",
   "metadata": {
    "papermill": {
     "duration": 0.027438,
     "end_time": "2021-12-25T12:37:56.638944",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.611506",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "da9d3db0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:56.698517Z",
     "iopub.status.busy": "2021-12-25T12:37:56.697945Z",
     "iopub.status.idle": "2021-12-25T12:37:56.701262Z",
     "shell.execute_reply": "2021-12-25T12:37:56.701663Z",
     "shell.execute_reply.started": "2021-12-25T11:34:00.223708Z"
    },
    "papermill": {
     "duration": 0.034891,
     "end_time": "2021-12-25T12:37:56.701793",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.666902",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = train['SeriousDlqin2yrs']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66b0da62",
   "metadata": {
    "papermill": {
     "duration": 0.027383,
     "end_time": "2021-12-25T12:37:56.756864",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.729481",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Impute NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c272c78",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:56.822579Z",
     "iopub.status.busy": "2021-12-25T12:37:56.821979Z",
     "iopub.status.idle": "2021-12-25T12:37:56.826204Z",
     "shell.execute_reply": "2021-12-25T12:37:56.827193Z",
     "shell.execute_reply.started": "2021-12-25T11:34:00.236418Z"
    },
    "papermill": {
     "duration": 0.042674,
     "end_time": "2021-12-25T12:37:56.827375",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.784701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 892 µs, sys: 3 µs, total: 895 µs\n",
      "Wall time: 779 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "param =  {   \"verbosity\": 0,\n",
    "            #\"objective\": \"binary:logistic\",\n",
    "            #\"eval_metric\": \"auc\",\n",
    "            'random_state': 42,\n",
    "            # regression\n",
    "            'objective':'reg:squarederror', \n",
    "             'eval_metric': 'mae',\n",
    "            #early_stopping_rounds=100 ,\n",
    "            'gpu_id':0, \n",
    "            'predictor':\"gpu_predictor\",\n",
    "            # use exact for small dataset.\n",
    "            #\"tree_method\": \"exact\",\n",
    "            # big data :\n",
    "             'tree_method': 'gpu_hist',\n",
    "            # defines booster, gblinear for linear functions.\n",
    "             'booster': 'gbtree', \n",
    "            'lambda': 8.544792472633987e-07,\n",
    "            'alpha': 0.31141671752487043,\n",
    "            'subsample': 0.8779467596981366, \n",
    "            'colsample_bytree': 0.9759532762677546,\n",
    "            'learning_rate': 0.008686087328805853, \n",
    "            'n_estimators': 6988,\n",
    "            'max_depth': 9,\n",
    "            'min_child_weight': 2, \n",
    "            'eta': 3.7603213457541647e-06,\n",
    "            'gamma': 2.1478058456847449e-07,\n",
    "            'grow_policy': 'lossguide'}\n",
    "                \n",
    "\n",
    "#model_xgb = XGBRegressor(\n",
    "       #objective=\"mae\",\n",
    " #   **xgb_params2)\n",
    "\n",
    "numeric_transformer1 = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "numeric_transformer2 = Pipeline(\n",
    "                            steps=[\n",
    "                            #('imputer', SimpleImputer(strategy='median'\n",
    "                             #                         ,add_indicator=True)),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                           # ('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "\n",
    "pipe_xgbr1 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer1),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )\n",
    "pipe_xgbr2 = Pipeline(\n",
    "                    steps=[\n",
    "                        ('preprocessor', numeric_transformer2),\n",
    "                        ('classifier', XGBRegressor(\n",
    "                      #objective=\"mae\",\n",
    "                       **param))\n",
    "                    ]\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c70484e0",
   "metadata": {
    "papermill": {
     "duration": 0.028512,
     "end_time": "2021-12-25T12:37:56.884519",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.856007",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# XGBR regression imputer \n",
    "## Predict income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "767e5289",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:37:56.949478Z",
     "iopub.status.busy": "2021-12-25T12:37:56.948587Z",
     "iopub.status.idle": "2021-12-25T12:40:08.975132Z",
     "shell.execute_reply": "2021-12-25T12:40:08.975756Z",
     "shell.execute_reply.started": "2021-12-25T11:34:00.249718Z"
    },
    "papermill": {
     "duration": 132.06334,
     "end_time": "2021-12-25T12:40:08.975962",
     "exception": false,
     "start_time": "2021-12-25T12:37:56.912622",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 8s, sys: 2.49 s, total: 2min 10s\n",
      "Wall time: 2min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "train=train.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "test=test.drop(['Unnamed: 0','SeriousDlqin2yrs'], axis=1)\n",
    "train_final= pd.concat( [train, test], axis=0) \n",
    "#testdf_income= train_final[train_final['MonthlyIncome'].isnull()==True]\n",
    "traindf_income = train_final[train_final['MonthlyIncome'].isnull()==False]\n",
    "y_income = traindf_income['MonthlyIncome']\n",
    "X_income=traindf_income.drop([\"MonthlyIncome\"],axis=1)\n",
    "pipe_xgbr1.fit(X_income, y_income)\n",
    "train_income_missing=train[train['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "test_income_missing=test[test['MonthlyIncome'].isnull()==True].drop([\"MonthlyIncome\"],axis=1)\n",
    "train_predicted = pipe_xgbr1.predict(train_income_missing)\n",
    "test_predicted = pipe_xgbr1.predict(test_income_missing)\n",
    "train.loc[(train.MonthlyIncome.isnull()), 'MonthlyIncome'] = train_predicted\n",
    "test.loc[(test.MonthlyIncome.isnull()), 'MonthlyIncome'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "65ef564d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:40:09.039245Z",
     "iopub.status.busy": "2021-12-25T12:40:09.038346Z",
     "iopub.status.idle": "2021-12-25T12:40:09.051133Z",
     "shell.execute_reply": "2021-12-25T12:40:09.051785Z",
     "shell.execute_reply.started": "2021-12-25T11:36:16.370593Z"
    },
    "papermill": {
     "duration": 0.04619,
     "end_time": "2021-12-25T12:40:09.051987",
     "exception": false,
     "start_time": "2021-12-25T12:40:09.005797",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    146076 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "117602de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:40:09.121494Z",
     "iopub.status.busy": "2021-12-25T12:40:09.120627Z",
     "iopub.status.idle": "2021-12-25T12:40:09.132002Z",
     "shell.execute_reply": "2021-12-25T12:40:09.132422Z",
     "shell.execute_reply.started": "2021-12-25T11:36:16.395142Z"
    },
    "papermill": {
     "duration": 0.045844,
     "end_time": "2021-12-25T12:40:09.132568",
     "exception": false,
     "start_time": "2021-12-25T12:40:09.086724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    98877 non-null   float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed91c8b3",
   "metadata": {
    "papermill": {
     "duration": 0.029621,
     "end_time": "2021-12-25T12:40:09.195305",
     "exception": false,
     "start_time": "2021-12-25T12:40:09.165684",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict number of dependents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c303f046",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:40:09.259802Z",
     "iopub.status.busy": "2021-12-25T12:40:09.258821Z",
     "iopub.status.idle": "2021-12-25T12:42:45.030204Z",
     "shell.execute_reply": "2021-12-25T12:42:45.030655Z",
     "shell.execute_reply.started": "2021-12-25T11:36:16.422238Z"
    },
    "papermill": {
     "duration": 155.806652,
     "end_time": "2021-12-25T12:42:45.030810",
     "exception": false,
     "start_time": "2021-12-25T12:40:09.224158",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 32s, sys: 950 ms, total: 2min 33s\n",
      "Wall time: 2min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "traindf_NumberOfDependents = train_final[train_final['NumberOfDependents'].isnull()==False]\n",
    "y_NumberOfDependents = traindf_NumberOfDependents['NumberOfDependents']\n",
    "X_NumberOfDependents=traindf_NumberOfDependents.drop([\"NumberOfDependents\"],axis=1)\n",
    "pipe_xgbr2.fit(X_NumberOfDependents, y_NumberOfDependents)\n",
    "train_NumberOfDependents_missing=train[train['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "test_NumberOfDependents_missing=test[test['NumberOfDependents'].isnull()==True].drop([\"NumberOfDependents\"],axis=1)\n",
    "train_predicted = pipe_xgbr2.predict(train_NumberOfDependents_missing)\n",
    "test_predicted = pipe_xgbr2.predict(test_NumberOfDependents_missing)\n",
    "train.loc[(train.NumberOfDependents.isnull()), 'NumberOfDependents'] = train_predicted\n",
    "test.loc[(test.NumberOfDependents.isnull()), 'NumberOfDependents'] = test_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7fad4bd4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:45.097781Z",
     "iopub.status.busy": "2021-12-25T12:42:45.096807Z",
     "iopub.status.idle": "2021-12-25T12:42:45.109329Z",
     "shell.execute_reply": "2021-12-25T12:42:45.109789Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.200486Z"
    },
    "papermill": {
     "duration": 0.047986,
     "end_time": "2021-12-25T12:42:45.109951",
     "exception": false,
     "start_time": "2021-12-25T12:42:45.061965",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  150000 non-null  float16\n",
      " 1   age                                   150000 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 3   DebtRatio                             150000 non-null  float32\n",
      " 4   MonthlyIncome                         150000 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       150000 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               150000 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          150000 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  int8   \n",
      " 9   NumberOfDependents                    150000 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 2.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "965ab381",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:45.179373Z",
     "iopub.status.busy": "2021-12-25T12:42:45.178537Z",
     "iopub.status.idle": "2021-12-25T12:42:45.189347Z",
     "shell.execute_reply": "2021-12-25T12:42:45.189987Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.218444Z"
    },
    "papermill": {
     "duration": 0.045969,
     "end_time": "2021-12-25T12:42:45.190187",
     "exception": false,
     "start_time": "2021-12-25T12:42:45.144218",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 10 columns):\n",
      " #   Column                                Non-Null Count   Dtype  \n",
      "---  ------                                --------------   -----  \n",
      " 0   RevolvingUtilizationOfUnsecuredLines  101503 non-null  float16\n",
      " 1   age                                   101503 non-null  int8   \n",
      " 2   NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 3   DebtRatio                             101503 non-null  float32\n",
      " 4   MonthlyIncome                         101503 non-null  float32\n",
      " 5   NumberOfOpenCreditLinesAndLoans       101503 non-null  int8   \n",
      " 6   NumberOfTimes90DaysLate               101503 non-null  int8   \n",
      " 7   NumberRealEstateLoansOrLines          101503 non-null  int8   \n",
      " 8   NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  int8   \n",
      " 9   NumberOfDependents                    101503 non-null  float16\n",
      "dtypes: float16(2), float32(2), int8(6)\n",
      "memory usage: 1.7 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21103c2b",
   "metadata": {
    "papermill": {
     "duration": 0.030253,
     "end_time": "2021-12-25T12:42:45.254131",
     "exception": false,
     "start_time": "2021-12-25T12:42:45.223878",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Add bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5f819aa6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:45.329797Z",
     "iopub.status.busy": "2021-12-25T12:42:45.328973Z",
     "iopub.status.idle": "2021-12-25T12:42:45.891947Z",
     "shell.execute_reply": "2021-12-25T12:42:45.891455Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.233986Z"
    },
    "papermill": {
     "duration": 0.607165,
     "end_time": "2021-12-25T12:42:45.892082",
     "exception": false,
     "start_time": "2021-12-25T12:42:45.284917",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Add bin data \n",
    "# initializing append_str\n",
    "append_str = 'cat_'\n",
    "# Append suffix / prefix to strings in list\n",
    "num_features1=[\"RevolvingUtilizationOfUnsecuredLines\", \"DebtRatio\",\"MonthlyIncome\"]\n",
    "num_features2=[\"NumberOfDependents\",\n",
    "                       \"NumberOfTime60-89DaysPastDueNotWorse\",\n",
    "                       \"NumberRealEstateLoansOrLines\",\n",
    "                       \"NumberOfTimes90DaysLate\",\n",
    "                       \"NumberOfOpenCreditLinesAndLoans\",\n",
    "                       \"NumberOfTime30-59DaysPastDueNotWorse\",\n",
    "                       \"age\"]\n",
    "cat_features1 = [append_str + sub for sub in num_features1]\n",
    "cat_features2 = [append_str + sub for sub in num_features2]\n",
    "\n",
    "# create the discretizer object with strategy quantile and 1000 bins\n",
    "discretizer1 = KBinsDiscretizer(n_bins=40, encode='ordinal',strategy='quantile')\n",
    "discretizer2 = KBinsDiscretizer(n_bins=4, encode='ordinal',strategy='quantile')\n",
    "\n",
    "pipeline1 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer1)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline1.fit(train.loc[:,num_features1])\n",
    "# apply the discretisation\n",
    "train_cat1 = pipeline1.transform(train.loc[:,num_features1])\n",
    "test_cat1 = pipeline1.transform(test.loc[:,num_features1])\n",
    "train_df1=pd.DataFrame(train_cat1,columns=cat_features1).astype('category')\n",
    "test_df1=pd.DataFrame(test_cat1,columns=cat_features1).astype('category')\n",
    "train_final1= pd.concat( [train.loc[:,num_features1], train_df1], axis=1) \n",
    "test_final1= pd.concat( [test.loc[:,num_features1], test_df1], axis=1) \n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('imputer', SimpleImputer( strategy='median')),\n",
    "        ('bin', discretizer2)\n",
    "    ])\n",
    "# fit the discretizer to the train set\n",
    "pipeline2.fit(train.loc[:,num_features2])\n",
    "# apply the discretisation\n",
    "train_cat2 = pipeline2.transform(train.loc[:,num_features2])\n",
    "test_cat2 = pipeline2.transform(test.loc[:,num_features2])\n",
    "train_df2=pd.DataFrame(train_cat2,columns=cat_features2).astype('category')\n",
    "test_df2=pd.DataFrame(test_cat2,columns=cat_features2).astype('category')\n",
    "train_final2= pd.concat( [train.loc[:,num_features2], train_df2], axis=1) \n",
    "test_final2= pd.concat( [test.loc[:,num_features2], test_df2], axis=1) \n",
    "train_final= pd.concat( [train_final1, train_final2], axis=1) \n",
    "test_final= pd.concat( [test_final1, test_final2], axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dad0416",
   "metadata": {
    "papermill": {
     "duration": 0.030685,
     "end_time": "2021-12-25T12:42:45.954627",
     "exception": false,
     "start_time": "2021-12-25T12:42:45.923942",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final X and y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "750c1081",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.021474Z",
     "iopub.status.busy": "2021-12-25T12:42:46.020821Z",
     "iopub.status.idle": "2021-12-25T12:42:46.023024Z",
     "shell.execute_reply": "2021-12-25T12:42:46.023496Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.829999Z"
    },
    "papermill": {
     "duration": 0.038132,
     "end_time": "2021-12-25T12:42:46.023628",
     "exception": false,
     "start_time": "2021-12-25T12:42:45.985496",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pour le train test\n",
    "target= \"SeriousDlqin2yrs\"\n",
    "X = train_final# axis=1\n",
    "X_test_final =test_final# axis=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e9a41155",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.091488Z",
     "iopub.status.busy": "2021-12-25T12:42:46.090568Z",
     "iopub.status.idle": "2021-12-25T12:42:46.110817Z",
     "shell.execute_reply": "2021-12-25T12:42:46.111364Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.836689Z"
    },
    "papermill": {
     "duration": 0.057908,
     "end_time": "2021-12-25T12:42:46.111524",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.053616",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101503 entries, 0 to 101502\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      101503 non-null  float16 \n",
      " 1   DebtRatio                                 101503 non-null  float32 \n",
      " 2   MonthlyIncome                             101503 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  101503 non-null  category\n",
      " 4   cat_DebtRatio                             101503 non-null  category\n",
      " 5   cat_MonthlyIncome                         101503 non-null  category\n",
      " 6   NumberOfDependents                        101503 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              101503 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   101503 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           101503 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      101503 non-null  int8    \n",
      " 12  age                                       101503 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    101503 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  101503 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          101503 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               101503 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       101503 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  101503 non-null  category\n",
      " 19  cat_age                                   101503 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 2.7 MB\n"
     ]
    }
   ],
   "source": [
    "X_test_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "09fe4c2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.180858Z",
     "iopub.status.busy": "2021-12-25T12:42:46.179985Z",
     "iopub.status.idle": "2021-12-25T12:42:46.203136Z",
     "shell.execute_reply": "2021-12-25T12:42:46.202654Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.862117Z"
    },
    "papermill": {
     "duration": 0.057195,
     "end_time": "2021-12-25T12:42:46.203260",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.146065",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 150000 entries, 0 to 149999\n",
      "Data columns (total 20 columns):\n",
      " #   Column                                    Non-Null Count   Dtype   \n",
      "---  ------                                    --------------   -----   \n",
      " 0   RevolvingUtilizationOfUnsecuredLines      150000 non-null  float16 \n",
      " 1   DebtRatio                                 150000 non-null  float32 \n",
      " 2   MonthlyIncome                             150000 non-null  float32 \n",
      " 3   cat_RevolvingUtilizationOfUnsecuredLines  150000 non-null  category\n",
      " 4   cat_DebtRatio                             150000 non-null  category\n",
      " 5   cat_MonthlyIncome                         150000 non-null  category\n",
      " 6   NumberOfDependents                        150000 non-null  float16 \n",
      " 7   NumberOfTime60-89DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 8   NumberRealEstateLoansOrLines              150000 non-null  int8    \n",
      " 9   NumberOfTimes90DaysLate                   150000 non-null  int8    \n",
      " 10  NumberOfOpenCreditLinesAndLoans           150000 non-null  int8    \n",
      " 11  NumberOfTime30-59DaysPastDueNotWorse      150000 non-null  int8    \n",
      " 12  age                                       150000 non-null  int8    \n",
      " 13  cat_NumberOfDependents                    150000 non-null  category\n",
      " 14  cat_NumberOfTime60-89DaysPastDueNotWorse  150000 non-null  category\n",
      " 15  cat_NumberRealEstateLoansOrLines          150000 non-null  category\n",
      " 16  cat_NumberOfTimes90DaysLate               150000 non-null  category\n",
      " 17  cat_NumberOfOpenCreditLinesAndLoans       150000 non-null  category\n",
      " 18  cat_NumberOfTime30-59DaysPastDueNotWorse  150000 non-null  category\n",
      " 19  cat_age                                   150000 non-null  category\n",
      "dtypes: category(10), float16(2), float32(2), int8(6)\n",
      "memory usage: 4.0 MB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0c8dc483",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.272422Z",
     "iopub.status.busy": "2021-12-25T12:42:46.271626Z",
     "iopub.status.idle": "2021-12-25T12:42:46.273605Z",
     "shell.execute_reply": "2021-12-25T12:42:46.274011Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.887092Z"
    },
    "papermill": {
     "duration": 0.03699,
     "end_time": "2021-12-25T12:42:46.274137",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.237147",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del test \n",
    "del train_final\n",
    "del test_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d859e94a",
   "metadata": {
    "papermill": {
     "duration": 0.030557,
     "end_time": "2021-12-25T12:42:46.335175",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.304618",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extract final cat  and num Features \n",
    "## Cat Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d92e25e0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.405765Z",
     "iopub.status.busy": "2021-12-25T12:42:46.404939Z",
     "iopub.status.idle": "2021-12-25T12:42:46.410231Z",
     "shell.execute_reply": "2021-12-25T12:42:46.410656Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.894028Z"
    },
    "papermill": {
     "duration": 0.044311,
     "end_time": "2021-12-25T12:42:46.410798",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.366487",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select non-numeric columns\n",
    "cat_columns = X.select_dtypes(exclude=['int64','int16','float16','float32','float64','int8']).columns\n",
    "cat_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36322010",
   "metadata": {
    "papermill": {
     "duration": 0.032482,
     "end_time": "2021-12-25T12:42:46.476598",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.444116",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "**Num Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e176fe28",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.548690Z",
     "iopub.status.busy": "2021-12-25T12:42:46.547878Z",
     "iopub.status.idle": "2021-12-25T12:42:46.553928Z",
     "shell.execute_reply": "2021-12-25T12:42:46.554317Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.909597Z"
    },
    "papermill": {
     "duration": 0.045453,
     "end_time": "2021-12-25T12:42:46.554451",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.508998",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select the float columns\n",
    "num_columns = X.select_dtypes(include=['int64','int16','float16','float32','float64','int8']).columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71c8211a",
   "metadata": {
    "papermill": {
     "duration": 0.03155,
     "end_time": "2021-12-25T12:42:46.618161",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.586611",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Define preprocess Pipe \n",
    "\n",
    "1. Features Engineer\n",
    "\n",
    "1. Kmeans\n",
    "\n",
    "1. Sparse\n",
    "\n",
    "1. Poly\n",
    "\n",
    "1. Scaler/Transformer\n",
    "\n",
    "1. imput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "61583a57",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.689822Z",
     "iopub.status.busy": "2021-12-25T12:42:46.689010Z",
     "iopub.status.idle": "2021-12-25T12:42:46.695954Z",
     "shell.execute_reply": "2021-12-25T12:42:46.695530Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.925191Z"
    },
    "papermill": {
     "duration": 0.046108,
     "end_time": "2021-12-25T12:42:46.696056",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.649948",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeaturesEngineer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X=X.copy()\n",
    "        # Calculate some metrics across rows\n",
    "        X[\"num_missing\"]  = X.isnull().sum(axis=1)\n",
    "        X[\"std_row\"] = X.std(axis=1)\n",
    "        X[\"sem_row\"] = X.sem(axis=1)\n",
    "        X[\"abs_sum_row\"] = X.abs().sum(axis=1)\n",
    "        X[\"mean_row\"] = X.mean(axis=1)\n",
    "        X[\"max_row\"]= X.max(axis=1)\n",
    "        X[\"min_row\"]= X.min(axis=1)\n",
    "        X['Weighted_Sum_PastDue'] = 2 * X['NumberOfTime30-59DaysPastDueNotWorse'] + 3 * X['NumberOfTime60-89DaysPastDueNotWorse'] + 6 * X['NumberOfTimes90DaysLate']\n",
    "        X['90days_out_of_TotalPastDue'] = X['NumberOfTimes90DaysLate'] / (X['Weighted_Sum_PastDue'])\n",
    "        X.loc[X['Weighted_Sum_PastDue']==0, '90days_out_of_TotalPastDue'] = 0\n",
    "        X['RemainingLines'] = X['NumberOfOpenCreditLinesAndLoans'] - X['NumberRealEstateLoansOrLines']\n",
    "        X['Loans_vs_Other_Lines'] = X['RemainingLines'] / (1 + X['NumberRealEstateLoansOrLines'])\n",
    "        X['Debt_per_Real_Estate_Loan'] = X['DebtRatio'] / X['NumberRealEstateLoansOrLines']\n",
    "        X.loc[X['NumberRealEstateLoansOrLines']==0, 'Debt_per_Real_Estate_Loan'] = 0\n",
    "        X['Disposable_Income_per_person'] = (X['MonthlyIncome'] - X['DebtRatio']) / (X['NumberOfDependents'] + 1)\n",
    "        X.loc[X['NumberOfDependents']==0, 'Disposable_Income_per_person'] = 0\n",
    "        X['RemainingLines_per_person'] = X['RemainingLines'] / (X['NumberOfDependents'] + 1)\n",
    "        X['NumberRE_X_DebtRatio_X_age'] = X['NumberRealEstateLoansOrLines'] * X['DebtRatio'] / X['age']\n",
    "        X['RevolvingUtilization_per_age'] = X['RevolvingUtilizationOfUnsecuredLines'] / X['age']\n",
    "        X[X==np.inf]=np.nan\n",
    "        X=reduce_mem_usage(X)\n",
    "        return X  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d72105b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.767943Z",
     "iopub.status.busy": "2021-12-25T12:42:46.767092Z",
     "iopub.status.idle": "2021-12-25T12:42:46.769003Z",
     "shell.execute_reply": "2021-12-25T12:42:46.769456Z",
     "shell.execute_reply.started": "2021-12-25T11:38:59.940671Z"
    },
    "papermill": {
     "duration": 0.041705,
     "end_time": "2021-12-25T12:42:46.769584",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.727879",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MiniKmeansTransformerEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, num_clusters = 11, encoder=ce.woe.WOEEncoder()):\n",
    "        self.num_clusters = num_clusters\n",
    "        self.encoder= encoder\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans = MiniBatchKMeans(n_clusters=self.num_clusters, random_state=0)\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        if self.num_clusters > 0:\n",
    "            self.kmeans.fit(X)\n",
    "            preds=self.kmeans.predict(X)\n",
    "            preds=pd.DataFrame(preds, columns=['kmeans']).astype('category')\n",
    "            self.encoder.fit(preds,y)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y=None):\n",
    "        pred_classes = self.kmeans.predict(X)\n",
    "        pred_classes=pd.DataFrame(pred_classes, columns=['kmeans']).astype('category')\n",
    "        pred_encoded = self.encoder.transform(pred_classes)\n",
    "        return np.hstack((X, pred_encoded))\n",
    "        #return pred_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "973ba3b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.836329Z",
     "iopub.status.busy": "2021-12-25T12:42:46.835492Z",
     "iopub.status.idle": "2021-12-25T12:42:46.842969Z",
     "shell.execute_reply": "2021-12-25T12:42:46.843399Z",
     "shell.execute_reply.started": "2021-12-25T11:41:51.590137Z"
    },
    "papermill": {
     "duration": 0.04221,
     "end_time": "2021-12-25T12:42:46.843529",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.801319",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cat pipeline\n",
    "categorical_transformer = Pipeline(\n",
    "                    steps=[\n",
    "                        ('imputer', SimpleImputer(strategy='most_frequent',\n",
    "                                                  fill_value='missing',\n",
    "                                                  add_indicator=True)),\n",
    "                        ('encoder',  ce.woe.WOEEncoder()),#(Numerical Input, Categorical Output)\n",
    "                        #('sparse_features', SparseInteractions(degree=2)),\n",
    "                        #('reducedim',  SelectPercentile( mutual_info_classif, percentile=90))\n",
    "\n",
    "                    ]\n",
    "                    ) \n",
    "#Define vnum pipeline\n",
    "numeric_transformer = Pipeline(\n",
    "                            steps=[\n",
    "                            ('imputer', SimpleImputer(strategy='median'\n",
    "                                                      ,add_indicator=True)),\n",
    "                            #('general_features',FeaturesEngineer()),\n",
    "                            ('scaler', PowerTransformer()),#(Numerical Input, Numerical Output)\n",
    "                            # Create an SelectKBest object to select features with two best ANOVA F-Values\n",
    "                            #The F-value scores examine if, when we group the numerical feature by the target vector, the means for each group are significantly different\n",
    "                            ('polynominal_features', PolynomialFeatures(degree=2)),\n",
    "                            ('kmeans',MiniKmeansTransformerEncoder()),\n",
    "                            #('reducedim',  SelectPercentile(f_classif,percentile=90))\n",
    "                            ]\n",
    "                            )\n",
    "# Features union cat + num \n",
    "# WOE+PowerTransformer\n",
    "preprocessor_woe_powertransformer2 = ColumnTransformer(\n",
    "            transformers=[\n",
    "                ('numerical', numeric_transformer, num_columns),\n",
    "                ('categorical', categorical_transformer, cat_columns)\n",
    "            ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb2853fd",
   "metadata": {
    "papermill": {
     "duration": 0.031822,
     "end_time": "2021-12-25T12:42:46.906707",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.874885",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Second pipe to tune : \n",
    "imputer + transformer+ ploy fe + kmeans fe\n",
    "\n",
    "imputer + encoder =cat feat "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "72a4ec8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:46.977492Z",
     "iopub.status.busy": "2021-12-25T12:42:46.976757Z",
     "iopub.status.idle": "2021-12-25T12:42:47.087250Z",
     "shell.execute_reply": "2021-12-25T12:42:47.087637Z",
     "shell.execute_reply.started": "2021-12-25T11:42:05.581988Z"
    },
    "papermill": {
     "duration": 0.148267,
     "end_time": "2021-12-25T12:42:47.087809",
     "exception": false,
     "start_time": "2021-12-25T12:42:46.939542",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"c48e2d04-92c0-4b61-8d90-9714ad29d1e9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"c48e2d04-92c0-4b61-8d90-9714ad29d1e9\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfT...\n",
       "                                                 ('encoder', WOEEncoder())]),\n",
       "                                 Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object'))])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"5985272e-d88d-4079-94d1-6e94fde5afd3\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"5985272e-d88d-4079-94d1-6e94fde5afd3\">numerical</label><div class=\"sk-toggleable__content\"><pre>Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'NumberRealEstateLoansOrLines', 'NumberOfTimes90DaysLate',\n",
       "       'NumberOfOpenCreditLinesAndLoans',\n",
       "       'NumberOfTime30-59DaysPastDueNotWorse', 'age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"91dfc3f3-ade1-49eb-9942-7de3a7cce98d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"91dfc3f3-ade1-49eb-9942-7de3a7cce98d\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, strategy='median')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"d0ec1e0d-e99e-4e75-a752-78fbdbe4826d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"d0ec1e0d-e99e-4e75-a752-78fbdbe4826d\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ba32bf19-e482-48bb-ad13-d219714b20ba\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ba32bf19-e482-48bb-ad13-d219714b20ba\">PolynomialFeatures</label><div class=\"sk-toggleable__content\"><pre>PolynomialFeatures()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"d974349c-1772-418a-b229-db2396e6436d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"d974349c-1772-418a-b229-db2396e6436d\">kmeans: MiniKmeansTransformerEncoder</label><div class=\"sk-toggleable__content\"><pre>MiniKmeansTransformerEncoder()</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"6a4b00a2-e2ee-4d39-95fb-5c3ca58ecf6a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"6a4b00a2-e2ee-4d39-95fb-5c3ca58ecf6a\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"78211738-335e-49a5-a082-080ad384ac55\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"78211738-335e-49a5-a082-080ad384ac55\">categorical</label><div class=\"sk-toggleable__content\"><pre>Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object')</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"8de221d9-8298-47e9-a922-0fa42c7ef23b\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"8de221d9-8298-47e9-a922-0fa42c7ef23b\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(add_indicator=True, fill_value='missing',\n",
       "              strategy='most_frequent')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4a8f0fe1-424b-4641-a24f-14d31e1c4b08\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4a8f0fe1-424b-4641-a24f-14d31e1c4b08\">WOEEncoder</label><div class=\"sk-toggleable__content\"><pre>WOEEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=True,\n",
       "                                                                strategy='median')),\n",
       "                                                 ('scaler', PowerTransformer()),\n",
       "                                                 ('polynominal_features',\n",
       "                                                  PolynomialFeatures()),\n",
       "                                                 ('kmeans',\n",
       "                                                  MiniKmeansTransformerEncoder())]),\n",
       "                                 Index(['RevolvingUtilizationOfUnsecuredLines', 'DebtRatio', 'MonthlyIncome',\n",
       "       'NumberOfDependents', 'NumberOfT...\n",
       "                                                 ('encoder', WOEEncoder())]),\n",
       "                                 Index(['cat_RevolvingUtilizationOfUnsecuredLines', 'cat_DebtRatio',\n",
       "       'cat_MonthlyIncome', 'cat_NumberOfDependents',\n",
       "       'cat_NumberOfTime60-89DaysPastDueNotWorse',\n",
       "       'cat_NumberRealEstateLoansOrLines', 'cat_NumberOfTimes90DaysLate',\n",
       "       'cat_NumberOfOpenCreditLinesAndLoans',\n",
       "       'cat_NumberOfTime30-59DaysPastDueNotWorse', 'cat_age'],\n",
       "      dtype='object'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "52b3228f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:47.157061Z",
     "iopub.status.busy": "2021-12-25T12:42:47.156550Z",
     "iopub.status.idle": "2021-12-25T12:42:51.904512Z",
     "shell.execute_reply": "2021-12-25T12:42:51.904030Z",
     "shell.execute_reply.started": "2021-12-25T11:42:14.262417Z"
    },
    "papermill": {
     "duration": 4.785135,
     "end_time": "2021-12-25T12:42:51.904648",
     "exception": false,
     "start_time": "2021-12-25T12:42:47.119513",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 77)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor_woe_powertransformer2.fit(X,y)\n",
    "X_pre=preprocessor_woe_powertransformer2.transform(X)\n",
    "X_pre.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bf3f6f5",
   "metadata": {
    "papermill": {
     "duration": 0.032984,
     "end_time": "2021-12-25T12:42:51.972354",
     "exception": false,
     "start_time": "2021-12-25T12:42:51.939370",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## CV Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "28bcbc14",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:52.042695Z",
     "iopub.status.busy": "2021-12-25T12:42:52.042021Z",
     "iopub.status.idle": "2021-12-25T12:42:52.044647Z",
     "shell.execute_reply": "2021-12-25T12:42:52.045038Z",
     "shell.execute_reply.started": "2021-12-25T11:42:23.652751Z"
    },
    "papermill": {
     "duration": 0.040296,
     "end_time": "2021-12-25T12:42:52.045166",
     "exception": false,
     "start_time": "2021-12-25T12:42:52.004870",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StratifiedKFold(n_splits=5, random_state=1, shuffle=True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N_FOLD = 5\n",
    "cross_validation_design = StratifiedKFold( n_splits=N_FOLD,\n",
    "                                           shuffle=True\n",
    "                                        ,random_state=1)\n",
    "cross_validation_design"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17255dfe",
   "metadata": {
    "papermill": {
     "duration": 0.031721,
     "end_time": "2021-12-25T12:42:52.109262",
     "exception": false,
     "start_time": "2021-12-25T12:42:52.077541",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "45648de0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:52.182607Z",
     "iopub.status.busy": "2021-12-25T12:42:52.180832Z",
     "iopub.status.idle": "2021-12-25T12:42:52.184896Z",
     "shell.execute_reply": "2021-12-25T12:42:52.184494Z",
     "shell.execute_reply.started": "2021-12-25T11:43:04.480446Z"
    },
    "papermill": {
     "duration": 0.043567,
     "end_time": "2021-12-25T12:42:52.185006",
     "exception": false,
     "start_time": "2021-12-25T12:42:52.141439",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    " def objective_lgbm_pipe2(trial,X=X_pre,y=y):\n",
    "    X_train,X_test, y_train,  y_test = train_test_split(X, y,stratify=y, \n",
    "                                                       test_size=0.25,\n",
    "                                                      random_state=42,\n",
    "                                                        shuffle=True)    \n",
    "    param = {\n",
    "        \"objective\": \"binary\",\n",
    "        'random_state': 42,\n",
    "         'metric': 'auc',\n",
    "        \"verbosity\": -1,\n",
    "        \"boosting_type\": \"gbdt\",\n",
    "        \"device\": \"gpu\",\n",
    "        \"learning_rate\": trial.suggest_float(\"learning_rate\",0.005 ,0.2),\n",
    "        \"n_estimators\" : trial.suggest_int(\"n_estimators\" , 1000 , 10000),\n",
    "        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n",
    "        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n",
    "        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n",
    "        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n",
    "        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n",
    "        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n",
    "        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n",
    "    }\n",
    "    \n",
    "    model  = lgbm.LGBMClassifier(\n",
    "                       **param)\n",
    "    \n",
    "    model.fit( X_train, y_train,\n",
    "              eval_set=[(X_test, y_test)],\n",
    "              early_stopping_rounds=100,\n",
    "              verbose=False)\n",
    "    preds = model.predict_proba(X_test)[:,1]\n",
    "    final_auc= roc_auc_score(y_true=y_test, y_score=preds)   \n",
    "    return final_auc              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "83cb384d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:42:52.253700Z",
     "iopub.status.busy": "2021-12-25T12:42:52.252933Z",
     "iopub.status.idle": "2021-12-25T12:56:01.496404Z",
     "shell.execute_reply": "2021-12-25T12:56:01.495693Z"
    },
    "papermill": {
     "duration": 789.279728,
     "end_time": "2021-12-25T12:56:01.496583",
     "exception": false,
     "start_time": "2021-12-25T12:42:52.216855",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:42:52,251]\u001b[0m A new study created in memory with name: no-name-96818af5-de78-4b85-8709-6766a9603ed6\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.41755950139669873, subsample=1.0 will be ignored. Current value: bagging_fraction=0.41755950139669873\n",
      "[LightGBM] [Warning] lambda_l1 is set=2.9308480894054424e-05, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.9308480894054424e-05\n",
      "[LightGBM] [Warning] feature_fraction is set=0.839587851377543, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.839587851377543\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=8.911393187494488e-06, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.911393187494488e-06\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:43:08,864]\u001b[0m Trial 0 finished with value: 0.8553185904723103 and parameters: {'learning_rate': 0.16229667967067407, 'n_estimators': 4516, 'lambda_l1': 2.9308480894054424e-05, 'lambda_l2': 8.911393187494488e-06, 'num_leaves': 211, 'feature_fraction': 0.839587851377543, 'bagging_fraction': 0.41755950139669873, 'bagging_freq': 4, 'min_child_samples': 30}. Best is trial 0 with value: 0.8553185904723103.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.4761391364837125, subsample=1.0 will be ignored. Current value: bagging_fraction=0.4761391364837125\n",
      "[LightGBM] [Warning] lambda_l1 is set=3.7401971073664054e-07, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.7401971073664054e-07\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5290207928586279, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5290207928586279\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=4.478411629792775e-05, reg_lambda=0.0 will be ignored. Current value: lambda_l2=4.478411629792775e-05\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:43:16,712]\u001b[0m Trial 1 finished with value: 0.8649302256398669 and parameters: {'learning_rate': 0.06649689432878483, 'n_estimators': 1842, 'lambda_l1': 3.7401971073664054e-07, 'lambda_l2': 4.478411629792775e-05, 'num_leaves': 105, 'feature_fraction': 0.5290207928586279, 'bagging_fraction': 0.4761391364837125, 'bagging_freq': 5, 'min_child_samples': 75}. Best is trial 1 with value: 0.8649302256398669.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.6040479926193344, subsample=1.0 will be ignored. Current value: bagging_fraction=0.6040479926193344\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.013469791355434994, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.013469791355434994\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6415099771032678, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6415099771032678\n",
      "[LightGBM] [Warning] bagging_freq is set=2, subsample_freq=0 will be ignored. Current value: bagging_freq=2\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.8469145541275133, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.8469145541275133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:43:28,120]\u001b[0m Trial 2 finished with value: 0.860822253145574 and parameters: {'learning_rate': 0.12320188409634539, 'n_estimators': 7173, 'lambda_l1': 0.013469791355434994, 'lambda_l2': 0.8469145541275133, 'num_leaves': 196, 'feature_fraction': 0.6415099771032678, 'bagging_fraction': 0.6040479926193344, 'bagging_freq': 2, 'min_child_samples': 51}. Best is trial 1 with value: 0.8649302256398669.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8942817402379518, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8942817402379518\n",
      "[LightGBM] [Warning] lambda_l1 is set=5.491866817550705e-07, reg_alpha=0.0 will be ignored. Current value: lambda_l1=5.491866817550705e-07\n",
      "[LightGBM] [Warning] feature_fraction is set=0.831015724528861, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.831015724528861\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=5.355911428709008e-05, reg_lambda=0.0 will be ignored. Current value: lambda_l2=5.355911428709008e-05\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:43:38,530]\u001b[0m Trial 3 finished with value: 0.8653198945380718 and parameters: {'learning_rate': 0.04670382900254358, 'n_estimators': 3554, 'lambda_l1': 5.491866817550705e-07, 'lambda_l2': 5.355911428709008e-05, 'num_leaves': 90, 'feature_fraction': 0.831015724528861, 'bagging_fraction': 0.8942817402379518, 'bagging_freq': 6, 'min_child_samples': 52}. Best is trial 3 with value: 0.8653198945380718.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.758875721656846, subsample=1.0 will be ignored. Current value: bagging_fraction=0.758875721656846\n",
      "[LightGBM] [Warning] lambda_l1 is set=6.042923948461356, reg_alpha=0.0 will be ignored. Current value: lambda_l1=6.042923948461356\n",
      "[LightGBM] [Warning] feature_fraction is set=0.4867694654867458, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.4867694654867458\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.00015099498016628708, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.00015099498016628708\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:43:47,699]\u001b[0m Trial 4 finished with value: 0.8680495723790936 and parameters: {'learning_rate': 0.055040775179911954, 'n_estimators': 9653, 'lambda_l1': 6.042923948461356, 'lambda_l2': 0.00015099498016628708, 'num_leaves': 32, 'feature_fraction': 0.4867694654867458, 'bagging_fraction': 0.758875721656846, 'bagging_freq': 3, 'min_child_samples': 8}. Best is trial 4 with value: 0.8680495723790936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.897729048982625, subsample=1.0 will be ignored. Current value: bagging_fraction=0.897729048982625\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.002767298918777063, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.002767298918777063\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8742842087039778, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8742842087039778\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.179821437059684e-07, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.179821437059684e-07\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:43:53,923]\u001b[0m Trial 5 finished with value: 0.8627934666807091 and parameters: {'learning_rate': 0.162357552640891, 'n_estimators': 6640, 'lambda_l1': 0.002767298918777063, 'lambda_l2': 1.179821437059684e-07, 'num_leaves': 73, 'feature_fraction': 0.8742842087039778, 'bagging_fraction': 0.897729048982625, 'bagging_freq': 3, 'min_child_samples': 41}. Best is trial 4 with value: 0.8680495723790936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.45475867632577677, subsample=1.0 will be ignored. Current value: bagging_fraction=0.45475867632577677\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.003140650651889292, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.003140650651889292\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5964996570121803, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5964996570121803\n",
      "[LightGBM] [Warning] bagging_freq is set=2, subsample_freq=0 will be ignored. Current value: bagging_freq=2\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.05312558519187435, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.05312558519187435\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:44:02,365]\u001b[0m Trial 6 finished with value: 0.8629879305270026 and parameters: {'learning_rate': 0.10498904846718957, 'n_estimators': 6457, 'lambda_l1': 0.003140650651889292, 'lambda_l2': 0.05312558519187435, 'num_leaves': 124, 'feature_fraction': 0.5964996570121803, 'bagging_fraction': 0.45475867632577677, 'bagging_freq': 2, 'min_child_samples': 87}. Best is trial 4 with value: 0.8680495723790936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8092215152171558, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8092215152171558\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.409272768685833e-05, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.409272768685833e-05\n",
      "[LightGBM] [Warning] feature_fraction is set=0.7583371601760884, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.7583371601760884\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n",
      "[LightGBM] [Warning] lambda_l2 is set=5.259735488715e-08, reg_lambda=0.0 will be ignored. Current value: lambda_l2=5.259735488715e-08\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:44:09,187]\u001b[0m Trial 7 finished with value: 0.8662322160255405 and parameters: {'learning_rate': 0.07312956415491834, 'n_estimators': 7573, 'lambda_l1': 1.409272768685833e-05, 'lambda_l2': 5.259735488715e-08, 'num_leaves': 64, 'feature_fraction': 0.7583371601760884, 'bagging_fraction': 0.8092215152171558, 'bagging_freq': 1, 'min_child_samples': 80}. Best is trial 4 with value: 0.8680495723790936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7893186625925774, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7893186625925774\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.009716612227747227, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.009716612227747227\n",
      "[LightGBM] [Warning] feature_fraction is set=0.44192315561822165, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.44192315561822165\n",
      "[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.7891956828267634e-06, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.7891956828267634e-06\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:44:13,388]\u001b[0m Trial 8 finished with value: 0.8650194839010368 and parameters: {'learning_rate': 0.1447090001497098, 'n_estimators': 1014, 'lambda_l1': 0.009716612227747227, 'lambda_l2': 1.7891956828267634e-06, 'num_leaves': 34, 'feature_fraction': 0.44192315561822165, 'bagging_fraction': 0.7893186625925774, 'bagging_freq': 7, 'min_child_samples': 16}. Best is trial 4 with value: 0.8680495723790936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.719570922676791, subsample=1.0 will be ignored. Current value: bagging_fraction=0.719570922676791\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.02309362833804563, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.02309362833804563\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5439783955150447, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5439783955150447\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.0992667965485176e-08, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.0992667965485176e-08\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:44:23,553]\u001b[0m Trial 9 finished with value: 0.8581743017763255 and parameters: {'learning_rate': 0.15128493563554024, 'n_estimators': 5480, 'lambda_l1': 0.02309362833804563, 'lambda_l2': 1.0992667965485176e-08, 'num_leaves': 250, 'feature_fraction': 0.5439783955150447, 'bagging_fraction': 0.719570922676791, 'bagging_freq': 3, 'min_child_samples': 29}. Best is trial 4 with value: 0.8680495723790936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9985066146105084, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9985066146105084\n",
      "[LightGBM] [Warning] lambda_l1 is set=8.65775300015011, reg_alpha=0.0 will be ignored. Current value: lambda_l1=8.65775300015011\n",
      "[LightGBM] [Warning] feature_fraction is set=0.4270023452065336, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.4270023452065336\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.005112037170091865, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.005112037170091865\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:44:43,571]\u001b[0m Trial 10 finished with value: 0.8687476056207742 and parameters: {'learning_rate': 0.01847739501443172, 'n_estimators': 9862, 'lambda_l1': 8.65775300015011, 'lambda_l2': 0.005112037170091865, 'num_leaves': 17, 'feature_fraction': 0.4270023452065336, 'bagging_fraction': 0.9985066146105084, 'bagging_freq': 5, 'min_child_samples': 6}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.99888190092216, subsample=1.0 will be ignored. Current value: bagging_fraction=0.99888190092216\n",
      "[LightGBM] [Warning] lambda_l1 is set=8.456482297491048, reg_alpha=0.0 will be ignored. Current value: lambda_l1=8.456482297491048\n",
      "[LightGBM] [Warning] feature_fraction is set=0.4048554013950601, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.4048554013950601\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.005114392208060388, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.005114392208060388\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:47:10,122]\u001b[0m Trial 11 finished with value: 0.8667559063026697 and parameters: {'learning_rate': 0.0058805443532921695, 'n_estimators': 9918, 'lambda_l1': 8.456482297491048, 'lambda_l2': 0.005114392208060388, 'num_leaves': 2, 'feature_fraction': 0.4048554013950601, 'bagging_fraction': 0.99888190092216, 'bagging_freq': 5, 'min_child_samples': 6}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.5934807642205197, subsample=1.0 will be ignored. Current value: bagging_fraction=0.5934807642205197\n",
      "[LightGBM] [Warning] lambda_l1 is set=3.6075189696627734, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.6075189696627734\n",
      "[LightGBM] [Warning] feature_fraction is set=0.4810857854770798, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.4810857854770798\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.0011950388735971443, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.0011950388735971443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:03,703]\u001b[0m Trial 12 finished with value: 0.8675040393425556 and parameters: {'learning_rate': 0.0056723314401275, 'n_estimators': 9981, 'lambda_l1': 3.6075189696627734, 'lambda_l2': 0.0011950388735971443, 'num_leaves': 2, 'feature_fraction': 0.4810857854770798, 'bagging_fraction': 0.5934807642205197, 'bagging_freq': 4, 'min_child_samples': 6}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9582380041209908, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9582380041209908\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.1949697989335364, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.1949697989335364\n",
      "[LightGBM] [Warning] feature_fraction is set=0.709866842340363, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.709866842340363\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.03681973671948539, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.03681973671948539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:11,916]\u001b[0m Trial 13 finished with value: 0.8675637634106333 and parameters: {'learning_rate': 0.044381295558678965, 'n_estimators': 8680, 'lambda_l1': 1.1949697989335364, 'lambda_l2': 0.03681973671948539, 'num_leaves': 34, 'feature_fraction': 0.709866842340363, 'bagging_fraction': 0.9582380041209908, 'bagging_freq': 5, 'min_child_samples': 20}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.6389343694245194, subsample=1.0 will be ignored. Current value: bagging_fraction=0.6389343694245194\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.2133806979814166, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.2133806979814166\n",
      "[LightGBM] [Warning] feature_fraction is set=0.4980572406901659, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.4980572406901659\n",
      "[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7\n",
      "[LightGBM] [Warning] lambda_l2 is set=5.7261668556764675, reg_lambda=0.0 will be ignored. Current value: lambda_l2=5.7261668556764675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:24,579]\u001b[0m Trial 14 finished with value: 0.8654681470648645 and parameters: {'learning_rate': 0.03870589548639766, 'n_estimators': 8588, 'lambda_l1': 0.2133806979814166, 'lambda_l2': 5.7261668556764675, 'num_leaves': 159, 'feature_fraction': 0.4980572406901659, 'bagging_fraction': 0.6389343694245194, 'bagging_freq': 7, 'min_child_samples': 100}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8035617236834485, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8035617236834485\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.0458135398497293e-08, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.0458135398497293e-08\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9792101361069964, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9792101361069964\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.00041974312584346145, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.00041974312584346145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:33,113]\u001b[0m Trial 15 finished with value: 0.8654669155232221 and parameters: {'learning_rate': 0.08094799751647519, 'n_estimators': 8464, 'lambda_l1': 1.0458135398497293e-08, 'lambda_l2': 0.00041974312584346145, 'num_leaves': 43, 'feature_fraction': 0.9792101361069964, 'bagging_fraction': 0.8035617236834485, 'bagging_freq': 3, 'min_child_samples': 20}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7197132341569394, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7197132341569394\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.32072077640469154, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.32072077640469154\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6054167087854878, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6054167087854878\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.012827951210641696, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.012827951210641696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:38,409]\u001b[0m Trial 16 finished with value: 0.8641217470595004 and parameters: {'learning_rate': 0.19285525175313312, 'n_estimators': 9184, 'lambda_l1': 0.32072077640469154, 'lambda_l2': 0.012827951210641696, 'num_leaves': 51, 'feature_fraction': 0.6054167087854878, 'bagging_fraction': 0.7197132341569394, 'bagging_freq': 6, 'min_child_samples': 66}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.5397285107158547, subsample=1.0 will be ignored. Current value: bagging_fraction=0.5397285107158547\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.2115226186085879, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.2115226186085879\n",
      "[LightGBM] [Warning] feature_fraction is set=0.41480949106274245, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.41480949106274245\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.35789988449949733, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.35789988449949733\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:49,288]\u001b[0m Trial 17 finished with value: 0.8659502386020707 and parameters: {'learning_rate': 0.02575145725757494, 'n_estimators': 7880, 'lambda_l1': 0.2115226186085879, 'lambda_l2': 0.35789988449949733, 'num_leaves': 144, 'feature_fraction': 0.41480949106274245, 'bagging_fraction': 0.5397285107158547, 'bagging_freq': 1, 'min_child_samples': 37}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8796561357618765, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8796561357618765\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.0002537473831373626, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.0002537473831373626\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5704605203911244, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5704605203911244\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.0014558837557699148, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.0014558837557699148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:49:54,729]\u001b[0m Trial 18 finished with value: 0.8667484315290899 and parameters: {'learning_rate': 0.09925442586049178, 'n_estimators': 5568, 'lambda_l1': 0.0002537473831373626, 'lambda_l2': 0.0014558837557699148, 'num_leaves': 23, 'feature_fraction': 0.5704605203911244, 'bagging_fraction': 0.8796561357618765, 'bagging_freq': 4, 'min_child_samples': 11}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.6678636435039981, subsample=1.0 will be ignored. Current value: bagging_fraction=0.6678636435039981\n",
      "[LightGBM] [Warning] lambda_l1 is set=9.417212163420217, reg_alpha=0.0 will be ignored. Current value: lambda_l1=9.417212163420217\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6536163750924682, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6536163750924682\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=7.095430167272056e-05, reg_lambda=0.0 will be ignored. Current value: lambda_l2=7.095430167272056e-05\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:50:10,286]\u001b[0m Trial 19 finished with value: 0.8670071750072216 and parameters: {'learning_rate': 0.02701477412075944, 'n_estimators': 3458, 'lambda_l1': 9.417212163420217, 'lambda_l2': 7.095430167272056e-05, 'num_leaves': 89, 'feature_fraction': 0.6536163750924682, 'bagging_fraction': 0.6678636435039981, 'bagging_freq': 6, 'min_child_samples': 26}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9518625781267105, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9518625781267105\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.1031705342377904, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.1031705342377904\n",
      "[LightGBM] [Warning] feature_fraction is set=0.47119642291771363, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.47119642291771363\n",
      "[LightGBM] [Warning] bagging_freq is set=2, subsample_freq=0 will be ignored. Current value: bagging_freq=2\n",
      "[LightGBM] [Warning] lambda_l2 is set=5.90794513544493e-07, reg_lambda=0.0 will be ignored. Current value: lambda_l2=5.90794513544493e-07\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:50:20,538]\u001b[0m Trial 20 finished with value: 0.8645300259203026 and parameters: {'learning_rate': 0.0637214506184792, 'n_estimators': 9468, 'lambda_l1': 0.1031705342377904, 'lambda_l2': 5.90794513544493e-07, 'num_leaves': 171, 'feature_fraction': 0.47119642291771363, 'bagging_fraction': 0.9518625781267105, 'bagging_freq': 2, 'min_child_samples': 39}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9981657712851278, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9981657712851278\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.260299920857668, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.260299920857668\n",
      "[LightGBM] [Warning] feature_fraction is set=0.7308435329194694, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.7308435329194694\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.04419304100444219, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.04419304100444219\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:50:27,918]\u001b[0m Trial 21 finished with value: 0.8675352098895895 and parameters: {'learning_rate': 0.046892053755677905, 'n_estimators': 8544, 'lambda_l1': 1.260299920857668, 'lambda_l2': 0.04419304100444219, 'num_leaves': 25, 'feature_fraction': 0.7308435329194694, 'bagging_fraction': 0.9981657712851278, 'bagging_freq': 5, 'min_child_samples': 18}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9367569508670197, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9367569508670197\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.2732111464427402, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.2732111464427402\n",
      "[LightGBM] [Warning] feature_fraction is set=0.6957821835960727, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.6957821835960727\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.15886327052157703, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.15886327052157703\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:50:42,168]\u001b[0m Trial 22 finished with value: 0.868234012844797 and parameters: {'learning_rate': 0.026487489280766902, 'n_estimators': 9335, 'lambda_l1': 1.2732111464427402, 'lambda_l2': 0.15886327052157703, 'num_leaves': 16, 'feature_fraction': 0.6957821835960727, 'bagging_fraction': 0.9367569508670197, 'bagging_freq': 5, 'min_child_samples': 5}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8553854934646004, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8553854934646004\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.3192783943590503, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.3192783943590503\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9451399458526056, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9451399458526056\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.18490498326786634, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.18490498326786634\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:50:55,320]\u001b[0m Trial 23 finished with value: 0.8682849336707636 and parameters: {'learning_rate': 0.02205146477165497, 'n_estimators': 9850, 'lambda_l1': 1.3192783943590503, 'lambda_l2': 0.18490498326786634, 'num_leaves': 14, 'feature_fraction': 0.9451399458526056, 'bagging_fraction': 0.8553854934646004, 'bagging_freq': 4, 'min_child_samples': 5}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8589165730494213, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8589165730494213\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.832000612096529, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.832000612096529\n",
      "[LightGBM] [Warning] feature_fraction is set=0.99539965369363, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.99539965369363\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.3445988856855822, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.3445988856855822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:51:23,427]\u001b[0m Trial 24 finished with value: 0.8685979391017253 and parameters: {'learning_rate': 0.022544600375267478, 'n_estimators': 9048, 'lambda_l1': 0.832000612096529, 'lambda_l2': 0.3445988856855822, 'num_leaves': 5, 'feature_fraction': 0.99539965369363, 'bagging_fraction': 0.8589165730494213, 'bagging_freq': 4, 'min_child_samples': 13}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8535203632451849, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8535203632451849\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.04160590226659046, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.04160590226659046\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9986998225285112, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9986998225285112\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=8.43388114604967, reg_lambda=0.0 will be ignored. Current value: lambda_l2=8.43388114604967\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:51:43,069]\u001b[0m Trial 25 finished with value: 0.8668623833405075 and parameters: {'learning_rate': 0.016785805585396857, 'n_estimators': 8072, 'lambda_l1': 0.04160590226659046, 'lambda_l2': 8.43388114604967, 'num_leaves': 59, 'feature_fraction': 0.9986998225285112, 'bagging_fraction': 0.8535203632451849, 'bagging_freq': 4, 'min_child_samples': 15}. Best is trial 10 with value: 0.8687476056207742.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8374510810346636, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8374510810346636\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.7144428626069527, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.7144428626069527\n",
      "[LightGBM] [Warning] feature_fraction is set=0.93215730134758, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.93215730134758\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.5270834245797604, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.5270834245797604\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:51:54,051]\u001b[0m Trial 26 finished with value: 0.8693215040261605 and parameters: {'learning_rate': 0.08800847932934244, 'n_estimators': 9037, 'lambda_l1': 0.7144428626069527, 'lambda_l2': 1.5270834245797604, 'num_leaves': 3, 'feature_fraction': 0.93215730134758, 'bagging_fraction': 0.8374510810346636, 'bagging_freq': 4, 'min_child_samples': 25}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8271825482025156, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8271825482025156\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.00046394105132617193, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.00046394105132617193\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9086963205411456, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9086963205411456\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.6828174641673666, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.6828174641673666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:52:06,727]\u001b[0m Trial 27 finished with value: 0.8691895352166402 and parameters: {'learning_rate': 0.08768668014056177, 'n_estimators': 7097, 'lambda_l1': 0.00046394105132617193, 'lambda_l2': 1.6828174641673666, 'num_leaves': 3, 'feature_fraction': 0.9086963205411456, 'bagging_fraction': 0.8271825482025156, 'bagging_freq': 6, 'min_child_samples': 26}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7515958237061474, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7515958237061474\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.0005528309501972623, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.0005528309501972623\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9107718315588751, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9107718315588751\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=3.0788554362039737, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.0788554362039737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:52:14,728]\u001b[0m Trial 28 finished with value: 0.8668419944844266 and parameters: {'learning_rate': 0.08842243705749245, 'n_estimators': 7154, 'lambda_l1': 0.0005528309501972623, 'lambda_l2': 3.0788554362039737, 'num_leaves': 48, 'feature_fraction': 0.9107718315588751, 'bagging_fraction': 0.7515958237061474, 'bagging_freq': 6, 'min_child_samples': 60}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8236775927364557, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8236775927364557\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.9136267301333162e-05, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.9136267301333162e-05\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8061107584298813, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8061107584298813\n",
      "[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.2267329092432617, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.2267329092432617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:52:22,043]\u001b[0m Trial 29 finished with value: 0.8648165418027882 and parameters: {'learning_rate': 0.1191936490932051, 'n_estimators': 5835, 'lambda_l1': 1.9136267301333162e-05, 'lambda_l2': 1.2267329092432617, 'num_leaves': 89, 'feature_fraction': 0.8061107584298813, 'bagging_fraction': 0.8236775927364557, 'bagging_freq': 7, 'min_child_samples': 34}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9190172180149009, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9190172180149009\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.0001493216057509871, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.0001493216057509871\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8887618988549013, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8887618988549013\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.0088750807407096, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.0088750807407096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:52:34,290]\u001b[0m Trial 30 finished with value: 0.8583290541062312 and parameters: {'learning_rate': 0.10993428775602881, 'n_estimators': 4707, 'lambda_l1': 0.0001493216057509871, 'lambda_l2': 0.0088750807407096, 'num_leaves': 254, 'feature_fraction': 0.8887618988549013, 'bagging_fraction': 0.9190172180149009, 'bagging_freq': 6, 'min_child_samples': 27}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8542981627196747, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8542981627196747\n",
      "[LightGBM] [Warning] lambda_l1 is set=2.9733486402506424e-06, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.9733486402506424e-06\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9368175394656676, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9368175394656676\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.1613333126430299, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.1613333126430299\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:52:47,118]\u001b[0m Trial 31 finished with value: 0.8691868782795783 and parameters: {'learning_rate': 0.08480132416876517, 'n_estimators': 9034, 'lambda_l1': 2.9733486402506424e-06, 'lambda_l2': 1.1613333126430299, 'num_leaves': 3, 'feature_fraction': 0.9368175394656676, 'bagging_fraction': 0.8542981627196747, 'bagging_freq': 5, 'min_child_samples': 45}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7710260886628589, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7710260886628589\n",
      "[LightGBM] [Warning] lambda_l1 is set=2.712232342109723e-06, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.712232342109723e-06\n",
      "[LightGBM] [Warning] feature_fraction is set=0.929902289914794, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.929902289914794\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.5268751262923677, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.5268751262923677\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:52:52,516]\u001b[0m Trial 32 finished with value: 0.8680089999238724 and parameters: {'learning_rate': 0.08453641905523034, 'n_estimators': 7950, 'lambda_l1': 2.712232342109723e-06, 'lambda_l2': 1.5268751262923677, 'num_leaves': 17, 'feature_fraction': 0.929902289914794, 'bagging_fraction': 0.7710260886628589, 'bagging_freq': 5, 'min_child_samples': 46}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8411427137787411, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8411427137787411\n",
      "[LightGBM] [Warning] lambda_l1 is set=1.7715820221914053e-08, reg_alpha=0.0 will be ignored. Current value: lambda_l1=1.7715820221914053e-08\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8548797458273656, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8548797458273656\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=9.487103597916912, reg_lambda=0.0 will be ignored. Current value: lambda_l2=9.487103597916912\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:00,227]\u001b[0m Trial 33 finished with value: 0.8653461731280259 and parameters: {'learning_rate': 0.1333528988458611, 'n_estimators': 8986, 'lambda_l1': 1.7715820221914053e-08, 'lambda_l2': 9.487103597916912, 'num_leaves': 70, 'feature_fraction': 0.8548797458273656, 'bagging_fraction': 0.8411427137787411, 'bagging_freq': 5, 'min_child_samples': 47}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9716683279656765, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9716683279656765\n",
      "[LightGBM] [Warning] lambda_l1 is set=6.792143760890717e-08, reg_alpha=0.0 will be ignored. Current value: lambda_l1=6.792143760890717e-08\n",
      "[LightGBM] [Warning] feature_fraction is set=0.7830636450167094, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.7830636450167094\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.0370382896415133, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.0370382896415133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:06,006]\u001b[0m Trial 34 finished with value: 0.8671496575333563 and parameters: {'learning_rate': 0.09735933913975639, 'n_estimators': 7472, 'lambda_l1': 6.792143760890717e-08, 'lambda_l2': 1.0370382896415133, 'num_leaves': 43, 'feature_fraction': 0.7830636450167094, 'bagging_fraction': 0.9716683279656765, 'bagging_freq': 6, 'min_child_samples': 24}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.9011557704023838, subsample=1.0 will be ignored. Current value: bagging_fraction=0.9011557704023838\n",
      "[LightGBM] [Warning] lambda_l1 is set=4.623146313186787e-06, reg_alpha=0.0 will be ignored. Current value: lambda_l1=4.623146313186787e-06\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9685337417967284, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9685337417967284\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.13347098681839556, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.13347098681839556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:14,900]\u001b[0m Trial 35 finished with value: 0.8634803647333728 and parameters: {'learning_rate': 0.06782148032380725, 'n_estimators': 6920, 'lambda_l1': 4.623146313186787e-06, 'lambda_l2': 0.13347098681839556, 'num_leaves': 107, 'feature_fraction': 0.9685337417967284, 'bagging_fraction': 0.9011557704023838, 'bagging_freq': 6, 'min_child_samples': 59}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.6922492624700094, subsample=1.0 will be ignored. Current value: bagging_fraction=0.6922492624700094\n",
      "[LightGBM] [Warning] lambda_l1 is set=7.935783047897483e-05, reg_alpha=0.0 will be ignored. Current value: lambda_l1=7.935783047897483e-05\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8362608260558095, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8362608260558095\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=2.618718625101654, reg_lambda=0.0 will be ignored. Current value: lambda_l2=2.618718625101654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:27,348]\u001b[0m Trial 36 finished with value: 0.8622419925960628 and parameters: {'learning_rate': 0.11902239928516181, 'n_estimators': 7988, 'lambda_l1': 7.935783047897483e-05, 'lambda_l2': 2.618718625101654, 'num_leaves': 224, 'feature_fraction': 0.8362608260558095, 'bagging_fraction': 0.6922492624700094, 'bagging_freq': 5, 'min_child_samples': 31}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7492443927714812, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7492443927714812\n",
      "[LightGBM] [Warning] lambda_l1 is set=7.877524536693637e-07, reg_alpha=0.0 will be ignored. Current value: lambda_l1=7.877524536693637e-07\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9058220472870859, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9058220472870859\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=7.90326117637413e-06, reg_lambda=0.0 will be ignored. Current value: lambda_l2=7.90326117637413e-06\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:34,751]\u001b[0m Trial 37 finished with value: 0.8673882630250012 and parameters: {'learning_rate': 0.05839516493205014, 'n_estimators': 6121, 'lambda_l1': 7.877524536693637e-07, 'lambda_l2': 7.90326117637413e-06, 'num_leaves': 29, 'feature_fraction': 0.9058220472870859, 'bagging_fraction': 0.7492443927714812, 'bagging_freq': 4, 'min_child_samples': 44}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.91950578394427, subsample=1.0 will be ignored. Current value: bagging_fraction=0.91950578394427\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.0010150445547398362, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.0010150445547398362\n",
      "[LightGBM] [Warning] feature_fraction is set=0.947092783644885, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.947092783644885\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.581492370107802, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.581492370107802\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:41,176]\u001b[0m Trial 38 finished with value: 0.8681273704610906 and parameters: {'learning_rate': 0.07708042324493775, 'n_estimators': 4854, 'lambda_l1': 0.0010150445547398362, 'lambda_l2': 0.581492370107802, 'num_leaves': 13, 'feature_fraction': 0.947092783644885, 'bagging_fraction': 0.91950578394427, 'bagging_freq': 5, 'min_child_samples': 54}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8833979798962921, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8833979798962921\n",
      "[LightGBM] [Warning] lambda_l1 is set=2.5970541227787553e-07, reg_alpha=0.0 will be ignored. Current value: lambda_l1=2.5970541227787553e-07\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8088833629729484, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8088833629729484\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.02022946107787824, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.02022946107787824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:48,060]\u001b[0m Trial 39 finished with value: 0.8643979031680771 and parameters: {'learning_rate': 0.09605679744935697, 'n_estimators': 9468, 'lambda_l1': 2.5970541227787553e-07, 'lambda_l2': 0.02022946107787824, 'num_leaves': 76, 'feature_fraction': 0.8088833629729484, 'bagging_fraction': 0.8833979798962921, 'bagging_freq': 3, 'min_child_samples': 22}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.4232446334278155, subsample=1.0 will be ignored. Current value: bagging_fraction=0.4232446334278155\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.005487583758868067, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.005487583758868067\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8749328731919259, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8749328731919259\n",
      "[LightGBM] [Warning] bagging_freq is set=7, subsample_freq=0 will be ignored. Current value: bagging_freq=7\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.0032969439630061723, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.0032969439630061723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:53:55,975]\u001b[0m Trial 40 finished with value: 0.8587105070252381 and parameters: {'learning_rate': 0.17827591814201096, 'n_estimators': 2681, 'lambda_l1': 0.005487583758868067, 'lambda_l2': 0.0032969439630061723, 'num_leaves': 111, 'feature_fraction': 0.8749328731919259, 'bagging_fraction': 0.4232446334278155, 'bagging_freq': 7, 'min_child_samples': 34}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8339989714782389, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8339989714782389\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.050912557627545166, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.050912557627545166\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9994084353883477, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9994084353883477\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.4042635849082133, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.4042635849082133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:54:13,810]\u001b[0m Trial 41 finished with value: 0.8690102318760288 and parameters: {'learning_rate': 0.05377238160821379, 'n_estimators': 8921, 'lambda_l1': 0.050912557627545166, 'lambda_l2': 0.4042635849082133, 'num_leaves': 3, 'feature_fraction': 0.9994084353883477, 'bagging_fraction': 0.8339989714782389, 'bagging_freq': 4, 'min_child_samples': 12}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8230911984769591, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8230911984769591\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.048931453973002116, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.048931453973002116\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9567752871830886, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9567752871830886\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.1200687391148458, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.1200687391148458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:54:39,300]\u001b[0m Trial 42 finished with value: 0.8678329407832358 and parameters: {'learning_rate': 0.05569346641002354, 'n_estimators': 8799, 'lambda_l1': 0.048931453973002116, 'lambda_l2': 0.1200687391148458, 'num_leaves': 2, 'feature_fraction': 0.9567752871830886, 'bagging_fraction': 0.8230911984769591, 'bagging_freq': 3, 'min_child_samples': 10}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7734748240360396, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7734748240360396\n",
      "[LightGBM] [Warning] lambda_l1 is set=3.350005205941339, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.350005205941339\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9371019607939282, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9371019607939282\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=3.620148996662062, reg_lambda=0.0 will be ignored. Current value: lambda_l2=3.620148996662062\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:54:45,906]\u001b[0m Trial 43 finished with value: 0.8673598235355908 and parameters: {'learning_rate': 0.10880962727328734, 'n_estimators': 8364, 'lambda_l1': 3.350005205941339, 'lambda_l2': 3.620148996662062, 'num_leaves': 35, 'feature_fraction': 0.9371019607939282, 'bagging_fraction': 0.7734748240360396, 'bagging_freq': 4, 'min_child_samples': 12}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7983769205030007, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7983769205030007\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.002103736496089536, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.002103736496089536\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9143920727303106, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9143920727303106\n",
      "[LightGBM] [Warning] bagging_freq is set=4, subsample_freq=0 will be ignored. Current value: bagging_freq=4\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.8276962470541366, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.8276962470541366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:54:52,005]\u001b[0m Trial 44 finished with value: 0.8676462139832795 and parameters: {'learning_rate': 0.07302186892741758, 'n_estimators': 7484, 'lambda_l1': 0.002103736496089536, 'lambda_l2': 0.8276962470541366, 'num_leaves': 15, 'feature_fraction': 0.9143920727303106, 'bagging_fraction': 0.7983769205030007, 'bagging_freq': 4, 'min_child_samples': 16}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8908926977970332, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8908926977970332\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.012962196264894553, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.012962196264894553\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8585174477643465, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8585174477643465\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.10290075157316898, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.10290075157316898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:54:58,296]\u001b[0m Trial 45 finished with value: 0.866872326898954 and parameters: {'learning_rate': 0.09099382257369257, 'n_estimators': 9594, 'lambda_l1': 0.012962196264894553, 'lambda_l2': 0.10290075157316898, 'num_leaves': 24, 'feature_fraction': 0.8585174477643465, 'bagging_fraction': 0.8908926977970332, 'bagging_freq': 5, 'min_child_samples': 23}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7399820440870808, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7399820440870808\n",
      "[LightGBM] [Warning] lambda_l1 is set=4.6910181391155836e-05, reg_alpha=0.0 will be ignored. Current value: lambda_l1=4.6910181391155836e-05\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9736008091696561, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9736008091696561\n",
      "[LightGBM] [Warning] bagging_freq is set=6, subsample_freq=0 will be ignored. Current value: bagging_freq=6\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.4116533218187526, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.4116533218187526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:55:08,901]\u001b[0m Trial 46 finished with value: 0.8663308362838258 and parameters: {'learning_rate': 0.038067883216233236, 'n_estimators': 6561, 'lambda_l1': 4.6910181391155836e-05, 'lambda_l2': 0.4116533218187526, 'num_leaves': 59, 'feature_fraction': 0.9736008091696561, 'bagging_fraction': 0.7399820440870808, 'bagging_freq': 6, 'min_child_samples': 30}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.8310220048688545, subsample=1.0 will be ignored. Current value: bagging_fraction=0.8310220048688545\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.3021019266392931, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.3021019266392931\n",
      "[LightGBM] [Warning] feature_fraction is set=0.9076650378860714, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.9076650378860714\n",
      "[LightGBM] [Warning] bagging_freq is set=5, subsample_freq=0 will be ignored. Current value: bagging_freq=5\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.0418858639112402, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.0418858639112402\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:55:15,696]\u001b[0m Trial 47 finished with value: 0.8665122948223116 and parameters: {'learning_rate': 0.06533897134209168, 'n_estimators': 9984, 'lambda_l1': 0.3021019266392931, 'lambda_l2': 0.0418858639112402, 'num_leaves': 38, 'feature_fraction': 0.9076650378860714, 'bagging_fraction': 0.8310220048688545, 'bagging_freq': 5, 'min_child_samples': 9}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.7879198000332519, subsample=1.0 will be ignored. Current value: bagging_fraction=0.7879198000332519\n",
      "[LightGBM] [Warning] lambda_l1 is set=3.479995976601658, reg_alpha=0.0 will be ignored. Current value: lambda_l1=3.479995976601658\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8767872760460791, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.8767872760460791\n",
      "[LightGBM] [Warning] bagging_freq is set=2, subsample_freq=0 will be ignored. Current value: bagging_freq=2\n",
      "[LightGBM] [Warning] lambda_l2 is set=1.8061855971853913, reg_lambda=0.0 will be ignored. Current value: lambda_l2=1.8061855971853913\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:55:29,920]\u001b[0m Trial 48 finished with value: 0.8685699386341045 and parameters: {'learning_rate': 0.05020062365782317, 'n_estimators': 8258, 'lambda_l1': 3.479995976601658, 'lambda_l2': 1.8061855971853913, 'num_leaves': 9, 'feature_fraction': 0.8767872760460791, 'bagging_fraction': 0.7879198000332519, 'bagging_freq': 2, 'min_child_samples': 73}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_fraction is set=0.5987787874307335, subsample=1.0 will be ignored. Current value: bagging_fraction=0.5987787874307335\n",
      "[LightGBM] [Warning] lambda_l1 is set=0.6024015633482548, reg_alpha=0.0 will be ignored. Current value: lambda_l1=0.6024015633482548\n",
      "[LightGBM] [Warning] feature_fraction is set=0.7576243732252947, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.7576243732252947\n",
      "[LightGBM] [Warning] bagging_freq is set=3, subsample_freq=0 will be ignored. Current value: bagging_freq=3\n",
      "[LightGBM] [Warning] lambda_l2 is set=0.000490744228538652, reg_lambda=0.0 will be ignored. Current value: lambda_l2=0.000490744228538652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-12-25 12:56:01,390]\u001b[0m Trial 49 finished with value: 0.8682409117586273 and parameters: {'learning_rate': 0.034197375961917534, 'n_estimators': 8921, 'lambda_l1': 0.6024015633482548, 'lambda_l2': 0.000490744228538652, 'num_leaves': 2, 'feature_fraction': 0.7576243732252947, 'bagging_fraction': 0.5987787874307335, 'bagging_freq': 3, 'min_child_samples': 20}. Best is trial 26 with value: 0.8693215040261605.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23min 53s, sys: 47.2 s, total: 24min 40s\n",
      "Wall time: 13min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "study_lgbm_pipe2 = optuna.create_study(direction=\"maximize\")\n",
    "study_lgbm_pipe2.optimize(objective_lgbm_pipe2 ,n_trials=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b620c054",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-12-25T12:56:01.706916Z",
     "iopub.status.busy": "2021-12-25T12:56:01.706135Z",
     "iopub.status.idle": "2021-12-25T12:56:01.720784Z",
     "shell.execute_reply": "2021-12-25T12:56:01.721762Z"
    },
    "papermill": {
     "duration": 0.127123,
     "end_time": "2021-12-25T12:56:01.721949",
     "exception": false,
     "start_time": "2021-12-25T12:56:01.594826",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of finished trials :  50\n",
      "Best Trials :  {'learning_rate': 0.08800847932934244, 'n_estimators': 9037, 'lambda_l1': 0.7144428626069527, 'lambda_l2': 1.5270834245797604, 'num_leaves': 3, 'feature_fraction': 0.93215730134758, 'bagging_fraction': 0.8374510810346636, 'bagging_freq': 4, 'min_child_samples': 25}\n",
      "Best Values :  0.8693215040261605\n"
     ]
    }
   ],
   "source": [
    "print(\"Numbers of finished trials : \" , len(study_lgbm_pipe2.trials))\n",
    "print(\"Best Trials : \", study_lgbm_pipe2.best_trial.params)\n",
    "print(\"Best Values : \" , study_lgbm_pipe2.best_value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1099.55372,
   "end_time": "2021-12-25T12:56:03.336881",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-12-25T12:37:43.783161",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}