lgbm-multilabel_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-02-08T02:20:11.370201Z",
     "iopub.status.busy": "2021-02-08T02:20:11.369463Z",
     "iopub.status.idle": "2021-02-08T02:20:11.372512Z",
     "shell.execute_reply": "2021-02-08T02:20:11.371963Z"
    },
    "papermill": {
     "duration": 0.0147,
     "end_time": "2021-02-08T02:20:11.372672",
     "exception": false,
     "start_time": "2021-02-08T02:20:11.357972",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:20:11.389280Z",
     "iopub.status.busy": "2021-02-08T02:20:11.388735Z",
     "iopub.status.idle": "2021-02-08T02:20:12.296466Z",
     "shell.execute_reply": "2021-02-08T02:20:12.295938Z"
    },
    "papermill": {
     "duration": 0.91744,
     "end_time": "2021-02-08T02:20:12.296595",
     "exception": false,
     "start_time": "2021-02-08T02:20:11.379155",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from random import choices\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:20:12.316500Z",
     "iopub.status.busy": "2021-02-08T02:20:12.316003Z",
     "iopub.status.idle": "2021-02-08T02:22:28.207630Z",
     "shell.execute_reply": "2021-02-08T02:22:28.206536Z"
    },
    "papermill": {
     "duration": 135.903937,
     "end_time": "2021-02-08T02:22:28.207791",
     "exception": false,
     "start_time": "2021-02-08T02:20:12.303854",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('../input/jane-street-market-prediction/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:22:28.230686Z",
     "iopub.status.busy": "2021-02-08T02:22:28.229333Z",
     "iopub.status.idle": "2021-02-08T02:22:33.512929Z",
     "shell.execute_reply": "2021-02-08T02:22:33.511691Z"
    },
    "papermill": {
     "duration": 5.297681,
     "end_time": "2021-02-08T02:22:33.513125",
     "exception": false,
     "start_time": "2021-02-08T02:22:28.215444",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = train.query('date > 85').reset_index(drop = True) \n",
    "train = train[train['weight'] != 0]\n",
    "train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64').columns}) #limit memory use\n",
    "train['feature_stock_id_sum'] = train['feature_41'] + train['feature_42'] + train['feature_43']\n",
    "train['feature_1_2_cross'] = train['feature_1']/(train['feature_2']+1e-5)\n",
    "train.fillna(-9999,inplace=True)\n",
    "\n",
    "\n",
    "features = [c for c in train.columns if \"feature\" in c]\n",
    "\n",
    "resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:22:33.549201Z",
     "iopub.status.busy": "2021-02-08T02:22:33.548433Z",
     "iopub.status.idle": "2021-02-08T02:22:34.290355Z",
     "shell.execute_reply": "2021-02-08T02:22:34.291408Z"
    },
    "papermill": {
     "duration": 0.766516,
     "end_time": "2021-02-08T02:22:34.291785",
     "exception": false,
     "start_time": "2021-02-08T02:22:33.525269",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1335359, 140)\n",
      "(236056, 140)\n"
     ]
    }
   ],
   "source": [
    "valid_df = train.loc[(train.date >= 450) & (train.date < 500)].reset_index(drop=True)\n",
    "train_df= train.loc[train.date < 450].reset_index(drop=True)\n",
    "print(train_df.shape)\n",
    "print(valid_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:22:34.517252Z",
     "iopub.status.busy": "2021-02-08T02:22:34.516136Z",
     "iopub.status.idle": "2021-02-08T02:22:34.989214Z",
     "shell.execute_reply": "2021-02-08T02:22:34.988135Z"
    },
    "papermill": {
     "duration": 0.689654,
     "end_time": "2021-02-08T02:22:34.989358",
     "exception": false,
     "start_time": "2021-02-08T02:22:34.299704",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train = train_df[features].values\n",
    "\n",
    "y_train= np.stack([(train_df[c] > 0).astype('int') for c in resp_cols]).T\n",
    "\n",
    "X_valid = valid_df[features].values\n",
    "\n",
    "y_valid= np.stack([(valid_df[c] > 0).astype('int') for c in resp_cols]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:22:35.012250Z",
     "iopub.status.busy": "2021-02-08T02:22:35.011610Z",
     "iopub.status.idle": "2021-02-08T02:28:48.110034Z",
     "shell.execute_reply": "2021-02-08T02:28:48.110734Z"
    },
    "papermill": {
     "duration": 373.113728,
     "end_time": "2021-02-08T02:28:48.110941",
     "exception": false,
     "start_time": "2021-02-08T02:22:34.997213",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 672284, number of negative: 663075\n",
      "[LightGBM] [Info] This is the GPU trainer!!\n",
      "[LightGBM] [Info] Total Bins 33408\n",
      "[LightGBM] [Info] Number of data points in the train set: 1335359, number of used features: 132\n",
      "[LightGBM] [Info] Using GPU Device: Tesla P100-PCIE-16GB, Vendor: NVIDIA Corporation\n",
      "[LightGBM] [Info] Compiling OpenCL Kernel with 256 bins...\n",
      "[LightGBM] [Info] GPU programs have been built\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 132 dense feature groups (168.10 MB) transferred to GPU in 0.196455 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.503448 -> initscore=0.013793\n",
      "[LightGBM] [Info] Start training from score 0.013793\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[50]\ttraining's auc: 0.621464\tvalid_1's auc: 0.581489\n",
      "[100]\ttraining's auc: 0.649934\tvalid_1's auc: 0.584025\n",
      "[150]\ttraining's auc: 0.672457\tvalid_1's auc: 0.584912\n",
      "[200]\ttraining's auc: 0.691887\tvalid_1's auc: 0.584729\n",
      "Early stopping, best iteration is:\n",
      "[168]\ttraining's auc: 0.680097\tvalid_1's auc: 0.585028\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 1/5 [01:50<07:22, 110.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 675016, number of negative: 660343\n",
      "[LightGBM] [Info] This is the GPU trainer!!\n",
      "[LightGBM] [Info] Total Bins 33408\n",
      "[LightGBM] [Info] Number of data points in the train set: 1335359, number of used features: 132\n",
      "[LightGBM] [Info] Using GPU Device: Tesla P100-PCIE-16GB, Vendor: NVIDIA Corporation\n",
      "[LightGBM] [Info] Compiling OpenCL Kernel with 256 bins...\n",
      "[LightGBM] [Info] GPU programs have been built\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 132 dense feature groups (168.10 MB) transferred to GPU in 0.204958 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.505494 -> initscore=0.021977\n",
      "[LightGBM] [Info] Start training from score 0.021977\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[50]\ttraining's auc: 0.617183\tvalid_1's auc: 0.566249\n",
      "[100]\ttraining's auc: 0.6504\tvalid_1's auc: 0.566979\n",
      "[150]\ttraining's auc: 0.674821\tvalid_1's auc: 0.567207\n",
      "Early stopping, best iteration is:\n",
      "[130]\ttraining's auc: 0.665876\tvalid_1's auc: 0.56761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 2/5 [03:20<04:55, 98.41s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 674778, number of negative: 660581\n",
      "[LightGBM] [Info] This is the GPU trainer!!\n",
      "[LightGBM] [Info] Total Bins 33408\n",
      "[LightGBM] [Info] Number of data points in the train set: 1335359, number of used features: 132\n",
      "[LightGBM] [Info] Using GPU Device: Tesla P100-PCIE-16GB, Vendor: NVIDIA Corporation\n",
      "[LightGBM] [Info] Compiling OpenCL Kernel with 256 bins...\n",
      "[LightGBM] [Info] GPU programs have been built\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 132 dense feature groups (168.10 MB) transferred to GPU in 0.212269 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.505316 -> initscore=0.021264\n",
      "[LightGBM] [Info] Start training from score 0.021264\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[50]\ttraining's auc: 0.636051\tvalid_1's auc: 0.539943\n",
      "[100]\ttraining's auc: 0.673384\tvalid_1's auc: 0.539476\n",
      "Early stopping, best iteration is:\n",
      "[55]\ttraining's auc: 0.640049\tvalid_1's auc: 0.540417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 3/5 [04:20<02:41, 80.67s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 671196, number of negative: 664163\n",
      "[LightGBM] [Info] This is the GPU trainer!!\n",
      "[LightGBM] [Info] Total Bins 33408\n",
      "[LightGBM] [Info] Number of data points in the train set: 1335359, number of used features: 132\n",
      "[LightGBM] [Info] Using GPU Device: Tesla P100-PCIE-16GB, Vendor: NVIDIA Corporation\n",
      "[LightGBM] [Info] Compiling OpenCL Kernel with 256 bins...\n",
      "[LightGBM] [Info] GPU programs have been built\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 132 dense feature groups (168.10 MB) transferred to GPU in 0.195163 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.502633 -> initscore=0.010534\n",
      "[LightGBM] [Info] Start training from score 0.010534\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[50]\ttraining's auc: 0.64841\tvalid_1's auc: 0.533403\n",
      "Early stopping, best iteration is:\n",
      "[48]\ttraining's auc: 0.646042\tvalid_1's auc: 0.534148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 4/5 [05:16<01:11, 71.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Number of positive: 671589, number of negative: 663770\n",
      "[LightGBM] [Info] This is the GPU trainer!!\n",
      "[LightGBM] [Info] Total Bins 33408\n",
      "[LightGBM] [Info] Number of data points in the train set: 1335359, number of used features: 132\n",
      "[LightGBM] [Info] Using GPU Device: Tesla P100-PCIE-16GB, Vendor: NVIDIA Corporation\n",
      "[LightGBM] [Info] Compiling OpenCL Kernel with 256 bins...\n",
      "[LightGBM] [Info] GPU programs have been built\n",
      "[LightGBM] [Info] Size of histogram bin entry: 8\n",
      "[LightGBM] [Info] 132 dense feature groups (168.10 MB) transferred to GPU in 0.209714 secs. 0 sparse feature groups\n",
      "[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.502928 -> initscore=0.011711\n",
      "[LightGBM] [Info] Start training from score 0.011711\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[50]\ttraining's auc: 0.660963\tvalid_1's auc: 0.52496\n",
      "[100]\ttraining's auc: 0.705434\tvalid_1's auc: 0.524002\n",
      "Early stopping, best iteration is:\n",
      "[54]\ttraining's auc: 0.665764\tvalid_1's auc: 0.526047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [06:13<00:00, 74.62s/it]\n"
     ]
    }
   ],
   "source": [
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "params={\"num_leaves\":300, # 300-500\n",
    "        \"feature_fraction\":0.52, #0.5-0.8\n",
    "        \"bagging_fraction\":0.52, #0.5-0.8\n",
    "        \"objective\":\"binary\",\n",
    "        \"learning_rate\":0.05, # 0.001-0.01\n",
    "        \"boosting_type\":\"gbdt\",\n",
    "        \"metric\":\"auc\",\n",
    "        'device':'gpu',\n",
    "        'seed':xx\n",
    "    \n",
    "       }\n",
    "models = [] \n",
    "    \n",
    "for i in tqdm(range(y_train.shape[1])):\n",
    "    d_train = lgbm.Dataset(X_train,label=y_train[:,i])\n",
    "    d_eval = lgbm.Dataset(X_valid,label=y_valid[:,i],reference=d_train)\n",
    "    clf = lgbm.train(params,d_train,valid_sets=[d_train,d_eval],num_boost_round=1500,\\\n",
    "                    early_stopping_rounds=50,verbose_eval=50)\n",
    "    models.append(clf)\n",
    "    model_name = 'lgb_model_'+str(i)+'.bin'\n",
    "    pickle.dump(clf, open(model_name, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-08T02:28:48.200211Z",
     "iopub.status.busy": "2021-02-08T02:28:48.199174Z",
     "iopub.status.idle": "2021-02-08T02:28:56.952200Z",
     "shell.execute_reply": "2021-02-08T02:28:56.952756Z"
    },
    "papermill": {
     "duration": 8.777154,
     "end_time": "2021-02-08T02:28:56.952960",
     "exception": false,
     "start_time": "2021-02-08T02:28:48.175806",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5506497969235709\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "valid_pred=np.stack([model.predict(X_valid) for model in models]).T\n",
    "valid_auc = roc_auc_score(y_valid, valid_pred)\n",
    "print(valid_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.019921,
     "end_time": "2021-02-08T02:28:57.041202",
     "exception": false,
     "start_time": "2021-02-08T02:28:57.021281",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 531.416942,
   "end_time": "2021-02-08T02:28:57.868902",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-02-08T02:20:06.451960",
   "version": "2.2.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}