g2-forestcovertype-training-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e836f542",
   "metadata": {
    "papermill": {
     "duration": 0.021433,
     "end_time": "2021-11-28T21:16:11.013641",
     "exception": false,
     "start_time": "2021-11-28T21:16:10.992208",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Import Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4f690447",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:11.061852Z",
     "iopub.status.busy": "2021-11-28T21:16:11.061055Z",
     "iopub.status.idle": "2021-11-28T21:16:14.076792Z",
     "shell.execute_reply": "2021-11-28T21:16:14.076178Z",
     "shell.execute_reply.started": "2021-11-28T21:01:08.384396Z"
    },
    "papermill": {
     "duration": 3.044323,
     "end_time": "2021-11-28T21:16:14.076944",
     "exception": false,
     "start_time": "2021-11-28T21:16:11.032621",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np, pandas as pd, matplotlib.pyplot as plt\n",
    "import joblib\n",
    "import optuna\n",
    "import sklearn \n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import VotingClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3aec531",
   "metadata": {
    "papermill": {
     "duration": 0.019126,
     "end_time": "2021-11-28T21:16:14.115647",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.096521",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9fe73d8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.157459Z",
     "iopub.status.busy": "2021-11-28T21:16:14.156914Z",
     "iopub.status.idle": "2021-11-28T21:16:14.262655Z",
     "shell.execute_reply": "2021-11-28T21:16:14.263379Z",
     "shell.execute_reply.started": "2021-11-28T21:01:11.902814Z"
    },
    "papermill": {
     "duration": 0.128992,
     "end_time": "2021-11-28T21:16:14.263533",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.134541",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type32</th>\n",
       "      <th>Soil_Type33</th>\n",
       "      <th>Soil_Type34</th>\n",
       "      <th>Soil_Type35</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Cover_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0   1       2596      51      3                               258   \n",
       "1   2       2590      56      2                               212   \n",
       "2   3       2804     139      9                               268   \n",
       "3   4       2785     155     18                               242   \n",
       "4   5       2595      45      2                               153   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  ...  Soil_Type32  \\\n",
       "0            221             232            148  ...            0   \n",
       "1            220             235            151  ...            0   \n",
       "2            234             238            135  ...            0   \n",
       "3            238             238            122  ...            0   \n",
       "4            220             234            150  ...            0   \n",
       "\n",
       "   Soil_Type33  Soil_Type34  Soil_Type35  Soil_Type36  Soil_Type37  \\\n",
       "0            0            0            0            0            0   \n",
       "1            0            0            0            0            0   \n",
       "2            0            0            0            0            0   \n",
       "3            0            0            0            0            0   \n",
       "4            0            0            0            0            0   \n",
       "\n",
       "   Soil_Type38  Soil_Type39  Soil_Type40  Cover_Type  \n",
       "0            0            0            0           5  \n",
       "1            0            0            0           5  \n",
       "2            0            0            0           2  \n",
       "3            0            0            0           2  \n",
       "4            0            0            0           5  \n",
       "\n",
       "[5 rows x 56 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "train = pd.read_csv('../input/forest-cover-type-prediction/train.csv')\n",
    "# view data\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d35335b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.317229Z",
     "iopub.status.busy": "2021-11-28T21:16:14.316484Z",
     "iopub.status.idle": "2021-11-28T21:16:14.319346Z",
     "shell.execute_reply": "2021-11-28T21:16:14.319757Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.031125Z"
    },
    "papermill": {
     "duration": 0.036459,
     "end_time": "2021-11-28T21:16:14.319878",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.283419",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>Horizontal_Distance_To_Fire_Points</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type32</th>\n",
       "      <th>Soil_Type33</th>\n",
       "      <th>Soil_Type34</th>\n",
       "      <th>Soil_Type35</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Cover_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>6279</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>6225</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>6121</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>6211</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>6172</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0       2596      51      3                               258   \n",
       "1       2590      56      2                               212   \n",
       "2       2804     139      9                               268   \n",
       "3       2785     155     18                               242   \n",
       "4       2595      45      2                               153   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  \\\n",
       "0            221             232            148   \n",
       "1            220             235            151   \n",
       "2            234             238            135   \n",
       "3            238             238            122   \n",
       "4            220             234            150   \n",
       "\n",
       "   Horizontal_Distance_To_Fire_Points  ...  Soil_Type32  Soil_Type33  \\\n",
       "0                                6279  ...            0            0   \n",
       "1                                6225  ...            0            0   \n",
       "2                                6121  ...            0            0   \n",
       "3                                6211  ...            0            0   \n",
       "4                                6172  ...            0            0   \n",
       "\n",
       "   Soil_Type34  Soil_Type35  Soil_Type36  Soil_Type37  Soil_Type38  \\\n",
       "0            0            0            0            0            0   \n",
       "1            0            0            0            0            0   \n",
       "2            0            0            0            0            0   \n",
       "3            0            0            0            0            0   \n",
       "4            0            0            0            0            0   \n",
       "\n",
       "   Soil_Type39  Soil_Type40  Cover_Type  \n",
       "0            0            0           5  \n",
       "1            0            0           5  \n",
       "2            0            0           2  \n",
       "3            0            0           2  \n",
       "4            0            0           5  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove ID column from set\n",
    "train = train.iloc[:, 1:]\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "989c59e7",
   "metadata": {
    "papermill": {
     "duration": 0.019671,
     "end_time": "2021-11-28T21:16:14.359247",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.339576",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Add Features from Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "21960456",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.407662Z",
     "iopub.status.busy": "2021-11-28T21:16:14.407061Z",
     "iopub.status.idle": "2021-11-28T21:16:14.414302Z",
     "shell.execute_reply": "2021-11-28T21:16:14.413867Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.059233Z"
    },
    "papermill": {
     "duration": 0.035357,
     "end_time": "2021-11-28T21:16:14.414412",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.379055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# add new features from feature engineering\n",
    "train['Elev_to_Horizontal_Hyd'] = train.Elevation - 0.2 * train.Horizontal_Distance_To_Hydrology \n",
    "train['Elev_to_Horizontal_Road'] = train.Elevation - 0.05 * train.Horizontal_Distance_To_Roadways  \n",
    "train['Elev_to_Verticle_Hyd'] = train.Elevation - train.Vertical_Distance_To_Hydrology \n",
    "train['Mean_Horizontal_Dist'] = (train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology + \n",
    "                                 train.Horizontal_Distance_To_Roadways)/3 \n",
    "train['Mean_Fire_Hydro'] = (train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology)/2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "762ed867",
   "metadata": {
    "papermill": {
     "duration": 0.019639,
     "end_time": "2021-11-28T21:16:14.453819",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.434180",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "81ad3472",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.503112Z",
     "iopub.status.busy": "2021-11-28T21:16:14.502586Z",
     "iopub.status.idle": "2021-11-28T21:16:14.517599Z",
     "shell.execute_reply": "2021-11-28T21:16:14.517999Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.088941Z"
    },
    "papermill": {
     "duration": 0.044585,
     "end_time": "2021-11-28T21:16:14.518163",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.473578",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cover_Type</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Elev_to_Horizontal_Hyd</th>\n",
       "      <th>Elev_to_Horizontal_Road</th>\n",
       "      <th>Elev_to_Verticle_Hyd</th>\n",
       "      <th>Mean_Horizontal_Dist</th>\n",
       "      <th>Mean_Fire_Hydro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2544.4</td>\n",
       "      <td>2570.50</td>\n",
       "      <td>2596</td>\n",
       "      <td>2349.000000</td>\n",
       "      <td>3268.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2547.6</td>\n",
       "      <td>2570.50</td>\n",
       "      <td>2596</td>\n",
       "      <td>2275.666667</td>\n",
       "      <td>3218.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2750.4</td>\n",
       "      <td>2645.00</td>\n",
       "      <td>2739</td>\n",
       "      <td>3189.666667</td>\n",
       "      <td>3194.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2736.6</td>\n",
       "      <td>2630.50</td>\n",
       "      <td>2667</td>\n",
       "      <td>3181.000000</td>\n",
       "      <td>3226.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2564.4</td>\n",
       "      <td>2575.45</td>\n",
       "      <td>2596</td>\n",
       "      <td>2238.666667</td>\n",
       "      <td>3162.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cover_Type  Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0           5       2596      51      3                               258   \n",
       "1           5       2590      56      2                               212   \n",
       "2           2       2804     139      9                               268   \n",
       "3           2       2785     155     18                               242   \n",
       "4           5       2595      45      2                               153   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  ...  Soil_Type36  \\\n",
       "0            221             232            148  ...            0   \n",
       "1            220             235            151  ...            0   \n",
       "2            234             238            135  ...            0   \n",
       "3            238             238            122  ...            0   \n",
       "4            220             234            150  ...            0   \n",
       "\n",
       "   Soil_Type37  Soil_Type38  Soil_Type39  Soil_Type40  Elev_to_Horizontal_Hyd  \\\n",
       "0            0            0            0            0                  2544.4   \n",
       "1            0            0            0            0                  2547.6   \n",
       "2            0            0            0            0                  2750.4   \n",
       "3            0            0            0            0                  2736.6   \n",
       "4            0            0            0            0                  2564.4   \n",
       "\n",
       "   Elev_to_Horizontal_Road  Elev_to_Verticle_Hyd  Mean_Horizontal_Dist  \\\n",
       "0                  2570.50                  2596           2349.000000   \n",
       "1                  2570.50                  2596           2275.666667   \n",
       "2                  2645.00                  2739           3189.666667   \n",
       "3                  2630.50                  2667           3181.000000   \n",
       "4                  2575.45                  2596           2238.666667   \n",
       "\n",
       "   Mean_Fire_Hydro  \n",
       "0           3268.5  \n",
       "1           3218.5  \n",
       "2           3194.5  \n",
       "3           3226.5  \n",
       "4           3162.5  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# move target to first column\n",
    "first_column = train.pop('Cover_Type')\n",
    "  \n",
    "# insert column using insert(position,column_name,first_column) function\n",
    "train.insert(0, 'Cover_Type', first_column)\n",
    "  \n",
    "# view\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c78346fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.568239Z",
     "iopub.status.busy": "2021-11-28T21:16:14.567644Z",
     "iopub.status.idle": "2021-11-28T21:16:14.572937Z",
     "shell.execute_reply": "2021-11-28T21:16:14.573376Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.144377Z"
    },
    "papermill": {
     "duration": 0.034162,
     "end_time": "2021-11-28T21:16:14.573497",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.539335",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create cat, num, and y\n",
    "X_cat = train.iloc[:, 11:55].values\n",
    "B = train.iloc[:, 55:60]\n",
    "A = train.iloc[:, 1:11]\n",
    "X_num = pd.concat([A, B], axis = 1).values\n",
    "y = train.iloc[:, 0].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6b211b39",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.619806Z",
     "iopub.status.busy": "2021-11-28T21:16:14.618918Z",
     "iopub.status.idle": "2021-11-28T21:16:14.626832Z",
     "shell.execute_reply": "2021-11-28T21:16:14.626115Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.162492Z"
    },
    "papermill": {
     "duration": 0.032943,
     "end_time": "2021-11-28T21:16:14.626968",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.594025",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Categorical Shape: (15120, 44)\n",
      "Numerical Shape: (15120, 15)\n",
      "Label Shape: (15120,)\n"
     ]
    }
   ],
   "source": [
    "# scale/standardizing numerical columns\n",
    "# scaler object\n",
    "scaler = StandardScaler()\n",
    "# fit to training data\n",
    "scaler.fit(X_num)\n",
    "# scale num columns\n",
    "X_num = scaler.transform(X_num)\n",
    "\n",
    "# shape\n",
    "print(f'Categorical Shape: {X_cat.shape}')\n",
    "print(f'Numerical Shape: {X_num.shape}')\n",
    "print(f'Label Shape: {y.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d3eb1e86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.674433Z",
     "iopub.status.busy": "2021-11-28T21:16:14.673436Z",
     "iopub.status.idle": "2021-11-28T21:16:14.679022Z",
     "shell.execute_reply": "2021-11-28T21:16:14.678601Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.187371Z"
    },
    "papermill": {
     "duration": 0.029963,
     "end_time": "2021-11-28T21:16:14.679151",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.649188",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15120, 59)\n"
     ]
    }
   ],
   "source": [
    "# combine num and cat\n",
    "X = np.hstack((X_num, X_cat))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18dca19e",
   "metadata": {
    "papermill": {
     "duration": 0.021049,
     "end_time": "2021-11-28T21:16:14.721399",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.700350",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Various Models\n",
    "The following sections include various models we built to predict the Forest Cover Type. We used GridSearchCV and Optuna to identify the optimal paramaters for each model. \n",
    "\n",
    "Our hyperparameter tuning can be found in the notebook: https://www.kaggle.com/emknowles/g2-forestcovertype-modelparams-notebook/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45791e29",
   "metadata": {
    "papermill": {
     "duration": 0.020804,
     "end_time": "2021-11-28T21:16:14.763336",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.742532",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Logistic Regression\n",
    "Logistic Regression did not yield high accuracy, even after GridSearchCV and Optuna were used to find optimal parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "64fee173",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:14.810923Z",
     "iopub.status.busy": "2021-11-28T21:16:14.809943Z",
     "iopub.status.idle": "2021-11-28T21:16:33.003405Z",
     "shell.execute_reply": "2021-11-28T21:16:33.004279Z",
     "shell.execute_reply.started": "2021-11-28T21:01:12.212880Z"
    },
    "papermill": {
     "duration": 18.219946,
     "end_time": "2021-11-28T21:16:33.004543",
     "exception": false,
     "start_time": "2021-11-28T21:16:14.784597",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7126322751322751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "# best model \n",
    "\n",
    "lr_model = LogisticRegression(random_state = 1, \n",
    "                              penalty = 'none', \n",
    "                              max_iter = 500, \n",
    "                              solver = 'saga')\n",
    "lr_model.fit(X, y)\n",
    "print(lr_model.score(X, y))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0594980",
   "metadata": {
    "papermill": {
     "duration": 0.038911,
     "end_time": "2021-11-28T21:16:33.083963",
     "exception": false,
     "start_time": "2021-11-28T21:16:33.045052",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Decision Tree\n",
    "The best parameters from Optuna were {'max_depth': 19, 'min_samples_leaf': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "82d271cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:33.150315Z",
     "iopub.status.busy": "2021-11-28T21:16:33.149319Z",
     "iopub.status.idle": "2021-11-28T21:16:33.419497Z",
     "shell.execute_reply": "2021-11-28T21:16:33.419922Z",
     "shell.execute_reply.started": "2021-11-28T21:01:33.877303Z"
    },
    "papermill": {
     "duration": 0.297545,
     "end_time": "2021-11-28T21:16:33.420062",
     "exception": false,
     "start_time": "2021-11-28T21:16:33.122517",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(max_depth=19, random_state=1)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dt best model\n",
    "dt_model = DecisionTreeClassifier(random_state = 1, \n",
    "                                  max_depth = 19, \n",
    "                                  min_samples_leaf = 1)\n",
    "dt_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97df2e95",
   "metadata": {
    "papermill": {
     "duration": 0.021681,
     "end_time": "2021-11-28T21:16:33.463564",
     "exception": false,
     "start_time": "2021-11-28T21:16:33.441883",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Random Forest\n",
    "The best parameters from Optuna were {'n_estimators': 130, 'max_depth': 50, 'min_samples_leaf': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d88f8bdd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:33.511179Z",
     "iopub.status.busy": "2021-11-28T21:16:33.510373Z",
     "iopub.status.idle": "2021-11-28T21:16:37.077462Z",
     "shell.execute_reply": "2021-11-28T21:16:37.076846Z",
     "shell.execute_reply.started": "2021-11-28T21:01:34.201681Z"
    },
    "papermill": {
     "duration": 3.592591,
     "end_time": "2021-11-28T21:16:37.077604",
     "exception": false,
     "start_time": "2021-11-28T21:16:33.485013",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(max_depth=50, n_estimators=130, random_state=1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best model\n",
    "rf_model = RandomForestClassifier(random_state = 1, \n",
    "                                  n_estimators = 130, \n",
    "                                  max_depth = 50, \n",
    "                                  min_samples_leaf = 1)\n",
    "\n",
    "rf_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dde115c",
   "metadata": {
    "papermill": {
     "duration": 0.022074,
     "end_time": "2021-11-28T21:16:37.121626",
     "exception": false,
     "start_time": "2021-11-28T21:16:37.099552",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extra Tree Classifier\n",
    "The best parameters from Optuna were {'max_depth': 33, 'min_samples_leaf': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1b3e2ef7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:37.171130Z",
     "iopub.status.busy": "2021-11-28T21:16:37.170307Z",
     "iopub.status.idle": "2021-11-28T21:16:51.698959Z",
     "shell.execute_reply": "2021-11-28T21:16:51.699412Z",
     "shell.execute_reply.started": "2021-11-28T21:01:38.110112Z"
    },
    "papermill": {
     "duration": 14.555552,
     "end_time": "2021-11-28T21:16:51.699576",
     "exception": false,
     "start_time": "2021-11-28T21:16:37.144024",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(max_depth=33, n_estimators=1000, random_state=1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best model\n",
    "tree_model = ExtraTreesClassifier(random_state = 1, \n",
    "                                  n_estimators = 1000, \n",
    "                                  max_depth = 33, \n",
    "                                  min_samples_leaf = 1)\n",
    "\n",
    "tree_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc6f10b0",
   "metadata": {
    "papermill": {
     "duration": 0.022562,
     "end_time": "2021-11-28T21:16:51.744618",
     "exception": false,
     "start_time": "2021-11-28T21:16:51.722056",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Gradient Boosting Classifier\n",
    "The best parameters from Optuna were {'max_depth': 19, 'min_samples_leaf': 16}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b45e62fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:16:51.794332Z",
     "iopub.status.busy": "2021-11-28T21:16:51.793497Z",
     "iopub.status.idle": "2021-11-28T21:20:21.604981Z",
     "shell.execute_reply": "2021-11-28T21:20:21.604534Z",
     "shell.execute_reply.started": "2021-11-28T21:01:55.203045Z"
    },
    "papermill": {
     "duration": 209.838109,
     "end_time": "2021-11-28T21:20:21.605123",
     "exception": false,
     "start_time": "2021-11-28T21:16:51.767014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 29s, sys: 135 ms, total: 3min 29s\n",
      "Wall time: 3min 29s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(max_depth=19, min_samples_leaf=16, random_state=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "gradb_model = GradientBoostingClassifier(random_state = 0,\n",
    "                                         max_depth = 19, \n",
    "                                         min_samples_leaf = 16)\n",
    "\n",
    "gradb_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf930392",
   "metadata": {
    "papermill": {
     "duration": 0.023634,
     "end_time": "2021-11-28T21:20:21.652694",
     "exception": false,
     "start_time": "2021-11-28T21:20:21.629060",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extreme Gradient Boosting Classifier\n",
    "The best parameters from Optuna were {'max_depth': 14}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0fc171db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:20:21.703244Z",
     "iopub.status.busy": "2021-11-28T21:20:21.702677Z",
     "iopub.status.idle": "2021-11-28T21:20:30.255799Z",
     "shell.execute_reply": "2021-11-28T21:20:30.256582Z",
     "shell.execute_reply.started": "2021-11-28T21:05:39.063104Z"
    },
    "papermill": {
     "duration": 8.580755,
     "end_time": "2021-11-28T21:20:30.256793",
     "exception": false,
     "start_time": "2021-11-28T21:20:21.676038",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21:20:23] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "CPU times: user 7.53 s, sys: 364 ms, total: 7.89 s\n",
      "Wall time: 8.54 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "xgb_model = XGBClassifier(random_state = 0,\n",
    "                           max_depth = 14,\n",
    "                           tree_method = 'gpu_hist')\n",
    "\n",
    "xgb_model.fit(X, y)\n",
    "xgb_model.score(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c9a4600",
   "metadata": {
    "papermill": {
     "duration": 0.024556,
     "end_time": "2021-11-28T21:20:30.305700",
     "exception": false,
     "start_time": "2021-11-28T21:20:30.281144",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Light Gradient Boosting Classifier\n",
    "The best parameters from Optuna were {'num_leaves': 10, 'max_depth': 16}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7930d586",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:20:30.359204Z",
     "iopub.status.busy": "2021-11-28T21:20:30.358435Z",
     "iopub.status.idle": "2021-11-28T21:20:31.488894Z",
     "shell.execute_reply": "2021-11-28T21:20:31.487925Z",
     "shell.execute_reply.started": "2021-11-28T21:05:48.648714Z"
    },
    "papermill": {
     "duration": 1.159399,
     "end_time": "2021-11-28T21:20:31.489052",
     "exception": false,
     "start_time": "2021-11-28T21:20:30.329653",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.12 s, sys: 40.9 ms, total: 2.16 s\n",
      "Wall time: 1.12 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(max_depth=16, num_leaves=10, random_state=0)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "lgbm_model = LGBMClassifier(random_state = 0,\n",
    "                            max_depth = 16,\n",
    "                            num_leaves = 10)\n",
    "\n",
    "lgbm_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "746f3e82",
   "metadata": {
    "papermill": {
     "duration": 0.026228,
     "end_time": "2021-11-28T21:20:31.541384",
     "exception": false,
     "start_time": "2021-11-28T21:20:31.515156",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model Selection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8470c442",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:20:31.596383Z",
     "iopub.status.busy": "2021-11-28T21:20:31.595598Z",
     "iopub.status.idle": "2021-11-28T21:24:36.414331Z",
     "shell.execute_reply": "2021-11-28T21:24:36.414898Z",
     "shell.execute_reply.started": "2021-11-28T21:05:50.674329Z"
    },
    "papermill": {
     "duration": 244.849602,
     "end_time": "2021-11-28T21:24:36.415126",
     "exception": false,
     "start_time": "2021-11-28T21:20:31.565524",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21:24:19] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Logistic Regression Accuracy 0.7126322751322751\n",
      "Decision Tree Accuracy 0.9808201058201058\n",
      "Random Forest Accuracy 1.0\n",
      "Extra Trees Accuracy 1.0\n",
      "Gradient Boosting Accuracy 1.0\n",
      "Extra Gradient Boosting Accuracy 1.0\n",
      "LightGBM Accuracy 0.8923941798941799\n",
      "Ensemble Accuracy: 1.0\n",
      "CPU times: user 4min 4s, sys: 891 ms, total: 4min 5s\n",
      "Wall time: 4min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# create ensemble classifier \n",
    "ensemble_model = VotingClassifier(\n",
    "    estimators = [('tree', tree_model), \n",
    "                  ('rf', rf_model), \n",
    "                  ('gradb', gradb_model), \n",
    "                  ('xgb', xgb_model)],\n",
    "    voting = 'hard'\n",
    ")\n",
    "\n",
    "# fit\n",
    "ensemble_model.fit(X, y)\n",
    "\n",
    "# print training accuracy\n",
    "print('Logistic Regression Accuracy', lr_model.score(X, y))\n",
    "print('Decision Tree Accuracy', dt_model.score(X, y))\n",
    "print('Random Forest Accuracy', rf_model.score(X, y))\n",
    "print('Extra Trees Accuracy', tree_model.score(X, y))\n",
    "print('Gradient Boosting Accuracy', gradb_model.score(X, y))\n",
    "print('Extra Gradient Boosting Accuracy', xgb_model.score(X, y))\n",
    "print('LightGBM Accuracy', lgbm_model.score(X, y))\n",
    "print('Ensemble Accuracy:', ensemble_model.score(X, y))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa19701c",
   "metadata": {
    "papermill": {
     "duration": 0.027739,
     "end_time": "2021-11-28T21:24:36.471398",
     "exception": false,
     "start_time": "2021-11-28T21:24:36.443659",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Save Preprocessor and Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "59c6bbd7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:24:36.531352Z",
     "iopub.status.busy": "2021-11-28T21:24:36.530680Z",
     "iopub.status.idle": "2021-11-28T21:24:36.535802Z",
     "shell.execute_reply": "2021-11-28T21:24:36.536310Z",
     "shell.execute_reply.started": "2021-11-28T21:10:13.629289Z"
    },
    "papermill": {
     "duration": 0.037148,
     "end_time": "2021-11-28T21:24:36.536440",
     "exception": false,
     "start_time": "2021-11-28T21:24:36.499292",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['forest_cover_scaler_final.joblib']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save scaler\n",
    "joblib.dump(scaler, 'forest_cover_scaler_final.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "860b8e23",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T21:24:36.597912Z",
     "iopub.status.busy": "2021-11-28T21:24:36.596955Z",
     "iopub.status.idle": "2021-11-28T21:24:47.219903Z",
     "shell.execute_reply": "2021-11-28T21:24:47.220882Z",
     "shell.execute_reply.started": "2021-11-28T21:15:04.085730Z"
    },
    "papermill": {
     "duration": 10.657152,
     "end_time": "2021-11-28T21:24:47.221119",
     "exception": false,
     "start_time": "2021-11-28T21:24:36.563967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model written to file.\n"
     ]
    }
   ],
   "source": [
    "joblib.dump(rf_model, 'rf_model_final.joblib')\n",
    "joblib.dump(tree_model, 'tree_model_final.joblib')\n",
    "joblib.dump(gradb_model, 'gradb_model_final.joblib')\n",
    "joblib.dump(xgb_model, 'xgb_model_final.joblib')\n",
    "joblib.dump(lgbm_model, 'lgbm_model_final.joblib')\n",
    "joblib.dump(ensemble_model, 'ensemble_model_final.joblib')\n",
    "print('Model written to file.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 535.065099,
   "end_time": "2021-11-28T21:24:58.507881",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-11-28T21:16:03.442782",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}