main_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('train.csv')\n",
    "\"basic view of raw dataset\"\n",
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f73eba297794f39b648ce7fae03d86b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Summarize dataset'), FloatProgress(value=0.0, max=26.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33870eeddba54b449ee9fc7d0fab538e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Generate report structure'), FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9c7e0d6c0dd4b1b84c982a0de8b299d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Render HTML'), FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3169007ea48147bca8bb28a2e3c9830b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Export report to file'), FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# import pandas_profiling\n",
    "# profile = pandas_profiling.ProfileReport(train_df)\n",
    "# profile.to_file(output_file=\"./reportb.html\")\n",
    "# # # a copy of the profile report is saved in the main directory\n",
    "# # pandas_profiling.ProfileReport(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Sex_mean_enc</th>\n",
       "      <th>Ageef</th>\n",
       "      <th>Agedt</th>\n",
       "      <th>Fareef</th>\n",
       "      <th>Faredt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.250000</td>\n",
       "      <td>0.205298</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.356618</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.212560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.283302</td>\n",
       "      <td>0.754789</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.413158</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.628571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.925000</td>\n",
       "      <td>0.754789</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.356618</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.212560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.099998</td>\n",
       "      <td>0.754789</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.413158</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.050000</td>\n",
       "      <td>0.205298</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.413158</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.212560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>29.125000</td>\n",
       "      <td>0.754789</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.413158</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.437299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>0.205298</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.413158</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.437299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.754789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.356618</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.437299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.205298</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.356618</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.437299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.750000</td>\n",
       "      <td>0.205298</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.413158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.212560</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>714 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Survived  Pclass   Age  SibSp  Parch       Fare  Sex_mean_enc  Ageef  \\\n",
       "0         0.0     3.0  22.0    1.0    0.0   7.250000      0.205298    2.0   \n",
       "1         1.0     1.0  38.0    1.0    0.0  71.283302      0.754789    5.0   \n",
       "2         1.0     3.0  26.0    0.0    0.0   7.925000      0.754789    3.0   \n",
       "3         1.0     1.0  35.0    1.0    0.0  53.099998      0.754789    5.0   \n",
       "4         0.0     3.0  35.0    0.0    0.0   8.050000      0.205298    5.0   \n",
       "..        ...     ...   ...    ...    ...        ...           ...    ...   \n",
       "885       0.0     3.0  39.0    0.0    5.0  29.125000      0.754789    6.0   \n",
       "886       0.0     2.0  27.0    0.0    0.0  13.000000      0.205298    3.0   \n",
       "887       1.0     1.0  19.0    0.0    0.0  30.000000      0.754789    1.0   \n",
       "889       1.0     1.0  26.0    0.0    0.0  30.000000      0.205298    3.0   \n",
       "890       0.0     3.0  32.0    0.0    0.0   7.750000      0.205298    4.0   \n",
       "\n",
       "        Agedt  Fareef    Faredt  \n",
       "0    0.356618     0.0  0.212560  \n",
       "1    0.413158     6.0  0.628571  \n",
       "2    0.356618     1.0  0.212560  \n",
       "3    0.413158     6.0  0.750000  \n",
       "4    0.413158     1.0  0.212560  \n",
       "..        ...     ...       ...  \n",
       "885  0.413158     5.0  0.437299  \n",
       "886  0.413158     3.0  0.437299  \n",
       "887  0.356618     5.0  0.437299  \n",
       "889  0.356618     5.0  0.437299  \n",
       "890  0.413158     0.0  0.212560  \n",
       "\n",
       "[714 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"Now defining classes necessary for preprocessing\"\"\"\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from feature_engine.discretisers import EqualFrequencyDiscretiser as efd\n",
    "from feature_engine.discretisers import DecisionTreeDiscretiser as dtd\n",
    "\n",
    "# global variables are declared here for encoding the test data set based on the target variable\n",
    "global dtd_g\n",
    "global efd_g\n",
    "efd_g = efd(q=8, variables = ['Age','Fare'])\n",
    "dtd_g = dtd(cv=5, scoring='accuracy', variables = ['Age','Fare'],\n",
    "            regression=False, param_grid = {'max_depth':[3,4,5,6]})\n",
    "\n",
    "\n",
    "class mean_encoder(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"A very common feautre engineering for improving quality of binary classification\"\"\"\n",
    "    \n",
    "    def __init__(self, target_col = None, column_array=[], encoder = []):\n",
    "        self.target_col = target_col\n",
    "        self.mean_encodings = encoder\n",
    "        self.column_array = column_array\n",
    "        \n",
    "    def fit(self, df):\n",
    "        if self.mean_encodings != []:\n",
    "            return self\n",
    "        for i in self.column_array:\n",
    "            self.mean_encodings.append(df.groupby([i])[self.target_col].mean().to_dict())   \n",
    "        # for test set pipelining\n",
    "        global encoding_g\n",
    "        encoding_g = self.mean_encodings\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        for encodings,column in zip(self.mean_encodings,self.column_array):\n",
    "            df[column+'_mean_enc'] = df[column].apply(lambda value: encodings[value])\n",
    "        return df\n",
    "\n",
    "\n",
    "class Drop_Col(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\"For dropping unnecessary columns\"\"\"\n",
    "    \n",
    "    def __init__(self,column_array=[]):\n",
    "        self.column_array = column_array\n",
    "        \n",
    "    def fit(self, df, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        return df.drop(self.column_array, axis = 1)\n",
    "\n",
    "\n",
    "class custom_pr(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\"For extracting useful info from the Cabin column of the dataset,\n",
    "    dropping null values for age, imputing fare values and for other custom transformations\"\"\"\n",
    "    \n",
    "    def __init__(self,drop_=True):\n",
    "        self.drop_ = drop_\n",
    "\n",
    "    def fit(self, df, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        df.Cabin.fillna('Z', inplace = True)\n",
    "        df['Cabin'] = df['Cabin'].apply(lambda x: str(str(x)[0]))\n",
    "        if self.drop_==True:\n",
    "            df.dropna(inplace= True, subset = ['Age'])\n",
    "        else:\n",
    "            df.Age.fillna(df.Age.median(), inplace=True)\n",
    "        df.Fare.fillna(df.Fare.median(), inplace=True)\n",
    "        return df\n",
    "\n",
    "    \n",
    "class One_Hot_Encoder(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"For One Hot Encoding categorical variables\"\"\"\n",
    "    \n",
    "    def __init__(self,column_array=[]):\n",
    "        self.column_array = column_array\n",
    "    \n",
    "    def fit(self, df):\n",
    "        return self\n",
    "    \n",
    "    def transform(self,df):\n",
    "        for i in self.column_array:\n",
    "            df[i+'i'] = df[i]\n",
    "        return pd.get_dummies(df, columns = self.column_array, drop_first=True)\n",
    "        \n",
    "\n",
    "class cv_discretizer(BaseEstimator, TransformerMixin):\n",
    "    \"\"\" For discretizing continuous variables for extracting useful features\"\"\"\n",
    "    \n",
    "    def __init__(self,column_array=[],use_global=False):\n",
    "        self.column_array = column_array\n",
    "        self.use_global = use_global\n",
    "    \n",
    "    def fit(self,df):\n",
    "        self.efd_ = efd(q=8, variables = self.column_array)\n",
    "        self.dtd_ = dtd(cv=5, scoring='accuracy', variables = self.column_array,\n",
    "                   regression=False, param_grid = {'max_depth':[3,4,5,6]})\n",
    "        if self.use_global==False:\n",
    "            self.dtd_.fit(df,df.Survived)\n",
    "            self.efd_.fit(df,df.Survived)\n",
    "            dtd_g.fit(df,df.Survived)\n",
    "            efd_g.fit(df,df.Survived)\n",
    "        return self\n",
    "        \n",
    "    def transform(self,df,y=None):\n",
    "        lf = df.copy()\n",
    "        for i in self.column_array:\n",
    "            if self.use_global == False:\n",
    "                lf[i+str('ef')] = self.efd_.transform(df)[i]\n",
    "                lf[i+str('dt')] = self.dtd_.transform(df)[i]\n",
    "            else:\n",
    "                df['Survived'] = pd.Series([0 for i in range(df.shape[0])]) # dummy column\n",
    "                lf[i+str('ef')] = efd_g.transform(df)[i]\n",
    "                lf[i+str('dt')] = dtd_g.transform(df)[i]\n",
    "        df = df.drop(df.index)\n",
    "        return lf\n",
    "\n",
    "# we drop the below columns as they are random and have no useful information. 'Cabin' has high correlation\n",
    "# with 'PClass' and 'Sex' is dropped as its target encoding has more feature importance.\n",
    "\n",
    "pre_pipe = Pipeline([(\"custom_pr\",custom_pr()),\n",
    "                     (\"mean_encoder\", mean_encoder('Survived',['Sex'])),\n",
    "                     (\"dropcol\", Drop_Col(['Ticket','Name','PassengerId','Embarked','Cabin','Sex'])),\n",
    "                     ('cv_discretizer', cv_discretizer(['Age','Fare']))\n",
    "                    ])\n",
    "\n",
    "train_df = pre_pipe.fit_transform(train_df).astype('float32')\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"According to the profile report we have several outliers. They will be removed accordingly, based on\n",
    "their effect on model performane\"\"\"\n",
    "train_df.drop(train_df[train_df.Age<1].index,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Splitting data into train and test set\"\"\"\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = train_df.drop(['Survived'],1).to_numpy()\n",
    "y = train_df['Survived'].to_numpy()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.18, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"In this notebook, we use xgboost's random forest classifier\"\"\"\n",
    "from xgboost import XGBRFClassifier\n",
    "\n",
    "# defining fixed params\n",
    "verbosity = 3\n",
    "objective = 'binary:logistic'\n",
    "n_jobs = 4\n",
    "folds = 6\n",
    "\n",
    "# defining a set of hyperparams for grid search\n",
    "params = {'n_estimators' : [200, 300, 400, 500, 600],\n",
    "          'max_depth' : [5, 7, 9 ,11, 12, 14, 16, 18],\n",
    "          'min_child_weight' : [0.5, 0.1, 1, 5, 10],\n",
    "          'reg_alpha' : [1, 10, 50, 100],\n",
    "          'reg_lambda' : [1, 10, 50, 100]\n",
    "         }\n",
    "\n",
    "# for runtime evaluation:\n",
    "eval_set = [(X_train[i], y_train[i]) for i in np.random.choice(X_train.shape[0], X_train.shape[0]//6)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"For hyperparameter tuning, we use sklearn's Grid Search CV to find the best parameters\"\"\"\n",
    "# Since this process takes up a lot of time, you can skip to the next cell by commenting this code\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 42)\n",
    "\n",
    "xgb = XGBRFClassifier(objective=objective, nthread=1, silent= True,\n",
    "                      eval_set=eval_set, eval_metric = 'auc', early_stopping_rounds = 50)\n",
    "\n",
    "random_search = GridSearchCV(xgb, param_grid=params,\n",
    "                             scoring='accuracy', n_jobs=n_jobs,\n",
    "                             cv=skf.split(X,y), verbose = verbosity)\n",
    "random_search.fit(X,y)\n",
    "random_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:07:15] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.1.0\\src\\learner.cc:480: \n",
      "Parameters: { early_stopping_rounds, eval_set, silent } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBRFClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "                colsample_bytree=1, early_stopping_rounds=50, eval_metric='auc',\n",
       "                eval_set=[(array([ 3.        , 30.        ,  1.        ,  1.        , 24.15      ,\n",
       "        0.7547893 ,  4.        ,  0.41315788,  4.        ,  0.43729904],\n",
       "      dtype=float32),\n",
       "                           0.0),\n",
       "                          (array([ 3.        , 30.        ,  0.        ,  0.        ,  8.6625    ,\n",
       "        0.7547893 ,  4.        ,  0.41315788,  2.        ,  0.21256039],\n",
       "      dtype=float32),\n",
       "                           0.0),\n",
       "                          (array([ 1.        , 25.        ,  1.        ,  0.        , 55.4417    ,\n",
       "        0.205298...\n",
       "                interaction_constraints='', learning_rate=0.001,\n",
       "                max_delta_step=0, max_depth=11, min_child_weight=0.5,\n",
       "                missing=nan, monotone_constraints='()', n_estimators=500,\n",
       "                n_jobs=1, nthread=1, num_parallel_tree=500,\n",
       "                objective='binary:logistic', random_state=0, reg_alpha=1,\n",
       "                reg_lambda=1, scale_pos_weight=1, silent=True,\n",
       "                tree_method='exact', validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"Training Final Model\"\"\" # based on best params\n",
    "xgbf = XGBRFClassifier(objective=objective, nthread=1, silent= True,\n",
    "                       learning_rate = 0.001,\n",
    "                       max_depth = 11,\n",
    "                       min_child_weight=0.5,\n",
    "                       n_estimators = 500,\n",
    "                       reg_alpha = 1,\n",
    "                       reg_lambda = 1,\n",
    "                       eval_set=eval_set,\n",
    "                       eval_metric = 'auc',\n",
    "                       early_stopping_rounds = 50)\n",
    "xgbf.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:title={'center':'Feature importance'}, xlabel='F score', ylabel='Features'>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3oAAAImCAYAAAAFaPhAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABGZklEQVR4nO3dfZxXZZ3/8dcHBEQx0QAX8YZQQwFxFDflp+moebek5Io3hOVNZm651aa1lDfhmgvetZparaZJaaaWimGLmjZCm3eow40K2AoVRuJNqNwIDHx+f8yRhmEGB4aZgcPr+XjMw3Ouc53z/ZyZqx6+va5zvpGZSJIkSZLKo11bFyBJkiRJ2rAMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkaSMQEd+KiB+1dR2SpHIIv0dPkrSpi4g5wA7AijrNH83MvzTzmmdn5m+aV92mJyJGAbtn5mltXYskaf04oydJKovjMrNLnZ/1DnkbQkRs0Zafv7421bolSasz6EmSSisito2IWyJiXkS8GhHfiYj2xbHdIuKxiHgzIt6IiDsiomtx7KfALsCvImJhRHwjIiojYm6968+JiE8U26Mi4hcRcXtEvAOcsbbPb6DWURFxe7HdOyIyIs6MiD9HxN8i4tyI+MeImBoRCyLihjrnnhER/xsR10fE2xExIyKOqHN8x4h4ICLeiog/RMTn631u3brPBb4FnFLc+5Si35kR8VJEvBsRr0TEF+pcozIi5kbE+RExv7jfM+sc7xwR10TEH4v6fhcRnYtjB0bE74t7mhIRlevxp5Yk1WPQkySV2VigBtgd2Bc4Cji7OBbAaGBHYC9gZ2AUQGZ+BvgTf58lvLKJnzcU+AXQFbjjAz6/KQ4A9gBOAa4FLgQ+AfQHTo6IQ+v1fQXoBnwbuDciti+O3QnMLe51GPCfdYNgvbpvAf4TuKu4932KPvOBTwIfAs4E/isi9qtzjX8AtgV6AZ8DboyI7YpjVwODgP8HbA98A1gZEb2AB4HvFO0XAL+MiO7r8DuSJDXAoCdJKov7i1mhBRFxf0TsABwLfDUzF2XmfOC/gFMBMvMPmflIZi7NzNeB7wKHNn75JnkiM+/PzJXUBqJGP7+JLsvM9zLzYWARcGdmzs/MV4FJ1IbH980Hrs3M5Zl5FzATGBIROwMHA/9eXKsa+BHwmYbqzswlDRWSmQ9m5v9lrceBh4GP1+myHPiP4vN/DSwE+kZEO+As4CuZ+WpmrsjM32fmUuA04NeZ+evisx8BJgP/tA6/I0lSA1yHL0kqi0/VfXFKRHwM6ADMi4j3m9sBfy6O9wC+R21Y2aY49rdm1vDnOtu7ru3zm+i1OttLGtjvUmf/1Vz9DWt/pHYGb0fgrcx8t96x/Rupu0ERcSy1M4UfpfY+tgKm1enyZmbW1NlfXNTXDdgS+L8GLrsrcFJEHFenrQPw2w+qR5K0dgY9SVJZ/RlYCnSrF0DeNxpIYGBmvhkRnwJuqHO8/mupF1EbbgAonrWrv8Sw7jkf9PkbWq+IiDphbxfgAeAvwPYRsU2dsLcL8Gqdc+vf62r7EdEJ+CXwWWBcZi6PiPupXf76Qd4A3gN2A6bUO/Zn4KeZ+fk1zpIkNYtLNyVJpZSZ86hdXnhNRHwoItoVL2B5f3nmNtQuL1xQPCv29XqXeA3oU2d/FrBlRAyJiA7ARUCnZnz+htYD+HJEdIiIk6h97vDXmfln4PfA6IjYMiIGUvsM3R1rudZrQO9i2SVAR2rv9XWgppjdO6opRRXLWG8Fvlu8FKZ9RAwuwuPtwHERcXTRvmXxYped1v32JUl1GfQkSWX2WWpDyovULsv8BdCzOHYpsB/wNrUvBLm33rmjgYuKZ/4uyMy3gS9S+3zbq9TO8M1l7db2+RvaU9S+uOUN4HJgWGa+WRwbDvSmdnbvPuDbxfNwjbmn+OebEfFcMRP4ZeBuau/j09TOFjbVBdQu83wGeAu4AmhXhNCh1L7l83VqZ/i+jv9+IknN5hemS5K0iYuIM6j9cveD27oWSdLGwf9iJkmSJEklY9CTJEmSpJJx6aYkSZIklYwzepIkSZJUMgY9SZIkSSoZvzB9I9a1a9fcfffd27oMbaIWLVrE1ltv3dZlaBPl+FFzOYbUHI4fNdfmNIaeffbZNzKze/12g95GbIcddmDy5MltXYY2UVVVVVRWVrZ1GdpEOX7UXI4hNYfjR821OY2hiPhjQ+0u3ZQkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklUxkZlvXoEbs0mf3bHfydW1dhjZR5+9dwzXTtmjrMrSJcvyouRxDag7Hj+qbM2YIACtWrGD//fenV69ejB8/nilTpnDuueeycOFCevfuzR133MGHPvQhxo0bx7XXXsszzzzDGWecwQ033ADAu+++y8c//vFV1507dy6nnXYa1157bVvc1gYREc9m5v71253RW08RcUJEZETs2da1SJIkSZuD6667jr322mvV/tlnn82YMWOYNm0aJ5xwAldddRUAHTt25LLLLuPqq69e7fxtttmG6urqVT+77ror//zP/9yq99BaDHrrbzjwO+DUti5EkiRJKru5c+fy4IMPcvbZZ69qmzlzJocccggARx55JL/85S8B6Ny5MwcffDBbbrllo9d7+eWXmT9//mozfGVi0FsPEdEFOAj4HEXQi4h2EfH9iHghIsZHxK8jYlhxbFBEPB4Rz0bEQxHRsw3LlyRJkjY5X/3qV7nyyitp1+7vEWbAgAE88MADANxzzz38+c9/bvL17rzzTk455RQiYoPXujFw8fP6+RQwITNnRcRbEbEf0AfoDewN9ABeAm6NiA7A9cDQzHw9Ik4BLgfOaujCEXEOcA5At27duWTvmpa+F5XUDp1rn3GQ1ofjR83lGFJzOH5U3+jRo1m+fDnvvvsu1dXVvPnmm1RVVXHuuefyne98h69//escdNBBtGvXjqqqKhYuXEhVVRUzZszg1Vdfpaqqao1r3nrrrXzzm99s8FgZGPTWz3Dg2mL758V+B+CezFwJ/DUiflsc7wsMAB4p/mtBe2BeYxfOzJuAm6D2ZSw+iKz15YPsag7Hj5rLMaTmcPyovuHxDs8++yxnnHEG7733Hu+88w4/+tGPuP322/nsZz8LwKxZs3jhhReorKykqqqKyspK5syZw8KFC6msrFztelOmTKFjx4584QtfaIO7aR3+L2gdRcSHgcOBARGR1Aa3BO5r7BTghcwc3EolSpIkSaUyevRoRo8eDUBVVRVXX301t99+O/Pnz6dHjx6sXLmS73znO5x77rlNut6dd97J8OHDW7LkNuczeutuGPCTzNw1M3tn5s7AbOAN4MTiWb0dgMqi/0yge0QMBoiIDhHRvy0KlyRJksrkzjvv5KMf/Sh77rknO+64I2eeeeaqY7179+ZrX/sat912GzvttBMvvvjiqmN333136YOe36O3jiKiChiTmRPqtH0Z2Iva2btDgFlAJ+C7mflIRFQA3wO2pXYW9drMvPmDPqtv3745c+bMDX4P2jy8v2RBWh+OHzWXY0jN4fhRc21OY6ix79Fz6eY6yszKBtq+B7Vv48zMhcXyzqeBacXxamoDoCRJkiS1OIPehjU+IroCHYHLMvOvbVyPJEmSpM2QQW8Dami2T5IkSZJamy9jkSRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJbNFWxegxi1ZvoLeIx9s6zK0iTp/7xrOcPxoPTl+1FyOoY3DnDFDeO+99zjkkENYunQpNTU1DBs2jEsvvRSA66+/nhtuuIEtttiCIUOGcOWVV7J8+XLOPvtsnnvuOWpqavjsZz/LN7/5TRYvXsxJJ53E//3f/9G+fXuOO+44xowZ08Z3KKkxBr11FBErgGl1mj6VmXPaqBxJkqS16tSpE4899hhdunRh+fLlHHzwwRx77LEsWbKEcePGMXXqVDp16sT8+fMBuOeee1i6dCnTpk1j8eLF9OvXj+HDh9OjRw8uuOACDjvsMJYtW8YRRxzB//zP/3Dssce28R1KaohBb90tycyKdTkhIgKIzFzZMiVJkiQ1LCLo0qULAMuXL2f58uVEBD/4wQ8YOXIknTp1AqBHjx6r+i9atIiamhqWLFlCx44d+dCHPsRWW23FYYcdBkDHjh3Zb7/9mDt3btvclKQP5DN6zRQRXSLi0Yh4LiKmRcTQor13RLwUEd8HngN2joivR8QzETE1Ii5t28olSdLmYsWKFVRUVNCjRw+OPPJIDjjgAGbNmsWkSZM44IADOPTQQ3nmmWcAGDZsGFtvvTU9e/Zkl1124YILLmD77bdf7XoLFizgV7/6FUcccURb3I6kJjDorbvOEVFd/NwHvAeckJn7AYcB1xQzeAB9gZ9k5r7F9h7Ax4AKYFBEHNL65UuSpM1N+/btqa6uZu7cuTz99NNMnz6dmpoa/va3v/Hkk09y1VVXcfLJJ5OZPP3007Rv356//OUvzJ49m2uuuYZXXnll1bVqamoYPnw4X/7yl+nTp08b3pWktXHp5rpbbelmRHQA/rMIbSuBXsAOxeE/ZuaTxfZRxc/zxX4XaoPfxLoXj4hzgHMAunXrziV717TQbajsduhc+zIEaX04ftRcjqGNQ1VV1RptvXv35sYbb2SrrbaiT58+PP744wAsW7aMcePGcdttt9GvXz/+93//F4A+ffowduzYVcs2r7jiCjp37kxFRUWD198QFi5c2GLX1ubBMWTQ2xBGAN2BQZm5PCLmAFsWxxbV6RfA6Mz877VdLDNvAm4C2KXP7nnNNP9EWj/n712D40fry/Gj5nIMbRzmjKjk9ddfp0OHDnTt2pUlS5Zw8cUX8+///u/ss88+/OUvf6GyspJZs2bRrl07hg4dysyZM5kxYwaHHnooixcv5o9//CNXXHEFAwcO5KKLLmKrrbbinnvuoV27llsYVlVVRWVlZYtdX+XnGDLobQjbAvOLkHcYsGsj/R4CLouIOzJzYUT0ApZn5vxWq1SSJG125s2bx+mnn86KFStYuXIlJ598Mp/85CdZtmwZZ511FgMGDKBjx46MHTuWiOBLX/oSZ555JgMGDCAzOfPMMxk4cCBz587l8ssvZ88992S//fYD4LzzzuPss89u4zuU1BCDXvPdAfwqIiYD1cCMhjpl5sMRsRfwRPEI30LgNMCgJ0mSWszAgQN5/vnn12jv2LEjt99++xrtXbp04Z577lmjfaeddiIzW6RGSRueQW8dZWaXevtvAIMb6T6gXt/rgOua+lmdO7Rn5pgh61yjBLVLFuaMqGzrMrSJcvyouRxDktS2fOumJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEkls0VbF6DGLVm+gt4jH2zrMrSJOn/vGs4Y+SBzxgzhrLPOYvz48fTo0YPp06cD8NZbb3HKKacwZ84cevfuzd133812223Hm2++ybBhw3jmmWc444wzuOGGG1Zds7Kyknnz5tG5c2cAHn74YXr06NEm9ydJkqTGOaNXiIgVEVEdEdMj4p6I2GotfUdFxAWtWZ/UHGeccQYTJkxYrW3MmDEcccQRvPzyyxxxxBGMGTMGgC233JLLLruMq6++usFr3XHHHVRXV1NdXW3IkyRJ2kgZ9P5uSWZWZOYAYBlwblsXJG0ohxxyCNtvv/1qbePGjeP0008H4PTTT+f+++8HYOutt+bggw9myy23bO0yJUmStIEY9Bo2CdgdICI+GxFTI2JKRPy0fseI+HxEPFMc/+X7M4ERcVIxOzglIiYWbf0j4uli5nBqROzRqncl1fHaa6/Rs2dPAHr27Mn8+fObdN6ZZ55JRUUFl112GZnZkiVKkiRpPfmMXj0RsQVwLDAhIvoDFwIHZeYbEbF9A6fcm5k3F+d+B/gccD1wCXB0Zr4aEV2LvucC12XmHRHREWjfwOefA5wD0K1bdy7Zu2bD3qA2Gzt0rn1Or6qqCoC//vWvLFq0aNV+Tc3fjzW0P2PGDF599dXV2r70pS/RvXt3Fi9ezLe//W0WL17M0Ucf3fI3o1a3cOHC1f720rpyDKk5HD9qLseQQa+uzhFRXWxPAm4BvgD8IjPfAMjMtxo4b0AR8LoCXYCHivb/BW6LiLuBe4u2J4ALI2InagPiy/Uvlpk3ATcB7NJn97xmmn8irZ/z967hmmlbMGdEJQBz5sxh6623prKydr9Xr1707duXnj17Mm/ePHbcccdVx97vv3DhwtXa6po/fz6TJ09u9Lg2bVVVVf5t1SyOITWH40fN5Rhy6WZd7z+jV5GZ/5qZy4AAPmht2m3AeZm5N3ApsCVAZp4LXATsDFRHxIcz82fA8cAS4KGIOLyF7kX6QMcffzxjx44FYOzYsQwdOnSt/WtqanjjjTcAWL58OePHj2fAgAEtXqckSZLWndNFa/cocF9E/FdmvhkR2zcwq7cNMC8iOgAjgFcBImK3zHwKeCoijgN2johtgVcy83sR0QcYCDzWerejzdXw4cOpqqrijTfeYKedduLSSy9l5MiRnHzyydxyyy3ssssu3HPPPav69+7dm3feeYdly5Zx//338/DDD7Prrrty9NFHs3z5clasWMEnPvEJPv/5z7fhXUmSJKkxBr21yMwXIuJy4PGIWAE8D5xRr9vFwFPAH4Fp1AY/gKuKl60EtYFxCjASOC0ilgN/Bf5jbZ/fuUN7Zo4ZsoHuRpubqqqqVcs277zzzgb7PProow22z5kzp8H2Z599dkOUJkmSpBZm0CtkZpdG2scCY+u1jaqz/QPgBw2c988NXG508SNJkiRJLcZn9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyW7R1AWrckuUr6D3ywbYuQ5uo8/euoRI466yzGD9+PD169GD69OkAvPXWW5xyyinMmTOH3r17c/fdd7PddtsBMHr0aG655Rbat2/P9773PY4++mgAli1bxnnnnUdVVRXt2rXj8ssv58QTT2yju5MkSdLaOKPXgIi4MCJeiIipEVEdEQdExI8iol9xfGEj5x0YEU8V57wUEaNatXCpAWeccQYTJkxYrW3MmDEcccQRvPzyyxxxxBGMGTMGgBdffJGf//znvPDCC0yYMIEvfvGLrFixAoDLL7+cHj16MGvWLF588UUOPfTQVr8XSZIkNY0zevVExGDgk8B+mbk0IroBHTPz7CacPhY4OTOnRER7oG9L1io1xSGHHMKcOXNWaxs3bhxVVVUAnH766VRWVnLFFVcwbtw4Tj31VDp16sRHPvIRdt99d55++mkGDx7MrbfeyowZMwBo164d3bp1a+U7kSRJUlM5o7emnsAbmbkUIDPfyMy/RERVROz/fqeIuCYinouIRyOie9HcA5hXnLciM18s+o6KiJ9GxGMR8XJEfL6V70lazWuvvUbPnj0B6NmzJ/Pnzwfg1VdfZeedd17Vb6edduLVV19lwYIFAFx88cXst99+nHTSSbz22mutXrckSZKaxhm9NT0MXBIRs4DfAHdl5uP1+mwNPJeZ50fEJcC3gfOA/wJmRkQVMAEYm5nvFecMBA4szn0+Ih7MzL/U//CIOAc4B6Bbt+5csnfNBr9BbR526MyqWbu//vWvLFq0aNV+TU3Nqu26+3PnzuWll15adWzevHm88MILdOjQgblz57Ltttvy3e9+l7vvvpvPfOYzfOtb32rdm1KrWbhw4WpjRFpXjiE1h+NHzeUYMuitITMXRsQg4OPAYcBdETGyXreVwF3F9u3AvcW5/xERdwBHAZ8GhgOVRb9xmbkEWBIRvwU+BtzfwOffBNwEsEuf3fOaaf6JtH7O37uGkysrAZgzZw5bb701lcV+r1696Nu3Lz179mTevHnsuOOOVFZW8sQTTwCs6jd69GiOOuooDjzwQLbaaisuvvhi2rVrx2677cYxxxyzqp/Kp6qqyr+vmsUxpOZw/Ki5HEMu3WxQseyyKjPfn6n7oFcLZp1z/y8zfwAcAewTER+u36eRfanVHH/88YwdOxaAsWPHMnTo0FXtP//5z1m6dCmzZ8/m5Zdf5mMf+xgRwXHHHbfqv4w9+uij9OvXr63KlyRJ0gcw6NUTEX0jYo86TRXAH+t1awcMK7Y/DfyuOHdIRETRvgewAlhQ7A+NiC2L4FcJPLPBi5caMHz4cAYPHszMmTPZaaeduOWWWxg5ciSPPPIIe+yxB4888ggjR9ZOWvfv35+TTz6Zfv36ccwxx3DjjTfSvn17AK644gpGjRrFwIED+elPf8o111zTlrclSZKktXBd4Jq6ANdHRFegBvgDtc/M/aJOn0VA/4h4FngbOKVo/wzwXxGxuDh3RGauKLLf08CDwC7AZQ09nye1hDvvvLPB9kcffbTB9gsvvJALL7xwjfZdd92ViRMnbtDaJEmS1DIMevVk5rPA/2vgUGWdPl2KzYvrnXvqWi49KzPPWZdaOndoz8wxQ9blFGmVzf0BZEmSpM2ZSzclSZIkqWSc0WsFmTmqrWuQJEmStPlwRk+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJXMFm1dgBq3ZPkKeo98sK3L0CZgzpghbV2CJEmSNiLO6Eklct111zFgwAD69+/PL37xi1Xt119/PX379qV///584xvfWNU+depUBg8eTP/+/dl7771577332qJsSZIkbWDO6AERsQKYVqfpU5k5ZwNefxSwMDOvjogzgIcz8y8b6voSwPTp07n55pt5+umn6dixIwcccAAvv/wyc+fOZdy4cUydOpVOnToxf/58AGpqajjttNP46U9/yj777MObb75Jhw4d2vguJEmStCEY9GotycyKdTkhIgKIzFy5jp91BjAdMOhpg3rppZc48MAD2WqrrQDYZ599uO+++5g8eTIjR46kU6dOAPTo0QOAhx9+mIEDB7LPPvsA8OEPf7htCpckSdIG59LNBkREl4h4NCKei4hpETG0aO8dES9FxPeB54CdI+LrEfFMREyNiEvrXOPCiJgZEb8B+hZtw4D9gTsiojoiOrfB7amkBgwYwMSJE3nzzTdZvHgxTz31FH/+85+ZNWsWkyZN4oADDuDQQw/lmWeeAWDWrFlEBEcffTT77bcfV155ZRvfgSRJkjYUZ/RqdY6I6mJ7NnAScEJmvhMR3YAnI+KB4nhf4MzM/GJEHAXsAXwMCOCBiDgEWAScCuxL7e/4OeDZzPxFRJwHXJCZkxsqJCLOAc4B6NatO5fsXdMCt6uyqaqqAmDo0KEMHjyYzp07s8suu/DXv/6Vt99+m2nTpjFmzBhmzJjB8ccfz89+9jNmzpzJb37zG374wx/SqVMnzj//fNq3b8+gQYPa9ma0UVi4cOGqcSWtD8eQmsPxo+ZyDBn03rfa0s2I6AD8ZxHaVgK9gB2Kw3/MzCeL7aOKn+eL/S7UBr9tgPsyc3FxvfdD4gfKzJuAmwB26bN7XjPNP5E+2JwRlQBUVlZy1VVXATBixAgOOugg3n33Xb785S9TWVnJYYcdxtVXX82AAQN47bXXWLJkCUOHDgXgmWeeYeXKlVRWVrbRXWhjUlVV5VhQsziG1ByOHzWXY8ilm40ZAXQHBhUB8DVgy+LYojr9AhidmRXFz+6ZeUtxLFutWqnw/otW/vSnPzFp0iSGDx/Opz71KR577DGgdrnmsmXL6NatG0cffTRTp05l8eLF1NTU8Pjjj9OvX7+2LF+SJEkbSJOmiyJiN2BuZi6NiEpgIPCTzFzQcqW1qW2B+Zm5PCIOA3ZtpN9DwGURcUdmLoyIXsByYCJwW0SMofZ3fBzw38U571I74ydtcCeeeOKqt2d+5StfYbvttuOss87irLPOYsCAAXTs2JGxY8cSEWy33XZ87Wtf4x//8R+JCP7pn/6JIUP8Pj5JkqQyaOq6wF8C+0fE7sAtwAPAz4B/aqnC2tgdwK8iYjJQDcxoqFNmPhwRewFP1L6Ek4XAaZn5XETcVZz7R2BSndNuA34YEUuAwZm5pLEiOndoz0y/CFvrYNKkvw+199eld+zYkdtvv73B/qeddhqnnXZaa5QmSZKkVtTUoLcyM2si4gTg2sy8PiKe/8CzNhGZ2aXe/hvA4Ea6D6jX9zrgugaueTlweQPtv6Q2OEuSJElSi2jqM3rLI2I4cDowvmjzm5UlSZIkaSPU1KB3JrUzXJdn5uyI+AjQ8FowSZIkSVKbatLSzcx8MSL+Hdil2J8NjGnJwiRJkiRJ66dJM3oRcRy1LxaZUOxXrMt3w0mSJEmSWk9Tl26OAj4GLADIzGrgIy1SkSRJkiSpWZoa9Goy8+16bX4huCRJkiRthJr69QrTI+LTQPuI2AP4MvD7litLkiRJkrS+mjqj969Af2AptV+U/jbw1RaqSZIkSZLUDB84oxcR7YEHMvMTwIUtX5IkSZIkqTk+cEYvM1cAiyNi21aoR5IkSZLUTE19Ru89YFpEPAIser8xM7/cIlVJkiRJktZbU4Peg8WPJEmSJGkj16Sgl5ljW7oQSZIkSdKG0aSgFxGzaeB78zKzzwavSJIkSZLULE1durl/ne0tgZOA7Td8OZIkSZKk5mrS9+hl5pt1fl7NzGuBw1u2NEmSJEnS+mjq0s396uy2o3aGb5sWqUiSJEmS1CxNXbp5TZ3tGmA2cPKGL0eSJEmS1FxNDXqfy8xX6jZExEdaoB5JkiRJUjM16Rk94BdNbJMkSZIktbG1zuhFxJ5Af2DbiPjnOoc+RO3bNyVJkiRJG5kPWrrZF/gk0BU4rk77u8DnW6gmSZIkSVIzrDXoZeY4YFxEDM7MJ1qpJkmSJElSMzT1ZSzPR8SXqF3GuWrJZmae1SJVSZIkSZLWW1NfxvJT4B+Ao4HHgZ2oXb4pSZIkSdrINDXo7Z6ZFwOLMnMsMATYu+XKkiRJkiStr6YGveXFPxdExABgW6B3i1QkSZIkSWqWpj6jd1NEbAdcDDwAdAEuabGqJEmSJEnrrUlBLzN/VGw+DvRpuXIkSZIkSc3VpKWbEbFDRNwSEf9T7PeLiM+1bGmSJEmSpPXR1Gf0bgMeAnYs9mcBX22BeiRJkiRJzdTUoNctM+8GVgJkZg2wosWqkiRJkiStt6YGvUUR8WEgASLiQODtFqtKkiRJkrTemvrWza9R+7bN3SLif4HuwLAWq0qSJEmStN7WGvQiYpfM/FNmPhcRhwJ9gQBmZubytZ2r5luyfAW9Rz7Y1mVoEzBnzBCuu+46br75ZjKTz3/+81RUVABw/fXXc8MNN7DFFlswZMgQrrzySh555BFGjhzJsmXL6NixI1dddRWHH354296EJEmSNpgPmtG7H9iv2L4rM09s2XLaXkScANwL7JWZMzbA9XoD4zNzQERUADtm5q+be12prunTp3PzzTfz9NNP07FjR4455hi6devGb3/7W8aNG8fUqVPp1KkT8+fPB6Bbt2786le/Yscdd2T69OkcffTRvPrqq218F5IkSdpQPijoRZ3tzeX784YDvwNOBUZt4GtXAPsDBj1tUC+99BIHHnggW221FQCHHnookyZN4m9/+xsjR46kU6dOAPTo0QOAfffdd9W5/fv357333mPp0qWr+kmSJGnT9kEvY8lGtkspIroABwGfozboERHtIuL7EfFCRIyPiF9HxLDi2KCIeDwino2IhyKiZ532KRHxBPCloq0j8B/AKRFRHRGntMU9qpwGDBjAxIkTefPNN1m8eDG//vWvef3115k1axaTJk3igAMO4NBDD+WZZ55Z49xf/vKX7LvvvoY8SZKkEvmgGb19IuIdamf2OhfbFPuZmR9q0epa36eACZk5KyLeioj9qJ3J7A3sDfQAXgJujYgOwPXA0Mx8vQhulwNnAT8G/jUzH4+IqwAyc1lEXALsn5nnNVZARJwDnAPQrVt3Ltm7poVuVWXy2muvMXToUAYPHkznzp3ZddddWbFiBW+//TbTpk1jzJgxzJgxg+OPP56f/exnRNRO1s+ePZuLLrqIK6+8kqqqqra9CW1UFi5c6JhQsziG1ByOHzWXYwgis/QTdU0WEQ8C12bmIxHxZWBnoAMwJTN/XPS5F/gZMAP4PfBKcXp7YB5wEjAtM3cp+g8EflY8o3cGHxD06tqlz+7Z7uTrNtj9qbzmjBmy2v63vvUtFi9ezIwZMxg5ciSVlZUA7Lbbbjz55JN0796duXPncvjhh/PjH/+Ygw46qA2q1sasqqpq1biR1odjSM3h+FFzbU5jKCKezcz967c39esVSq/4nsDDgQERkdQGtwTua+wU4IXMHFzvOl3ZDJa5auMzf/58evTowZ/+9CfuvfderrrqKvbcc08ee+wxKisrmTVrFsuWLaNbt24sWLCAIUOGMHr0aEOeJElSCTX1C9M3B8OAn2TmrpnZOzN3BmYDbwAnFs/q7QBUFv1nAt0jYjBARHSIiP6ZuQB4OyIOLvqNqPMZ7wLbtMK9aDN04okn0q9fP4477jhuvPFGttlmG8466yxeeeUVBgwYwKmnnsrYsWOJCG644Qb+8Ic/cNlll1FRUUFFRcWqN3JKkiRp0+eM3t8NB8bUa/slsBcwF5gOzAKeAt4unrkbBnwvIral9nd5LfACcCa1z/EtBh6qc73fAiMjohoYnZl3tdztaHMzadKk1farqqro2LEjt99++xp9L7roIi666KLWKk2SJEmtzKBXyMzKBtq+B7Vv48zMhcXyzqeBacXxauCQBs57FtinTtOoov0t4B+bWlPnDu2ZWe/ZK0mSJEn6IAa9phlfPHvXEbgsM//axvVIkiRJUqMMek3Q0GyfJEmSJG2sfBmLJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqmS3augA1bsnyFfQe+WBbl1Fac8YMaesSJEmSpBbhjJ42ewsWLGDYsGHsueee7LXXXjzxxBPcc8899O/fn3bt2jF58uRVfefMmUPnzp2pqKigoqKCc889tw0rlyRJkhrWojN6EXEh8GlgBbAS+EJmPtWSnymtq6985Sscc8wx/OIXv2DZsmUsXryYrl27cu+99/KFL3xhjf677bYb1dXVrV+oJEmS1EQtFvQiYjDwSWC/zFwaEd2Aji31edL6eOedd5g4cSK33XYbAB07dqRjx4507dq1TeuSJEmSmqMll272BN7IzKUAmflGZv4lIgZFxOMR8WxEPBQRPSNi24iYGRF9ASLizoj4fGMXjoiFEXFFcY3fRMTHIqIqIl6JiOOLPu0j4qqIeCYipkbEF4r2LhHxaEQ8FxHTImJo0d47Il6KiJsj4oWIeDgiOq+lht0iYkJRw6SI2LNovy0ivhcRvy/qGVbnnG8UnzklIsZsgN+xmumVV16he/funHnmmey7776cffbZLFq0aK3nzJ49m3333ZdDDz2USZMmtVKlkiRJUtNFZrbMhSO6AL8DtgJ+A9wF/B54HBiama9HxCnA0Zl5VkQcCfwHcB1wRmYes5ZrJ/BPmfk/EXEfsDUwBOgHjM3Miog4B+iRmd+JiE7A/wInAX8GtsrMd4pZxieBPYBdgT8A+2dmdUTcDTyQmbc3UsOjwLmZ+XJEHACMzszDI+K2op5TgD2La+weEccCFwOfyMzFEbF9Zr7VwHXPAc4B6Nat+6BLrr25Sb9vrbu9e23LzJkz+eIXv8j1119Pv379uP7669l6660566yzAPjqV7/Kv/zLv9C3b18Ali1bxpIlS9h229pzL774Yn784x+z9dZbt+WtNGjhwoV06dKlrcvQJsrxo+ZyDKk5HD9qrs1pDB122GHPZub+9dtbbOlmZi6MiEHAx4HDqA163wEGAI9EBEB7YF7R/5GIOAm4EdjnAy6/DJhQbE8Dlmbm8oiYBvQu2o8CBtaZUduW2kA3F/jPiDiE2ucGewE7FH1mZ2Z1sf1snWutpgix/w+4p7gPgE51utyfmSuBFyPi/Wt/AvhxZi4u7neNkFe03wTcBLBLn93zmmm+GLWlzBlRyZ577sno0aP54he/CED79u0ZM2YMlZWVAHTt2pVBgwax//5r/G+HyspK7rzzTnbYYYcGj7e1qqqqVfchrSvHj5rLMaTmcPyouRxDLfwylsxcAVQBVUUI+xLwQmYOrt83ItoBewFLgO2pDWSNWZ5/n4pcCby/PHRlRLx/TwH8a2Y+VO9zzgC6A4OKcDgH2LI4vLRO1xVAY0s32wELMrOikeN1rxN1/tky06dab//wD//AzjvvzMyZM+nbty+PPvoo/fr1a7T/66+/zvbbb0/79u155ZVXePnll+nTp08rVixJkiR9sBZ7Ri8i+kbEHnWaKoCXgO7Fi1qIiA4R0b84/m/F8eHArRHRoZklPAT8y/vXiYiPRsTW1M7szS9C3mHULtlcJ5n5DjC7mIEkan3QLOTDwFkRsVVxzvbr+rlqGddffz0jRoxg4MCBVFdX861vfYv77ruPnXbaiSeeeIIhQ4Zw9NFHAzBx4kQGDhzIPvvsw7Bhw/jhD3/I9tv7p5QkSdLGpSVn9LoA10dEV6CG2uffzqF2WeL3ImLb4vOvjYjlwNnAxzLz3YiYCFwEfLsZn/8japdePhe16ytfBz4F3AH8KiImA9XAjPW8/gjgBxFxEdAB+DkwpbHOmTkhIiqAyRGxDPg18K21fUDnDu2Z6Zd6t7iKiorVvisP4IQTTuCEE05Yo++JJ57IiSee2FqlSZIkSeulJZ/Re5ba59jqewM4pIH2veqc+7UPuHaXOtujGjpWPCP3LRoOU2ssHS0MqHOdqz+ghtnAGi+Mycwz1lLrGMC3bUqSJElqUS359QqSJEmSpDawUb/SMSKeYvW3WQJ8JjOntWINNwIH1Wu+LjN/3Fo1SJIkSdK62KiDXmYesBHU8KW2rkGSJEmS1oVLNyVJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSSMehJkiRJUskY9CRJkiSpZAx6kiRJklQyBj1JkiRJKhmDniRJkiSVjEFPkiRJkkrGoCdJkiRJJWPQkyRJkqSS2aKtC1DjlixfQe+RD7Z1GZuEOWOGANC7d2+22WYb2rdvzxZbbMHkyZP5+te/zq9+9Ss6duzIbrvtxo9//GO6du266tw//elP9OvXj1GjRnHBBRe00R1IkiRJG44zeg2IiBURUR0R0yPinojYqpnX6x0R0zdUfVq73/72t1RXVzN58mQAjjzySKZPn87UqVP56Ec/yujRo1fr/2//9m8ce+yxbVGqJEmS1CIMeg1bkpkVmTkAWAac25STIsIZ0o3QUUcdxRZb1P5pDjzwQObOnbvq2P3330+fPn3o379/W5UnSZIkbXAGvQ82Cdg9Io6LiKci4vmI+E1E7AAQEaMi4qaIeBj4SUTsEBH3RcSU4uf/FddpHxE3R8QLEfFwRHRuszsqsYjgqKOOYtCgQdx0001rHL/11ltXzd4tWrSIK664gm9/+9utXaYkSZLUopyBWotihu5YYALwO+DAzMyIOBv4BnB+0XUQcHBmLomIu4DHM/OEiGgPdAG2A/YAhmfm5yPibuBE4PYGPvMc4ByAbt26c8neNS17kyVRVVUFwFVXXUW3bt3429/+xgUXXMCSJUvYZ599ALj99ttZsGABvXr1oqqqih/84AccddRRTJ48mTlz5tC5c+dV1ymDhQsXlup+1LocP2oux5Caw/Gj5nIMGfQa0zkiqovtScAtQF/grojoCXQEZtfp/0BmLim2Dwc+C5CZK4C3I2I7YHZmvn/NZ4HeDX1wZt4E3ASwS5/d85pp/omaYs6IyjXapkyZwvLly6msrGTs2LG88MILPProo2y1Ve0jlxdffDFPPfUUY8eOZcGCBbRr147+/ftz3nnntXL1LaOqqorKysq2LkObKMePmssxpOZw/Ki5HEMGvcYsycyKug0RcT3w3cx8ICIqgVF1Di9qwjWX1tleAbh0cwNbtGgRK1euZJtttmHRokU8/PDDXHLJJUyYMIErrriCxx9/fFXIA5g0adKq7VGjRtGlS5fShDxJkiRt3gx6Tbct8Gqxffpa+j0K/AtwbbF0c+uWLky1XnvtNU444QQAampq+PSnP80xxxzD7rvvztKlSznyyCOB2hey/PCHP2zLUiVJkqQWZdBrulHAPRHxKvAk8JFG+n0FuCkiPkftzN2/APNapcLNXJ8+fZgyZcoa7X/4wx8+8NxRo0a1QEWSJElS2zDoNSAzuzTQNg4Y10D7qHr7rwFDG7jsgDp9rm5KHZ07tGdm8UXgkiRJktRUfr2CJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSmaLti5AjVuyfAW9Rz7Y1mVsNOaMGbJqe8WKFey///706tWL8ePHc8oppzBz5kwAFixYQNeuXamuruaOO+7gqquuWnXe1KlTee6556ioqGjt8iVJkqRWY9BrQEScANwL7JWZMzbwtbsD44GOwJczc9KGvP7m4rrrrmOvvfbinXfeAeCuu+5adez8889n2223BWDEiBGMGDECgGnTpjF06FBDniRJkkrPpZsNGw78Dji1Ba59BDAjM/c15K2fuXPn8uCDD3L22WevcSwzufvuuxk+fPgax+68884G2yVJkqSyMejVExFdgIOAz1EEvYhoFxHfj4gXImJ8RPw6IoYVxwZFxOMR8WxEPBQRPYv23SJiQtE+KSL2jIgK4ErgnyKiOiI6t81dbtq++tWvcuWVV9Ku3ZrDd9KkSeywww7sscceaxy76667DHqSJEnaLBj01vQpYEJmzgLeioj9gH8GegN7A2cDgwEiogNwPTAsMwcBtwKXF9e5CfjXov0C4PuZWQ1cAtyVmRWZuaS1bqosxo8fT48ePRg0aFCDxxubtXvqqafYaqutGDBgQEuXKEmSJLW5yMy2rmGjEhEPAtdm5iMR8WVgZ6ADMCUzf1z0uRf4GTAD+D3wSnF6e2AetcHwdWBmnUt3ysy9IuIMYP/MPK+Rzz8HOAegW7fugy659uYNfIebrr17bcvNN9/Mww8/TPv27Vm2bBmLFy/m4x//OBdeeCErVqzgpJNO4r//+7/p3r37aufeeOONbLvttpx22mltVH3rW7hwIV26dGnrMrSJcvyouRxDag7Hj5prcxpDhx122LOZuX/9dl/GUkdEfBg4HBgQEUltcEvgvsZOAV7IzMH1rvMhYEFmVqxrDZl5E7WzgezSZ/e8Zpp/ovfNGVFJZWXlqv2qqiquvvpqxo8fD8CECRPYe++9Oemkk1Y7b+XKlZx22mlMnDiRPn36tGbJbaqqqmq135e0Lhw/ai7HkJrD8aPmcgy5dLO+YcBPMnPXzOydmTsDs4E3gBOLZ/V2ACqL/jOB7hGxailnRPTPzHeA2RFxUtEeEbFPq9/NZubnP/95g8s2J06cyE477bRZhTxJkiRt3pwuWt1wYEy9tl8CewFzgenALOAp4O3MXFa8lOV7EbEttb/Pa4EXgBHADyLiImqXfv4cmNIaN7G5qKxcfYbvtttua7Tfk08+2TpFSZIkSRsBg14dmVnZQNv3oPZtnJm5sFje+TQwrTheDRzSwHmzgWMaaL8NuK0p9XTu0J6Zdb4kXJIkSZKawqDXdOMjoiu1X3R+WWb+tY3rkSRJkqQGGfSaqKHZPkmSJEnaGPkyFkmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMQU+SJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqmS3augA1bsnyFfQe+WBbl9Fkc8YM4b333uOQQw5h6dKl1NTUMGzYMC699FLeeustTjnlFObMmUPv3r25++672W677bjjjju46qqrVl1j6tSpPPfcc1RUVLTdjUiSJEmbOGf0ChGxIiKq6/z0bsHPOikiXoqI37bUZ7SVTp068dhjjzFlyhSqq6uZMGECTz75JGPGjOGII47g5Zdf5ogjjmDMmDEAjBgxgurqaqqrq/npT39K7969DXmSJElSMxn0/m5JZlbU+ZnzQSdErfX5HX4O+GJmHrYe527UIoIuXboAsHz5cpYvX05EMG7cOE4//XQATj/9dO6///41zr3zzjsZPnx4a5YrSZIklZJBrxER0SUiHo2I5yJiWkQMLdp7F7Nx3weeA3aOiK9HxDMRMTUiLq1zjdMi4ulihvC/I6J9RFwCHAz8MCKuavjTN20rVqygoqKCHj16cOSRR3LAAQfw2muv0bNnTwB69uzJ/Pnz1zjvrrvuMuhJkiRJG4DP6P1d54ioLrZnAycBJ2TmOxHRDXgyIh4ojvcFzszML0bEUcAewMeAAB6IiEOA14FTgIMyc3kRDEdk5n9ExOHABZk5uX4REXEOcA5At27duWTvmha74Q2tqqpq1fa1117LwoULufjii9lzzz2pqalZ7Xj9/RdffJHM5I033litXetv4cKF/i613hw/ai7HkJrD8aPmcgwZ9OpakpkV7+9ERAfgP4vQthLoBexQHP5jZj5ZbB9V/Dxf7HehNvgNBAYBz0QEQGdgzWmsejLzJuAmgF367J7XTNt0/kRzRlSu0fbss8/y5ptv0qtXL/r27UvPnj2ZN28eO+64I5WVf+8/btw4zj777NXa1DxVVVX+PrXeHD9qLseQmsPxo+ZyDLl0c21GAN2BQUUAfA3Ysji2qE6/AEbXebZv98y8pWgfW6e9b2aOasX628Trr7/OggULAFiyZAm/+c1v2HPPPTn++OMZO3YsAGPHjmXo0KGrzlm5ciX33HMPp556aluULEmSJJXOpjNd1Pq2BeYXyy4PA3ZtpN9DwGURcUdmLoyIXsBy4FFgXET8V2bOj4jtgW0y84+tU37bmDdvHqeffjorVqxg5cqVnHzyyXzyk59k8ODBnHzyydxyyy3ssssu3HPPPavOmThxIjvttBN9+vRpw8olSZKk8jDoNe4O4FcRMRmoBmY01CkzH46IvYAniiWaC4HTMvPFiLgIeLh4M+dy4EtAqYPewIEDef7559do//CHP8yjjz7a4DmVlZU8+eSTDR6TJEmStO4MeoXM7FJv/w1gcCPdB9Trex1wXQPXvAu4q4H2yqbU1LlDe2aOGdKUrpIkSZK0is/oSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSVj0JMkSZKkkjHoSZIkSVLJGPQkSZIkqWQMepIkSZJUMgY9SZIkSSoZg54kSZIklYxBT5IkSZJKxqAnSZIkSSUTmdnWNagREfEuMLOt69AmqxvwRlsXoU2W40fN5RhSczh+1Fyb0xjaNTO712/coi0qUZPNzMz927oIbZoiYrLjR+vL8aPmcgypORw/ai7HkEs3JUmSJKl0DHqSJEmSVDIGvY3bTW1dgDZpjh81h+NHzeUYUnM4ftRcm/0Y8mUskiRJklQyzuhJkiRJUskY9DZCEXFMRMyMiD9ExMi2rkcbh4i4NSLmR8T0Om3bR8QjEfFy8c/t6hz7ZjGGZkbE0XXaB0XEtOLY9yIiWvte1PoiYueI+G1EvBQRL0TEV4p2x5CaJCK2jIinI2JKMYYuLdodQ2qyiGgfEc9HxPhi3/GjJouIOcXfvjoiJhdtjqFGGPQ2MhHRHrgROBboBwyPiH5tW5U2ErcBx9RrGwk8mpl7AI8W+xRj5lSgf3HO94uxBfAD4Bxgj+Kn/jVVTjXA+Zm5F3Ag8KVinDiG1FRLgcMzcx+gAjgmIg7EMaR18xXgpTr7jh+tq8Mys6LOVyc4hhph0Nv4fAz4Q2a+kpnLgJ8DQ9u4Jm0EMnMi8Fa95qHA2GJ7LPCpOu0/z8ylmTkb+APwsYjoCXwoM5/I2gd0f1LnHJVYZs7LzOeK7Xep/RetXjiG1ERZa2Gx26H4SRxDaqKI2AkYAvyoTrPjR83lGGqEQW/j0wv4c539uUWb1JAdMnMe1P6LPNCjaG9sHPUqtuu3azMSEb2BfYGncAxpHRTL7qqB+cAjmekY0rq4FvgGsLJOm+NH6yKBhyPi2Yg4p2hzDDVii7YuQGtoaI2wr0bVumpsHDm+NnMR0QX4JfDVzHxnLY8lOIa0hsxcAVRERFfgvogYsJbujiGtEhGfBOZn5rMRUdmUUxpoc/zooMz8S0T0AB6JiBlr6bvZjyFn9DY+c4Gd6+zvBPyljWrRxu+1YgkCxT/nF+2NjaO5xXb9dm0GIqIDtSHvjsy8t2h2DGmdZeYCoIra51ocQ2qKg4DjI2IOtY+lHB4Rt+P40TrIzL8U/5wP3EftI0+OoUYY9DY+zwB7RMRHIqIjtQ+RPtDGNWnj9QBwerF9OjCuTvupEdEpIj5C7YPGTxdLGt6NiAOLN0x9ts45KrHi730L8FJmfrfOIceQmiQiuhczeUREZ+ATwAwcQ2qCzPxmZu6Umb2p/XebxzLzNBw/aqKI2Doitnl/GzgKmI5jqFEu3dzIZGZNRJwHPAS0B27NzBfauCxtBCLiTqAS6BYRc4FvA2OAuyPic8CfgJMAMvOFiLgbeJHaty1+qVhyBfAv1L7BszPwP8WPyu8g4DPAtOIZK4Bv4RhS0/UExhZvrWsH3J2Z4yPiCRxDWn/+f5Caagdql4xDbYb5WWZOiIhncAw1KGpfNiNJkiRJKguXbkqSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJWMX68gSVIriIgVwLQ6TZ/KzDltVI4kqeT8egVJklpBRCzMzC6t+HlbZGZNa32eJGnj4tJNSZI2AhHRMyImRkR1REyPiI8X7cdExHMRMSUiHi3ato+I+yNiakQ8GREDi/ZREXFTRDwM/CQiukfELyPimeLnoDa8RUlSK3LppiRJraNzRFQX27Mz84R6xz8NPJSZl0dEe2CriOgO3AwckpmzI2L7ou+lwPOZ+amIOBz4CVBRHBsEHJyZSyLiZ8B/ZebvImIX4CFgrxa7Q0nSRsOgJ0lS61iSmRVrOf4McGtEdADuz8zqiKgEJmbmbIDMfKvoezBwYtH2WER8OCK2LY49kJlLiu1PAP0i4v3P+FBEbJOZ726om5IkbZwMepIkbQQyc2JEHAIMAX4aEVcBC4CGHqaPBtre77eoTls7YHCd4CdJ2kz4jJ4kSRuBiNgVmJ+ZNwO3APsBTwCHRsRHij7vL92cCIwo2iqBNzLznQYu+zBwXp3PqGih8iVJGxln9CRJ2jhUAl+PiOXAQuCzmfl6RJwD3BsR7YD5wJHAKODHETEVWAyc3sg1vwzcWPTbgtqAeG6L3oUkaaPg1ytIkiRJUsm4dFOSJEmSSsagJ0mSJEklY9CTJEmSpJIx6EmSJElSyRj0JEmSJKlkDHqSJEmSVDIGPUmSJEkqGYOeJEmSJJXM/wc9ba2BxhFrnQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1008x648 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from xgboost import plot_importance\n",
    "\"\"\"Plotting Feature Importance\"\"\"\n",
    "og_names = xgbf.get_booster().feature_names \n",
    "xgbf.get_booster().feature_names = train_df.drop('Survived',1).columns.tolist()\n",
    "fig, ax = plt.subplots(1,1,figsize=(14,9))\n",
    "plot_importance(xgbf,ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.9140625\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Survived       0.91      0.96      0.94        84\n",
      "    Survived       0.92      0.82      0.87        44\n",
      "\n",
      "    accuracy                           0.91       128\n",
      "   macro avg       0.92      0.89      0.90       128\n",
      "weighted avg       0.91      0.91      0.91       128\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "\"\"\"Accuracy Score and Classification report\"\"\"\n",
    "\n",
    "xgbf.get_booster().feature_names = og_names\n",
    "y_pred = xgbf.predict(X_test)\n",
    "print('Accuracy = {}'.format(accuracy_score(y_test, y_pred)))\n",
    "print(classification_report(y_test, y_pred, target_names=['Not Survived','Survived']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # optional: the following cell is for prediction on given test set from kaggle\n",
    "\n",
    "# test_df = pd.read_csv('test.csv')\n",
    "# \"basic view of test dataset\"\n",
    "# test_df_0 = test_df.copy()\n",
    "# test_df.head(5)\n",
    "\n",
    "# pret_pipe = Pipeline([(\"custom_pr\",custom_pr(drop_=False)),\n",
    "#                      (\"mean_encoder\", mean_encoder(None ,['Sex'],encoding_g)),\n",
    "#                      (\"dropcol\", Drop_Col(['Ticket','Name','PassengerId','Embarked','Cabin','Sex'])),\n",
    "#                       ('cv_discretizer', cv_discretizer(['Age','Fare'],use_global=True)),\n",
    "#                      ])\n",
    "# test_df = pret_pipe.fit_transform(test_df).astype('float32')\n",
    "# X_tf = test_df.to_numpy()\n",
    "# xgbf.predict(X_tf)\n",
    "\n",
    "# data = {'PassengerId':test_df_0.PassengerId.to_list(),'Survived':xgbf.predict(X_tf).tolist()}\n",
    "# pd.DataFrame(data = data).astype('int16').to_csv('Submission7.csv',index=False,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}