airbnbModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load in our libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import sklearn\n",
    "#!C:/Users/sdk19/anaconda3/python -m pip install xgboost\n",
    "#!C:/Users/sdk19/anaconda3/python -m pip install plotly    \n",
    "#!C:/Users/sdk19/anaconda3/python -m pip install sklearn.cross_validation\n",
    "import xgboost as xgb\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import plotly.offline as py\n",
    "py.init_notebook_mode(connected=True)\n",
    "import plotly.graph_objs as go\n",
    "import plotly.tools as tls\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.ensemble import (RandomForestClassifier, AdaBoostClassifier, \n",
    "                              GradientBoostingClassifier, ExtraTreesClassifier)\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import KFold\n",
    "import numpy as np  # linear algebra\n",
    "import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from subprocess import check_output\n",
    "#!apt-get install libgomp1 \n",
    "from sklearn import metrics \n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from xgboost import plot_importance\n",
    "from matplotlib import pyplot\n",
    "from numpy import sort\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from numpy.random import seed\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "import xgboost as xgb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>high_booking_rate</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>amenities</th>\n",
       "      <th>availability_30</th>\n",
       "      <th>availability_365</th>\n",
       "      <th>availability_60</th>\n",
       "      <th>availability_90</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bed_type</th>\n",
       "      <th>...</th>\n",
       "      <th>host_sinceD</th>\n",
       "      <th>market</th>\n",
       "      <th>property_type</th>\n",
       "      <th>review_scores_ratingOG</th>\n",
       "      <th>topic</th>\n",
       "      <th>price_score</th>\n",
       "      <th>Area</th>\n",
       "      <th>bins</th>\n",
       "      <th>rating_score</th>\n",
       "      <th>{randomControl}</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>19</td>\n",
       "      <td>67</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>...</td>\n",
       "      <td>359.0</td>\n",
       "      <td>Miami</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>90</td>\n",
       "      <td>6</td>\n",
       "      <td>73</td>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>113890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>...</td>\n",
       "      <td>2090.0</td>\n",
       "      <td>New York</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>98</td>\n",
       "      <td>17</td>\n",
       "      <td>87</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "      <td>116337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>...</td>\n",
       "      <td>2260.0</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>House</td>\n",
       "      <td>99</td>\n",
       "      <td>3</td>\n",
       "      <td>56</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>112297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>24</td>\n",
       "      <td>173</td>\n",
       "      <td>54</td>\n",
       "      <td>84</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>...</td>\n",
       "      <td>778.0</td>\n",
       "      <td>Minneapolis</td>\n",
       "      <td>Townhouse</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>125404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>64</td>\n",
       "      <td>34</td>\n",
       "      <td>64</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>...</td>\n",
       "      <td>440.0</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>House</td>\n",
       "      <td>100</td>\n",
       "      <td>5</td>\n",
       "      <td>84</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>112921</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 255 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  high_booking_rate  accommodates  amenities  availability_30  \\\n",
       "0           1                  0             4         26               19   \n",
       "1           2                  0             4         19                0   \n",
       "2           3                  0             4         18                0   \n",
       "3           4                  0             2         11               24   \n",
       "4           5                  0             2         12                4   \n",
       "\n",
       "   availability_365  availability_60  availability_90  bathrooms  bed_type  \\\n",
       "0                67               29               29        2.0  Real Bed   \n",
       "1                 0                0                0        1.0  Real Bed   \n",
       "2                 0                0                0        1.0  Real Bed   \n",
       "3               173               54               84        2.5  Real Bed   \n",
       "4                64               34               64        1.0  Real Bed   \n",
       "\n",
       "   ...  host_sinceD       market property_type  review_scores_ratingOG  topic  \\\n",
       "0  ...        359.0        Miami     Apartment                      90      6   \n",
       "1  ...       2090.0     New York     Apartment                      98     17   \n",
       "2  ...       2260.0  Los Angeles         House                      99      3   \n",
       "3  ...        778.0  Minneapolis     Townhouse                      20      2   \n",
       "4  ...        440.0  Los Angeles         House                     100      5   \n",
       "\n",
       "   price_score Area bins rating_score  {randomControl}  \n",
       "0           73   23    2           15           113890  \n",
       "1           87   15    1           58           116337  \n",
       "2           56   39    1           60           112297  \n",
       "3            8   22    1            1           125404  \n",
       "4           84   18    1           73           112921  \n",
       "\n",
       "[5 rows x 255 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df=pd.read_csv(\"completeData8.csv\",encoding='mac_roman')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>amenities</th>\n",
       "      <th>availability_30</th>\n",
       "      <th>availability_365</th>\n",
       "      <th>availability_60</th>\n",
       "      <th>availability_90</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bed_type</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>...</th>\n",
       "      <th>amenities_Treadmill</th>\n",
       "      <th>host_sinceD</th>\n",
       "      <th>market</th>\n",
       "      <th>property_type</th>\n",
       "      <th>review_scores_ratingOG</th>\n",
       "      <th>Area</th>\n",
       "      <th>bins</th>\n",
       "      <th>price_score</th>\n",
       "      <th>rating_score</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>27</td>\n",
       "      <td>361</td>\n",
       "      <td>57</td>\n",
       "      <td>87</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1290.0</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>House</td>\n",
       "      <td>93</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1822.0</td>\n",
       "      <td>Miami</td>\n",
       "      <td>House</td>\n",
       "      <td>98</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>245</td>\n",
       "      <td>14</td>\n",
       "      <td>25</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2501.0</td>\n",
       "      <td>The Big Island</td>\n",
       "      <td>Guest suite</td>\n",
       "      <td>100</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>79</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1624.0</td>\n",
       "      <td>San Diego</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>100</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>91</td>\n",
       "      <td>82</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Real Bed</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2079.0</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>House</td>\n",
       "      <td>100</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>85</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 385 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  accommodates  amenities  availability_30  availability_365  \\\n",
       "0           1             1         20               27               361   \n",
       "1           2             3         18                0                 1   \n",
       "2           3             2         22                5               245   \n",
       "3           4             2         20                0                 0   \n",
       "4           5             2          9                0                 0   \n",
       "\n",
       "   availability_60  availability_90  bathrooms  bed_type  bedrooms  ...  \\\n",
       "0               57               87        1.0  Real Bed         1  ...   \n",
       "1                0                0        1.0  Real Bed         1  ...   \n",
       "2               14               25        1.0  Real Bed         1  ...   \n",
       "3                0                0        1.0  Real Bed         0  ...   \n",
       "4                0                0        1.0  Real Bed         1  ...   \n",
       "\n",
       "   amenities_Treadmill host_sinceD          market  property_type  \\\n",
       "0                    0      1290.0     Los Angeles          House   \n",
       "1                    0      1822.0           Miami          House   \n",
       "2                    0      2501.0  The Big Island    Guest suite   \n",
       "3                    0      1624.0       San Diego      Apartment   \n",
       "4                    0      2079.0         Seattle          House   \n",
       "\n",
       "   review_scores_ratingOG Area bins price_score  rating_score topic  \n",
       "0                      93   18    1          28            27     0  \n",
       "1                      98   23    1          17            56     0  \n",
       "2                     100    7    1          11            79    19  \n",
       "3                     100   30    1          91            82    15  \n",
       "4                     100   46    1          27            85    17  \n",
       "\n",
       "[5 rows x 385 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dff=pd.read_csv(\"completeData2.csv\",encoding='mac_roman')\n",
    "#dff.head()\n",
    "df3=pd.read_csv(\"completeTestData2.csv\",encoding='mac_roman')\n",
    "df3.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating dummy columns for categorical columns of both training and testing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3=df3.iloc[:,1:]\n",
    "df3 = df3.drop('review_scores_rating', 1)\n",
    "df3=pd.get_dummies(df3, columns=[ 'bed_type','cancellation_policy','host_has_profile_pic','host_identity_verified', 'host_is_superhost', 'host_response_rate', 'host_response_time','instant_bookable',\n",
    "                                 'is_location_exact','require_guest_phone_verification','require_guest_profile_picture','requires_license', 'room_type', 'market', 'property_type','review_scores_accuracy','review_scores_checkin','review_scores_cleanliness','review_scores_communication','review_scores_location','topic','bins','Area','review_scores_value'])\n",
    "#df2=pd.get_dummies(df2, columns=[ 'cancellation_policy','host_identity_verified', 'host_is_superhost', 'host_response_rate', 'host_response_time', 'room_type', 'market', 'property_type','review_scores_accuracy','review_scores_checkin','review_scores_cleanliness','review_scores_communication','review_scores_location','topic'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2=df.iloc[:,1:]\n",
    "df2 = df2.drop('review_scores_rating', 1)\n",
    "df2=pd.get_dummies(df2, columns=[ 'bed_type','cancellation_policy','host_has_profile_pic','host_identity_verified', 'host_is_superhost', 'host_response_rate', 'host_response_time','instant_bookable',\n",
    "                                 'is_location_exact','require_guest_phone_verification','require_guest_profile_picture','requires_license', 'room_type', 'market', 'property_type','review_scores_accuracy','review_scores_checkin','review_scores_cleanliness','review_scores_communication','review_scores_location','review_scores_value'\n",
    ",'topic','bins','Area'])\n",
    "\n",
    "#df2=pd.get_dummies(df2, columns=[ 'bed_type','cancellation_policy','host_has_profile_pic','host_identity_verified', 'host_is_superhost', 'host_response_rate', 'host_response_time','instant_bookable',\n",
    "#                                 'is_location_exact','require_guest_phone_verification','require_guest_profile_picture','requires_license', 'room_type', 'market', 'property_type','topic','bins','Area'])\n",
    "\n",
    "\n",
    "\n",
    "#df2=pd.get_dummies(df2, columns=[ 'cancellation_policy','host_identity_verified', 'host_is_superhost', 'host_response_rate', 'host_response_time', 'room_type', 'market', 'property_type','review_scores_accuracy','review_scores_checkin','review_scores_cleanliness','review_scores_communication','review_scores_location','topic'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Renaming training and test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=df2.drop('high_booking_rate', 1)\n",
    "y_train=df2['high_booking_rate']\n",
    "X_test=df3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating untunned xgboost model for shortlisting important features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster=None, colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints=None,\n",
       "              learning_rate=0.300000012, max_delta_step=0, max_depth=6,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=0, num_parallel_tree=1,\n",
       "              objective='binary:logistic', random_state=42, reg_alpha=0,\n",
       "              reg_lambda=1, scale_pos_weight=1, seed=42, subsample=1,\n",
       "              tree_method=None, validate_parameters=False, verbose=True,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from numpy.random import seed\n",
    "mlp = XGBClassifier(seed=42,verbose=True)\n",
    "mlp.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sorting columns according to feature importance from xgboost."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 ['host_is_superhost_TRUE', 0.14447111]\n",
      "2 ['review_scores_value_medium', 0.08434792]\n",
      "3 ['review_scores_checkin_high', 0.0723559]\n",
      "4 ['review_scores_accuracy_high', 0.03077895]\n",
      "5 ['review_scores_value_high', 0.027872413]\n",
      "6 ['review_scores_ratingOG', 0.027493183]\n",
      "7 ['amenities_\"Self check-in\"', 0.026101973]\n",
      "8 ['review_scores_checkin_medium', 0.0226714]\n",
      "9 ['amenities_\"Hot water\"', 0.019800983]\n",
      "10 ['amenities_\"Coffee maker\"', 0.017909545]\n",
      "11 ['amenities_\"Family/kid friendly\"', 0.01732288]\n",
      "12 ['host_is_superhost_FALSE', 0.017318059]\n",
      "13 ['host_response_time_within an hour', 0.016312983]\n",
      "14 ['amenities_\"translation missing: en.hosting_amenity_50\"', 0.013060003]\n",
      "15 ['host_response_rate_unknown', 0.011206152]\n",
      "16 ['amenities_\"translation missing: en.hosting_amenity_49\"', 0.0091285305]\n",
      "17 ['amenities_\"Bedroom comforts\"', 0.008357264]\n",
      "18 ['review_scores_communication_medium', 0.008351697]\n",
      "19 ['host_listings_count', 0.0076994346]\n",
      "20 ['amenities_Internet', 0.007175032]\n",
      "21 ['property_type_Guest suite', 0.006974144]\n",
      "22 ['amenities_Washer', 0.006796336]\n",
      "23 ['amenities_\"Host greets you\"', 0.006787772]\n",
      "24 ['amenities_\"Hair dryer\"', 0.006576035]\n",
      "25 ['amenities_\"Body soap\"', 0.0063497904]\n",
      "26 ['review_scores_cleanliness_medium', 0.00599345]\n",
      "27 ['minimum_nights', 0.0059866686]\n",
      "28 ['review_scores_cleanliness_high', 0.005425896]\n",
      "29 ['amenities_\"24-hour check-in\"', 0.0054170415]\n",
      "30 ['review_scores_accuracy_medium', 0.0053989943]\n",
      "31 ['host_has_profile_pic_Unknown', 0.005392958]\n",
      "32 ['requires_license_False', 0.0051481817]\n",
      "33 ['amenities_Gym', 0.0044378173]\n",
      "34 ['amenities_Shampoo', 0.0044287774]\n",
      "35 ['cancellation_policy_super_strict_30', 0.0042533036]\n",
      "36 ['amenities_Dryer', 0.0041157296]\n",
      "37 ['availability_30', 0.0040918165]\n",
      "38 ['amenities', 0.0038775003]\n",
      "39 ['cancellation_policy_moderate', 0.003875989]\n",
      "40 ['host_sinceD', 0.0038389948]\n",
      "41 ['amenities_\"Safety card\"', 0.0038224694]\n",
      "42 ['cleaning_fee', 0.0038053112]\n",
      "43 ['amenities_\"Carbon monoxide detector\"', 0.0035195183]\n",
      "44 ['market_Miami', 0.0034442379]\n",
      "45 ['room_type_Entire home/apt', 0.0033547175]\n",
      "46 ['market_Austin', 0.003232529]\n",
      "47 ['amenities_Refrigerator', 0.0032115518]\n",
      "48 ['amenities_\"Dishes and silverware\"', 0.0032111262]\n",
      "49 ['market_The Big Island', 0.0031036697]\n",
      "50 ['guests_included', 0.0030651323]\n",
      "51 ['amenities_Kitchen', 0.003028134]\n",
      "52 ['cancellation_policy_flexible', 0.0029396059]\n",
      "53 ['amenities_\"Bath towel\"', 0.002939426]\n",
      "54 ['amenities_\"Cable TV\"', 0.002779998]\n",
      "55 ['amenities_Other', 0.0027428742]\n",
      "56 ['amenities_\"Long term stays allowed\"', 0.0027284012]\n",
      "57 ['amenities_\"Pets live on this property\"', 0.0026989724]\n",
      "58 ['Area_4', 0.0026777308]\n",
      "59 ['amenities_Keypad', 0.002637138]\n",
      "60 ['review_scores_communication_high', 0.0025874397]\n",
      "61 ['review_scores_value_low', 0.0025480806]\n",
      "62 ['amenities_Pool', 0.0024468754]\n",
      "63 ['require_guest_profile_picture_False', 0.0024437846]\n",
      "64 ['amenities_Oven', 0.0023514442]\n",
      "65 ['review_scores_location_medium', 0.0023143354]\n",
      "66 ['review_scores_accuracy_low', 0.0022933667]\n",
      "67 ['availability_90', 0.0022267378]\n",
      "68 ['amenities_\"Cleaning before checkout\"', 0.0021713648]\n",
      "69 ['instant_bookable_False', 0.0021005275]\n",
      "70 ['amenities_\"BBQ grill\"', 0.0020405338]\n",
      "71 ['bathrooms', 0.0020182517]\n",
      "72 ['property_type_Condominium', 0.0020052139]\n",
      "73 ['Area_24', 0.0020007647]\n",
      "74 ['price', 0.0019935078]\n",
      "75 ['market_New York', 0.0019573276]\n",
      "76 ['Area_27', 0.0019309062]\n",
      "77 ['amenities_\"Private entrance\"', 0.0019259062]\n",
      "78 ['market_Nashville', 0.0019253901]\n",
      "79 ['review_scores_location_high', 0.0019218477]\n",
      "80 ['amenities_\"Laptop friendly workspace\"', 0.0019203741]\n",
      "81 ['amenities_Beachfront', 0.0019168153]\n",
      "82 ['amenities_\"Full kitchen\"', 0.0019058739]\n",
      "83 ['amenities_\"Pack ín Play/travel crib\"', 0.0018988393]\n",
      "84 ['review_scores_cleanliness_low', 0.0018752562]\n",
      "85 ['host_identity_verified_TRUE', 0.0018723413]\n",
      "86 ['topic_6', 0.0018394085]\n",
      "87 ['topic_2', 0.0017861604]\n",
      "88 ['accommodates', 0.0017748345]\n",
      "89 ['price_score', 0.0017737016]\n",
      "90 ['market_New Orleans', 0.0017700738]\n",
      "91 ['security_deposit', 0.0017547288]\n",
      "92 ['is_location_exact_False', 0.0017307234]\n",
      "93 ['amenities_\"Extra pillows and blankets\"', 0.0016845115]\n",
      "94 ['Area_23', 0.0016548887]\n",
      "95 ['Area_18', 0.0016521905]\n",
      "96 ['amenities_Dog(s)', 0.0016290306]\n",
      "97 ['host_response_rate_90-95%', 0.0016269633]\n",
      "98 ['property_type_Apartment', 0.0016130997]\n",
      "99 ['amenities_Heating', 0.0015950408]\n",
      "100 ['host_has_profile_pic_TRUE', 0.0015768864]\n",
      "101 ['host_response_time_within a day', 0.0015710453]\n",
      "102 ['amenities_\"Smart TV\"', 0.0015687083]\n",
      "103 ['maximum_nights', 0.0015328005]\n",
      "104 ['availability_60', 0.0015135714]\n",
      "105 ['host_response_time_a few days or more', 0.0014975492]\n",
      "106 ['topic_9', 0.0014768471]\n",
      "107 ['amenities_Iron', 0.00147442]\n",
      "108 ['property_type_House', 0.0014699339]\n",
      "109 ['host_response_rate_50-89%', 0.0014696819]\n",
      "110 ['availability_365', 0.0014686998]\n",
      "111 ['amenities_\"Paid parking off premises\"', 0.0014658998]\n",
      "112 ['amenities_\"Lock on bedroom door\"', 0.0014604075]\n",
      "113 ['topic_7', 0.0014486143]\n",
      "114 ['room_type_Private room', 0.0014453896]\n",
      "115 ['Area_11', 0.0014372881]\n",
      "116 ['Area_46', 0.0014094275]\n",
      "117 ['latitude', 0.0013795121]\n",
      "118 ['amenities_\"Changing table\"', 0.0013679259]\n",
      "119 ['amenities_\"Garden or backyard\"', 0.0013263057]\n",
      "120 ['require_guest_phone_verification_False', 0.0013230348]\n",
      "121 ['topic_10', 0.0013217794]\n",
      "122 ['Area_38', 0.0013193815]\n",
      "123 ['amenities_\"Indoor fireplace\"', 0.0013175855]\n",
      "124 ['room_type_Shared room', 0.001314996]\n",
      "125 ['market_Las Vegas', 0.0012982563]\n",
      "126 ['amenities_\"Private living room\"', 0.0012967672]\n",
      "127 ['amenities_Microwave', 0.0012929864]\n",
      "128 ['review_scores_location_low', 0.001291103]\n",
      "129 ['amenities_Dishwasher', 0.0012867807]\n",
      "130 ['amenities_\"Gas oven\"', 0.001284155]\n",
      "131 ['amenities_\"Free street parking\"', 0.0012840902]\n",
      "132 ['host_identity_verified_FALSE', 0.0012817708]\n",
      "133 ['amenities_\"Fire extinguisher\"', 0.0012761827]\n",
      "134 ['amenities_Breakfast', 0.0012704893]\n",
      "135 ['rating_score', 0.0012619721]\n",
      "136 ['amenities_Stove', 0.0012486381]\n",
      "137 ['topic_19', 0.0012467271]\n",
      "138 ['amenities_\"Baby monitor\"', 0.0012333094]\n",
      "139 ['Area_15', 0.0012174406]\n",
      "140 ['host_response_rate_100%', 0.0012165806]\n",
      "141 ['market_Other', 0.0012009381]\n",
      "142 ['amenities_Waterfront', 0.0011931583]\n",
      "143 ['bed_type_Futon', 0.0011879085]\n",
      "144 ['amenities_\"Buzzer/wireless intercom\"', 0.0011771071]\n",
      "145 ['amenities_\"Luggage dropoff allowed\"', 0.0011726604]\n",
      "146 ['market_Chicago', 0.0011723195]\n",
      "147 ['amenities_Hangers', 0.0011676318]\n",
      "148 ['amenities_\"Building staff\"', 0.0011635299]\n",
      "149 ['market_Minneapolis', 0.0011546108]\n",
      "150 ['Area_14', 0.0011494685]\n",
      "151 ['bedrooms', 0.001147914]\n",
      "152 ['Area_13', 0.0011476345]\n",
      "153 ['extra_people', 0.0011467885]\n",
      "154 ['amenities_Elevator', 0.0011390697]\n",
      "155 ['Area_21', 0.0011268376]\n",
      "156 ['topic_3', 0.0011193528]\n",
      "157 ['amenities_\"Toilet paper\"', 0.001105658]\n",
      "158 ['amenities_\"Pets allowed\"', 0.0011015577]\n",
      "159 ['bed_type_Pull-out Sofa', 0.0011002464]\n",
      "160 ['amenities_\"Wide entrance for guests\"', 0.0010892022]\n",
      "161 ['host_verifications', 0.0010875422]\n",
      "162 ['amenities_\"Game console\"', 0.0010860512]\n",
      "163 ['{randomControl}', 0.0010859915]\n",
      "164 ['amenities_\"Smoke detector\"', 0.0010398834]\n",
      "165 ['amenities_Doorman', 0.0010357006]\n",
      "166 ['longitude', 0.0010339741]\n",
      "167 ['Area_26', 0.0010298488]\n",
      "168 ['bed_type_Real Bed', 0.001019503]\n",
      "169 ['bed_type_Couch', 0.001008947]\n",
      "170 ['host_response_rate_0-49%', 0.0010037539]\n",
      "171 ['amenities_\"First aid kit\"', 0.0010007492]\n",
      "172 ['amenities_Cat(s)', 0.0009987354]\n",
      "173 ['Area_43', 0.0009954325]\n",
      "174 ['market_D.C.', 0.0009928695]\n",
      "175 ['Area_25', 0.0009919613]\n",
      "176 ['topic_11', 0.000989619]\n",
      "177 ['amenities_\"Bed linens\"', 0.000978973]\n",
      "178 ['market_Denver', 0.0009773198]\n",
      "179 ['Area_49', 0.0009727817]\n",
      "180 ['amenities_\"Free parking on premises\"', 0.0009726048]\n",
      "181 ['Area_19', 0.0009719129]\n",
      "182 ['host_response_time_within a few hours', 0.0009693916]\n",
      "183 ['review_scores_communication_low', 0.0009690582]\n",
      "184 ['market_San Francisco', 0.00094419165]\n",
      "185 ['bins_1', 0.00094294886]\n",
      "186 ['amenities_\"Espresso machine\"', 0.00093908084]\n",
      "187 ['Area_36', 0.0009160621]\n",
      "188 ['topic_13', 0.0009158802]\n",
      "189 ['market_Los Angeles', 0.0009151433]\n",
      "190 ['Area_20', 0.0009142942]\n",
      "191 ['amenities_\"Well-lit path to entrance\"', 0.0009138258]\n",
      "192 ['amenities_\"Pillow-top mattress\"', 0.0009079586]\n",
      "193 ['amenities_TV', 0.00090564205]\n",
      "194 ['topic_14', 0.0009026475]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195 ['cancellation_policy_strict_14_with_grace_period', 0.00089880894]\n",
      "196 ['amenities_\"Childrenís books and toys\"', 0.00089362764]\n",
      "197 ['amenities_\"Room-darkening shades\"', 0.000892162]\n",
      "198 ['amenities_\"Accessible-height toilet\"', 0.00088744395]\n",
      "199 ['amenities_Crib', 0.00088632305]\n",
      "200 ['amenities_\"Electric profiling bed\"', 0.0008772286]\n",
      "201 ['Area_33', 0.0008640807]\n",
      "202 ['amenities_toilet\"', 0.0008619024]\n",
      "203 ['amenities_\"Pocket wifi\"', 0.00085276965]\n",
      "204 ['bins_3', 0.00085220777]\n",
      "205 ['amenities_\"Patio or balcony\"', 0.00085066736]\n",
      "206 ['beds', 0.0008331735]\n",
      "207 ['property_type_Other', 0.00083180284]\n",
      "208 ['topic_17', 0.00082245964]\n",
      "209 ['amenities_\"Handheld shower head\"', 0.0008205723]\n",
      "210 ['amenities_\"Fixed grab bars for shower\"', 0.00081586326]\n",
      "211 ['Area_35', 0.0008155307]\n",
      "212 ['room_type_Hotel room', 0.0008146815]\n",
      "213 ['topic_15', 0.0008116069]\n",
      "214 ['amenities_\"Smart lock\"', 0.0008075273]\n",
      "215 ['Area_34', 0.00080701016]\n",
      "216 ['amenities_\"Beach essentials\"', 0.0007858037]\n",
      "217 ['bed_type_Airbed', 0.0007743895]\n",
      "218 ['amenities_Wifi', 0.00076814217]\n",
      "219 ['amenities_\"Outlet covers\"', 0.0007641943]\n",
      "220 ['amenities_\"Ethernet connection\"', 0.0007497203]\n",
      "221 ['amenities_\"Pool with pool hoist\"', 0.0007494084]\n",
      "222 ['Area_16', 0.00074609293]\n",
      "223 ['amenities_\"Babysitter recommendations\"', 0.00073766964]\n",
      "224 ['amenities_Bathtub', 0.0007288583]\n",
      "225 ['amenities_\"Cooking basics\"', 0.00071527733]\n",
      "226 ['amenities_\"Air conditioning\"', 0.00071150874]\n",
      "227 ['amenities_\"Flat path to guest entrance\"', 0.0007087961]\n",
      "228 ['Area_39', 0.00070827093]\n",
      "229 ['amenities_\"Wheelchair accessible\"', 0.00070566946]\n",
      "230 ['amenities_\"Mountain view\"', 0.0007037719]\n",
      "231 ['amenities_\"Hot tub\"', 0.0007035392]\n",
      "232 ['amenities_\"Fireplace guards\"', 0.00068926584]\n",
      "233 ['amenities_Lockbox', 0.0006796258]\n",
      "234 ['amenities_\"Wide hallways\"', 0.0006785469]\n",
      "235 ['bins_2', 0.0006772977]\n",
      "236 ['amenities_\"Step-free shower\"', 0.0006685307]\n",
      "237 ['amenities_\"Suitable for events\"', 0.0006673404]\n",
      "238 ['amenities_\"Soaking tub\"', 0.0006659755]\n",
      "239 ['amenities_\"Single level home\"', 0.0006648078]\n",
      "240 ['Area_3', 0.00066356297]\n",
      "241 ['amenities_Printer', 0.0006589721]\n",
      "242 ['amenities_\"EV charger\"', 0.0006586442]\n",
      "243 ['review_scores_checkin_low', 0.0006434094]\n",
      "244 ['amenities_\"Childrenís dinnerware\"', 0.0006424675]\n",
      "245 ['amenities_\"Paid parking on premises\"', 0.0006361137]\n",
      "246 ['amenities_\"Wide doorway to guest bathroom\"', 0.00063099526]\n",
      "247 ['Area_41', 0.00062695093]\n",
      "248 ['amenities_Netflix', 0.00062100147]\n",
      "249 ['Area_2', 0.0006164107]\n",
      "250 ['amenities_Essentials', 0.0006156051]\n",
      "251 ['amenities_\"Ground floor access\"', 0.00061537133]\n",
      "252 ['topic_0', 0.00061302254]\n",
      "253 ['amenities_\"Smoking allowed\"', 0.0006073876]\n",
      "254 ['amenities_\"Accessible-height bed\"', 0.0006044292]\n",
      "255 ['amenities_\"Disabled parking spot\"', 0.00060207763]\n",
      "256 ['amenities_\"No stairs or steps to enter\"', 0.0005777478]\n",
      "257 ['amenities_\"Outdoor seating\"', 0.00057486835]\n",
      "258 ['Area_48', 0.00055808626]\n",
      "259 ['property_type_Hotel', 0.0005503237]\n",
      "260 ['property_type_Townhouse', 0.00053327973]\n",
      "261 ['amenities_\"Extra space around bed\"', 0.000521217]\n",
      "262 ['Area_22', 0.0005096141]\n",
      "263 ['Area_30', 0.00049123185]\n",
      "264 ['amenities_\"Memory foam mattress\"', 0.00047372846]\n",
      "265 ['amenities_\"Wide entryway\"', 0.00046651365]\n",
      "266 ['amenities_\"Hot water kettle\"', 0.00044790324]\n",
      "267 ['topic_5', 0.00044151972]\n",
      "268 ['amenities_Terrace', 0.0004305872]\n",
      "269 ['Area_50', 0.00041620384]\n",
      "270 ['Area_37', 0.0003890857]\n",
      "271 ['amenities_\"Window guards\"', 0.00037187085]\n",
      "272 ['market_East Bay, CA', 0.00035454024]\n",
      "273 ['amenities_\"Walk-in shower\"', 0.0003110945]\n",
      "274 ['amenities_\"Baby bath\"', 0.00028133125]\n",
      "275 ['amenities_\"Other pet(s)\"', 0.00027965312]\n",
      "276 ['amenities_Kitchenette', 0.00027702644]\n",
      "277 ['Area_12', 0.00024930073]\n",
      "278 ['amenities_\"Fixed grab bars for toilet\"', 0.00023842862]\n",
      "279 ['market_San Diego', 0.00022640593]\n",
      "280 ['topic_18', 0.00011167697]\n",
      "281 ['Area_6', 4.796131e-05]\n",
      "282 ['amenities_\"Satellite TV\"', 0.0]\n",
      "283 ['requires_license_True', 0.0]\n",
      "284 ['require_guest_profile_picture_True', 0.0]\n",
      "285 ['require_guest_phone_verification_True', 0.0]\n",
      "286 ['is_location_exact_True', 0.0]\n",
      "287 ['instant_bookable_True', 0.0]\n",
      "288 ['host_response_time_unknown', 0.0]\n",
      "289 ['host_is_superhost_Unknown', 0.0]\n",
      "290 ['host_identity_verified_Unknown', 0.0]\n",
      "291 ['host_has_profile_pic_FALSE', 0.0]\n",
      "292 ['cancellation_policy_super_strict_60', 0.0]\n",
      "293 ['cancellation_policy_strict', 0.0]\n",
      "294 ['cancellation_policy_luxury_super_strict_95', 0.0]\n",
      "295 ['amenities_\"Washer / Dryer\"', 0.0]\n",
      "296 ['cancellation_policy_luxury_super_strict_125', 0.0]\n",
      "297 ['cancellation_policy_luxury_no_refund', 0.0]\n",
      "298 ['cancellation_policy_luxury_moderate', 0.0]\n",
      "299 ['market_Boston', 0.0]\n",
      "300 ['amenities_Office', 0.0]\n",
      "301 ['amenities_\"Media room\"', 0.0]\n",
      "302 ['amenities_\"Projector and screen\"', 0.0]\n",
      "303 ['amenities_Windsurfers', 0.0]\n",
      "304 ['amenities_Bidet', 0.0]\n",
      "305 ['amenities_\"Fax machine\"', 0.0]\n",
      "306 ['amenities_\"Wine storage\"', 0.0]\n",
      "307 ['amenities_\"High-resolution computer monitor\"', 0.0]\n",
      "308 ['amenities_\"Lake access\"', 0.0]\n",
      "309 ['market_South Bay, CA', 0.0]\n",
      "310 ['market_Portland', 0.0]\n",
      "311 ['market_Providence', 0.0]\n",
      "312 ['Area_47', 0.0]\n",
      "313 ['Area_45', 0.0]\n",
      "314 ['Area_44', 0.0]\n",
      "315 ['Area_42', 0.0]\n",
      "316 ['Area_40', 0.0]\n",
      "317 ['Area_32', 0.0]\n",
      "318 ['Area_31', 0.0]\n",
      "319 ['Area_29', 0.0]\n",
      "320 ['Area_28', 0.0]\n",
      "321 ['Area_17', 0.0]\n",
      "322 ['Area_10', 0.0]\n",
      "323 ['Area_9', 0.0]\n",
      "324 ['Area_8', 0.0]\n",
      "325 ['Area_7', 0.0]\n",
      "326 ['Area_5', 0.0]\n",
      "327 ['Area_1', 0.0]\n",
      "328 ['topic_16', 0.0]\n",
      "329 ['topic_12', 0.0]\n",
      "330 ['topic_8', 0.0]\n",
      "331 ['topic_4', 0.0]\n",
      "332 ['topic_1', 0.0]\n",
      "333 ['amenities_\"Snorkeling equipment\"', 0.0]\n",
      "334 ['market_Seattle', 0.0]\n",
      "335 ['amenities_Mudroom', 0.0]\n",
      "336 ['amenities_\"Wet bar\"', 0.0]\n",
      "337 ['amenities_\"Ceiling fans\"', 0.0]\n",
      "338 ['amenities_\"DVD player\"', 0.0]\n",
      "339 ['amenities_\"Amazon Echo\"', 0.0]\n",
      "340 ['amenities_\"Shared pool\"', 0.0]\n",
      "341 ['amenities_\"Wide entrance\"', 0.0]\n",
      "342 ['amenities_\"Day bed\"', 0.0]\n",
      "343 ['amenities_\"Mini fridge\"', 0.0]\n",
      "344 ['amenities_\"High chair\"', 0.0]\n",
      "345 ['amenities_\"Fire pit\"', 0.0]\n",
      "346 ['amenities_\"HBO GO\"', 0.0]\n",
      "347 ['amenities_\"Convection oven\"', 0.0]\n",
      "348 ['amenities_\"Outdoor parking\"', 0.0]\n",
      "349 ['amenities_\"Sun loungers\"', 0.0]\n",
      "350 ['amenities_\"Rain shower\"', 0.0]\n",
      "351 ['amenities_\"Wireless intercom\"', 0.0]\n",
      "352 ['amenities_\"Dining area\"', 0.0]\n",
      "353 ['amenities_\"Bathtub with bath chair\"', 0.0]\n",
      "354 ['amenities_\"Table corner guards\"', 0.0]\n",
      "355 ['amenities_\"Formal dining area\"', 0.0]\n",
      "356 ['amenities_\"Sound system\"', 0.0]\n",
      "357 ['amenities_Balcony', 0.0]\n",
      "358 ['amenities_\"Wide clearance to shower', 0.0]\n",
      "359 ['amenities_\"Shower chair\"', 0.0]\n",
      "360 ['amenities_\"En suite bathroom\"', 0.0]\n",
      "361 ['amenities_\"Breakfast table\"', 0.0]\n",
      "362 ['amenities_\"Central air conditioning\"', 0.0]\n",
      "363 ['amenities_\"Ceiling fan\"', 0.0]\n",
      "364 ['amenities_\"Bathroom essentials\"', 0.0]\n",
      "365 ['amenities_\"Wine cooler\"', 0.0]\n",
      "366 ['amenities_\"Private hot tub\"', 0.0]\n",
      "367 ['amenities_\"Alfresco shower\"', 0.0]\n",
      "368 ['amenities_\"Ironing Board\"', 0.0]\n",
      "369 ['amenities_\"Stair gates\"', 0.0]\n",
      "370 ['amenities_\"Pool toys\"', 0.0]\n",
      "371 ['amenities_\"Murphy bed\"', 0.0]\n",
      "372 ['amenities_\"Outdoor kitchen\"', 0.0]\n",
      "373 ['amenities_\"Heated pool\"', 0.0]\n",
      "374 ['amenities_\"Shared hot tub\"', 0.0]\n",
      "375 ['amenities_\"Shared gym\"', 0.0]\n",
      "376 ['amenities_\"Garage parking\"', 0.0]\n",
      "377 ['amenities_Safe', 0.0]\n",
      "378 ['amenities_\"Warming drawer\"', 0.0]\n",
      "379 ['amenities_Hammock', 0.0]\n",
      "380 ['amenities_\"Ice Machine\"', 0.0]\n",
      "381 ['amenities_\"Tennis court\"', 0.0]\n",
      "382 ['amenities_\"Chef\\'s kitchen\"', 0.0]\n",
      "383 ['amenities_\"Jetted tub\"', 0.0]\n",
      "384 ['amenities_\"Beach view\"', 0.0]\n",
      "385 ['amenities_Toaster', 0.0]\n",
      "386 ['amenities_\"Private bathroom\"', 0.0]\n",
      "387 ['amenities_\"Double oven\"', 0.0]\n",
      "388 ['amenities_\"Heated floors\"', 0.0]\n",
      "389 ['amenities_\"Exercise equipment\"', 0.0]\n",
      "390 ['amenities_\"Private pool\"', 0.0]\n",
      "391 ['amenities_\"Firm mattress\"', 0.0]\n",
      "392 ['amenities_\"Breakfast bar\"', 0.0]\n",
      "393 ['amenities_Parking', 0.0]\n",
      "394 ['amenities_Ski-in/Ski-out', 0.0]\n",
      "395 ['amenities_\"Heat lamps\"', 0.0]\n"
     ]
    }
   ],
   "source": [
    "#fscore = mlp.best_estimator_.booster().get_fscore()\n",
    "sorted_idx = np.argsort(mlp.feature_importances_)[::-1]\n",
    "count=1\n",
    "col_names=[]\n",
    "for index in sorted_idx:\n",
    "    print(count,[X_train.columns[index], mlp.feature_importances_[index]]) \n",
    "    col_names.append(X_train.columns[index])\n",
    "    count=count+1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using just top 285 important columns according to feature important."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(df2.iloc[:,1:], df2['high_booking_rate'], random_state=42)\n",
    "df_new = X_train[col_names[:285]]\n",
    "\n",
    "X_test2=X_test[list(df_new)]\n",
    "#df_new.head()\n",
    "#df_new_test = X_test[col_names[:120]]\n",
    "#df_new.head()\n",
    "X_test2\n",
    "df_new_test=X_test2\n",
    "X_test=df_new_test\n",
    "X_train=df_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## parameter tunning\n",
    "### Learning rate and n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'mean_fit_time': array([ 583.88682707,  802.82598233, 1166.80305068, 1433.31632765,\n",
       "         1600.37407851, 1713.46820052, 1937.57878288,  570.55467852,\n",
       "          796.28323038, 1136.90775657, 1363.49388242, 1590.45010098,\n",
       "         1702.55376116, 1927.61271826,  569.78682494,  794.81109111,\n",
       "         1134.69968613, 1361.56975961, 1586.6384813 , 1700.93101962,\n",
       "         1929.82838122,  568.91457836,  796.62827547, 1133.3175923 ,\n",
       "         1363.14173428, 1587.93012842, 1649.70505047, 1855.77240745,\n",
       "          546.94404324,  765.14640649, 1092.11185193, 1309.72701597,\n",
       "         1530.19878999, 1641.12656029, 1855.88519382,  547.13357337,\n",
       "          764.31616863, 1091.82879559, 1309.44300167, 1526.95234227,\n",
       "         1637.64146368, 1584.81103897]),\n",
       "  'std_fit_time': array([  1.10486404,   6.05008145,  26.28883627,   7.96477106,\n",
       "           1.50115266,   2.19681263,   1.34674783,   1.09189318,\n",
       "           0.40964733,   1.21906128,   0.81523644,   2.19692481,\n",
       "           0.48329598,   1.56467685,   0.74941738,   0.87583502,\n",
       "           0.82473163,   0.78786135,   1.00665492,   0.74160305,\n",
       "           1.87164437,   0.33631768,   0.8461385 ,   0.73956185,\n",
       "           4.61729883,   8.06257015,   9.17088189,   1.71315664,\n",
       "           0.43040876,   1.14163383,   0.31559788,   0.37621717,\n",
       "           1.28545499,   1.59869966,   1.48051792,   0.59239249,\n",
       "           0.93291124,   0.31041907,   0.53415181,   1.55303287,\n",
       "           0.53331036, 143.92328357]),\n",
       "  'mean_score_time': array([1.60952592, 1.88949855, 2.50150379, 2.68168704, 2.92792972,\n",
       "         3.14295491, 3.47811246, 1.48367318, 1.83328366, 2.25686995,\n",
       "         2.50798043, 2.9519554 , 3.09666856, 3.33770625, 1.50684516,\n",
       "         1.78975121, 2.26250354, 2.58869505, 2.7688671 , 3.03583574,\n",
       "         3.31407205, 1.48369884, 1.7628483 , 2.24205152, 2.59754626,\n",
       "         2.89652101, 2.92659903, 3.26554441, 1.51605431, 1.78068376,\n",
       "         2.17020607, 2.56676022, 2.84056536, 2.99265496, 3.33774352,\n",
       "         1.49218233, 1.73169994, 2.16491461, 2.54164918, 2.83875132,\n",
       "         2.98716688, 2.55937107]),\n",
       "  'std_score_time': array([0.05171333, 0.04687161, 0.29573481, 0.03944786, 0.03424876,\n",
       "         0.02297655, 0.11097842, 0.02397656, 0.01563923, 0.09898807,\n",
       "         0.08415706, 0.03181716, 0.03363345, 0.10151844, 0.05151863,\n",
       "         0.0543629 , 0.03722419, 0.0212052 , 0.12022376, 0.00481502,\n",
       "         0.08849451, 0.02813424, 0.04394486, 0.06653693, 0.00833537,\n",
       "         0.13572743, 0.0117419 , 0.01520563, 0.03537853, 0.045967  ,\n",
       "         0.07511101, 0.02450237, 0.06916644, 0.02918339, 0.05400477,\n",
       "         0.03428103, 0.04283262, 0.04017782, 0.01158957, 0.01462062,\n",
       "         0.03667515, 0.27156228]),\n",
       "  'param_learning_rate': masked_array(data=[0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.05, 0.05,\n",
       "                     0.05, 0.05, 0.05, 0.05, 0.05, 0.1, 0.1, 0.1, 0.1, 0.1,\n",
       "                     0.1, 0.1, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15,\n",
       "                     0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.25, 0.25, 0.25,\n",
       "                     0.25, 0.25, 0.25, 0.25],\n",
       "               mask=[False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False],\n",
       "         fill_value='?',\n",
       "              dtype=object),\n",
       "  'param_n_estimators': masked_array(data=[500, 700, 1000, 1200, 1400, 1500, 1700, 500, 700, 1000,\n",
       "                     1200, 1400, 1500, 1700, 500, 700, 1000, 1200, 1400,\n",
       "                     1500, 1700, 500, 700, 1000, 1200, 1400, 1500, 1700,\n",
       "                     500, 700, 1000, 1200, 1400, 1500, 1700, 500, 700, 1000,\n",
       "                     1200, 1400, 1500, 1700],\n",
       "               mask=[False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False,\n",
       "                     False, False],\n",
       "         fill_value='?',\n",
       "              dtype=object),\n",
       "  'params': [{'learning_rate': 0.01, 'n_estimators': 500},\n",
       "   {'learning_rate': 0.01, 'n_estimators': 700},\n",
       "   {'learning_rate': 0.01, 'n_estimators': 1000},\n",
       "   {'learning_rate': 0.01, 'n_estimators': 1200},\n",
       "   {'learning_rate': 0.01, 'n_estimators': 1400},\n",
       "   {'learning_rate': 0.01, 'n_estimators': 1500},\n",
       "   {'learning_rate': 0.01, 'n_estimators': 1700},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 500},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 700},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 1000},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 1200},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 1400},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 1500},\n",
       "   {'learning_rate': 0.05, 'n_estimators': 1700},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 500},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 700},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 1000},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 1200},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 1400},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 1500},\n",
       "   {'learning_rate': 0.1, 'n_estimators': 1700},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 500},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 700},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 1000},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 1200},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 1400},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 1500},\n",
       "   {'learning_rate': 0.15, 'n_estimators': 1700},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 500},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 700},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 1000},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 1200},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 1400},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 1500},\n",
       "   {'learning_rate': 0.2, 'n_estimators': 1700},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 500},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 700},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 1000},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 1200},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 1400},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 1500},\n",
       "   {'learning_rate': 0.25, 'n_estimators': 1700}],\n",
       "  'split0_test_score': array([0.93578682, 0.93952681, 0.94302824, 0.94433913, 0.94530039,\n",
       "         0.94573097, 0.94641028, 0.94744488, 0.94840631, 0.94899075,\n",
       "         0.94900795, 0.94915361, 0.94916399, 0.94921175, 0.94877839,\n",
       "         0.94904483, 0.94921239, 0.94908158, 0.94898352, 0.94892513,\n",
       "         0.94889822, 0.94816698, 0.9481301 , 0.94826913, 0.94813616,\n",
       "         0.94808854, 0.94806548, 0.94803735, 0.94748649, 0.94749613,\n",
       "         0.94769951, 0.94793484, 0.94796449, 0.94794796, 0.94789362,\n",
       "         0.94662432, 0.94665583, 0.94693117, 0.94727051, 0.94727317,\n",
       "         0.94735709, 0.947452  ]),\n",
       "  'split1_test_score': array([0.93806776, 0.94105503, 0.94385626, 0.94501411, 0.94587927,\n",
       "         0.94623467, 0.94685117, 0.94818358, 0.94926063, 0.94982953,\n",
       "         0.94993691, 0.95009796, 0.95015257, 0.95018766, 0.94957976,\n",
       "         0.94958009, 0.94975889, 0.94967389, 0.94950496, 0.94954649,\n",
       "         0.94951105, 0.94906488, 0.9493791 , 0.94940906, 0.94953531,\n",
       "         0.94965644, 0.94963995, 0.94974858, 0.94873406, 0.9485909 ,\n",
       "         0.94908329, 0.94923617, 0.9495153 , 0.94963986, 0.94959855,\n",
       "         0.94750188, 0.94775986, 0.94811505, 0.94835446, 0.94858052,\n",
       "         0.94863404, 0.94862521]),\n",
       "  'split2_test_score': array([0.93645151, 0.93976188, 0.94273602, 0.94402279, 0.94504606,\n",
       "         0.94543102, 0.94608662, 0.9476051 , 0.94839408, 0.94928841,\n",
       "         0.94957855, 0.94968806, 0.94978715, 0.94985533, 0.94851822,\n",
       "         0.94906559, 0.94924949, 0.94927593, 0.94916014, 0.94919907,\n",
       "         0.94918148, 0.94893517, 0.9490412 , 0.94888843, 0.94871965,\n",
       "         0.94841812, 0.94844998, 0.94847669, 0.94814858, 0.9483347 ,\n",
       "         0.94837197, 0.94817034, 0.94818486, 0.94820083, 0.94823304,\n",
       "         0.94777648, 0.94794173, 0.94799014, 0.94807516, 0.94832233,\n",
       "         0.94829381, 0.9483007 ]),\n",
       "  'mean_test_score': array([0.93676869, 0.94011457, 0.94320684, 0.94445868, 0.94540857,\n",
       "         0.94579889, 0.94644936, 0.94774452, 0.948687  , 0.94936956,\n",
       "         0.9495078 , 0.94964654, 0.94970123, 0.94975158, 0.94895879,\n",
       "         0.94923017, 0.94940692, 0.9493438 , 0.94921621, 0.94922357,\n",
       "         0.94919692, 0.94872234, 0.94885013, 0.94885554, 0.94879704,\n",
       "         0.94872103, 0.94871847, 0.94875421, 0.94812304, 0.94814058,\n",
       "         0.94838492, 0.94844712, 0.94855488, 0.94859621, 0.94857507,\n",
       "         0.94730089, 0.94745247, 0.94767879, 0.94790004, 0.94805867,\n",
       "         0.94809498, 0.94812597]),\n",
       "  'std_test_score': array([0.00095782, 0.00067189, 0.00047446, 0.00041344, 0.00034865,\n",
       "         0.00033159, 0.00031335, 0.00031728, 0.00040564, 0.0003472 ,\n",
       "         0.00038253, 0.00038665, 0.00040813, 0.00040511, 0.00045176,\n",
       "         0.00024758, 0.00024934, 0.00024653, 0.00021654, 0.00025426,\n",
       "         0.00025043, 0.00039625, 0.0005275 , 0.00046596, 0.00057382,\n",
       "         0.00067498, 0.00067022, 0.00072565, 0.00050964, 0.00046754,\n",
       "         0.000565  , 0.00056616, 0.00068505, 0.00074515, 0.00073686,\n",
       "         0.00049137, 0.00056818, 0.0005311 , 0.00045952, 0.00056535,\n",
       "         0.00053994, 0.00049464]),\n",
       "  'rank_test_score': array([42, 41, 40, 39, 38, 37, 36, 32, 20,  6,  4,  3,  2,  1, 12,  8,  5,\n",
       "          7, 10,  9, 11, 17, 14, 13, 15, 18, 19, 16, 28, 26, 25, 24, 23, 21,\n",
       "         22, 35, 34, 33, 31, 30, 29, 27])},\n",
       " {'learning_rate': 0.05, 'n_estimators': 1700},\n",
       " 0.9497515771416293)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {\n",
    " 'learning_rate':[0.01,0.05,0.10,0.15,0.20,0.25],\n",
    " 'n_estimators':[500,700,1000,1200,1400,1500,1700]\n",
    "}\n",
    "gsearch2 = GridSearchCV(estimator = XGBClassifier(scale_pos_weight=2.61, seed=42,verbose=True), \n",
    " param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=3)\n",
    "gsearch2.fit(X_train,y_train)\n",
    "gsearch2.cv_results_, gsearch2.best_params_, gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the tuning the best learning rate is 0.05 at 1700 n_estimators. To increase the auc furthur lower learning rate and higher n_estimators combinations can be tried."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### min_child_weight and max_depth\n",
    "Using the above tunned leaning rate and n_estimator to tune min_child_weight and max_depth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_test1 = {\n",
    " 'min_child_weight':[1,2,3,4,5,6],\n",
    " 'max_depth':[2,4,6,7,8]\n",
    "}\n",
    "gsearch2 = GridSearchCV(estimator = XGBClassifier(scale_pos_weight=2.61, seed=42,n_estimators=1700,learning_rate=0.05,verbose=True), \n",
    "param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=3)\n",
    "#gsearch2.fit(X_train,y_train)\n",
    "#gsearch2.cv_results_, gsearch2.best_params_, gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_result = gsearch2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.950530 using {'max_depth': 7, 'min_child_weight': 4}\n",
      "0.941559 (0.000662) with: {'max_depth': 2, 'min_child_weight': 1}\n",
      "0.941593 (0.000557) with: {'max_depth': 2, 'min_child_weight': 2}\n",
      "0.941600 (0.000595) with: {'max_depth': 2, 'min_child_weight': 3}\n",
      "0.941629 (0.000558) with: {'max_depth': 2, 'min_child_weight': 4}\n",
      "0.941596 (0.000626) with: {'max_depth': 2, 'min_child_weight': 5}\n",
      "0.941789 (0.000506) with: {'max_depth': 2, 'min_child_weight': 6}\n",
      "0.948392 (0.000540) with: {'max_depth': 4, 'min_child_weight': 1}\n",
      "0.948464 (0.000318) with: {'max_depth': 4, 'min_child_weight': 2}\n",
      "0.948473 (0.000392) with: {'max_depth': 4, 'min_child_weight': 3}\n",
      "0.948713 (0.000283) with: {'max_depth': 4, 'min_child_weight': 4}\n",
      "0.948604 (0.000193) with: {'max_depth': 4, 'min_child_weight': 5}\n",
      "0.948666 (0.000329) with: {'max_depth': 4, 'min_child_weight': 6}\n",
      "0.949752 (0.000405) with: {'max_depth': 6, 'min_child_weight': 1}\n",
      "0.949823 (0.000341) with: {'max_depth': 6, 'min_child_weight': 2}\n",
      "0.949780 (0.000259) with: {'max_depth': 6, 'min_child_weight': 3}\n",
      "0.949927 (0.000193) with: {'max_depth': 6, 'min_child_weight': 4}\n",
      "0.949954 (0.000141) with: {'max_depth': 6, 'min_child_weight': 5}\n",
      "0.950025 (0.000335) with: {'max_depth': 6, 'min_child_weight': 6}\n",
      "0.950292 (0.000544) with: {'max_depth': 7, 'min_child_weight': 1}\n",
      "0.950273 (0.000566) with: {'max_depth': 7, 'min_child_weight': 2}\n",
      "0.950440 (0.000494) with: {'max_depth': 7, 'min_child_weight': 3}\n",
      "0.950530 (0.000338) with: {'max_depth': 7, 'min_child_weight': 4}\n",
      "0.950318 (0.000405) with: {'max_depth': 7, 'min_child_weight': 5}\n",
      "0.950525 (0.000399) with: {'max_depth': 7, 'min_child_weight': 6}\n",
      "0.950355 (0.000429) with: {'max_depth': 8, 'min_child_weight': 1}\n",
      "0.950398 (0.000562) with: {'max_depth': 8, 'min_child_weight': 2}\n",
      "0.950273 (0.000543) with: {'max_depth': 8, 'min_child_weight': 3}\n",
      "0.950488 (0.000525) with: {'max_depth': 8, 'min_child_weight': 4}\n",
      "0.950493 (0.000562) with: {'max_depth': 8, 'min_child_weight': 5}\n",
      "0.950443 (0.000537) with: {'max_depth': 8, 'min_child_weight': 6}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "# plot results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There has been increase in training auc(0.950).The best values are max_depth=7 and min_child_weight=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEHCAYAAAC5u6FsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5QkVZnv/e8Teams6qqmobtBodDGdaCxUWmhhtHltKAjDuAVWi5e5hyOzmI5yFlzHJFBPUcdlIO6mNfL4IuDwojn9RXwKNry8irKfYYZpYAG5TrQ4tBwlObS3XXPSzznj4jMiszKyqroyuysrvp91sqOHTt2ROyortpP7h2ZO8zdERERma+g2xUQEZF9iwKHiIikosAhIiKpKHCIiEgqChwiIpJKttsV2BvWrFnj69at63Y1RET2Kffcc89z7r62MX9ZBI5169YxPDzc7WqIiOxTzOx3zfI1VCUiIqkocIiISCoKHCIikooCh4iIpKLAISIiqShwiIhIKgocIiKSyrL4HofIUubuUCoRTk3h8auaDicn8akiXqxPe6lMMNBPdv/9yey/P5lVq8jsvz9BodDty5EF8HKZyq5dVF58kcrOnVR272bgzW9u+3kUOETaxMvlqLEuFvHJyboGPGrEp6JGu1l6aipu3FOkE8dnlufqhBZQzq6gmOunlFtBqbbsAwLcrH6HXJ6gUMAKBYJCL0FvASv0Ruu91fw4r6eHoLcXMpn4B1C3mJGoq+KMst5831o5n2W/5DGbHyNZJAiMIGPTy9orIMgYFhiZeFnNqyszY//6Mlbb3rhvVNYMrPFnPotwYiJq/BOvcjUgJF8vTqfDkZEZx1l/373R/1MbKXDIsuBhSDg6SjgyQmVkpGE5ik9ORI1w3KDXpxOBoDh7mnJ5QXW0fB7r6cEKPQT5npnpgQHKuX7K+X6KuRWUgj5KmV6KVqBInqLnmQqzFCtZpkoBUyWjWJpfIzWrUvyqa48q4Lvr695kpdZAWvSPGcQtZ/0+1cQs+dPrllw0Pbk17NRYNqx49AqjpYd7/0F2QQCGE5hjhJiHWFghCMsQlrFKGSuXsLCEhZVou1cIPJEO+ghyA2RyhxHsnyNzUJagJ0+mEL96e8j0Fsj2FQgt0/Z7Egoci4y74w5e/eUOfTpdccIwxMPpX/669Cz7eNgi3eQ4mayR68mQ68mQzWdq6cZXtidDJtP522TuHjXOycZ+9wjhaMOyaVCIluHY2KzvyuvkcgQtGvDMypXYgWubN+yFHoKeHiw/Szp+BT09kM9T8hxT5YCpYsDUlDM5XmZ8tMTkaInJsXg5WmKimjdegjLRq0EmF9Dbn6PQn6O3P8f+/flaurAiyp9ez1NYkcUyjQ31LA06QKUyPQTy4ouUd+6M0zunh0VefJHyzhdr74LD3fXBJcl6e8nsv4rMqlVkV9UPl9Xyq8No8bZODKO5J/4e4qBSqST+xirTf3dhxalMlSjvHqW8eyRajsSv0XEqo+OUxyeojE1SGZ+gPDFFODFJZapEiOEW4JapvcIgwC0LhV7oKUC+APkevLcHcn14Nhe9Mlk8yEKQJbQMjlFyI0zUs1Kt/1hIuLt6TRVgjOM+0P5mXoGjhftu+nfGdk2laJTDORvlpumGhn9fElSDTJMAk60ucwFZq5ClTCYskilPEVQmyRQnyEyNYVNjBOMj2MRugtFdBGMv4rt31zX8XirNUZGAzMAAwcAAwcoBMv0D5A49lEKcV92WWTlA0N+47CfojYZerDrskoK7U5qs1Br4idFireGfGC0x+VwpsT7K5OgLTI6VZ323G2SN3hU5CnHjv2awv77hj4NBb7y90J8jl09f71SyWbKrV5NdvXreu9SNt88RbIpPb19wsMnWrc8dbMKJiel6VIeCqnXcuavJkNCLhKOjM3808cvy+enzr1pF5qVRXWuvWt2n84KVK7GgM2++qm9C5zkylooCRwuP3fXv7HqhRBAYFkBghmWI1+NxziA5rgkWj21mAsNy1bHOePy0buwzwDJB3dipJcZOa8dvls40nrc+HZUL0u2fyAvLIaWpCqWpCsXJMqXRCaZeHKW4a4ziyDjFkUmKY5MUx4qUJ0sUJ8qUJyuUdoeUyzBRMUbCgDIZKmQpB3kqmR6w5B+IAX3xq2HyzTzY6grZNRWygZPNONkc5HJBFJQKWXK9OfJ9eXIresgP9JLv7yVXiAJVkI+DVmFmjymTbf1H6u6Ui2HU+De+4x+rphsCw2iJsNI8CFhgde/8D3hJH4X+/eK8fC0IJINCricz73HwxcwWWbDJ9K2gMjJSCwQ+NTXrfkF/f12jn1+3Lk7vV5dfC1arVmG9vYvq/83MOhI0QIGjpaG7v0DxiSf2zsnMosHPIIh++eJ1q+Y3rgcBbtHNzzAIIDDMmpQzm94W74eRWK/fDwOfmKwb6kmO3WeA3vhVk82S6e8nWLkyWjZ5Zx8MDEDfAJW+Abywgkp+BWG+l0q2QBjkKVWgNFmhXAwpTZUpTVWD13S6XKxQnKowNlWhtLtCaWqS0tRYqnHqIDNzGC6bDyhNVWqBoFIKZ/k/In6nHzXwK9f0ctC6lbWeQbI3UF3P92YXVWOy2HUy2IRjY+QOOYTCUUe17AVk9tsPy+c7eJX7PgWOFg765CcIR0bBQzwMo49neAjxfYboZkS8LYzSuNet1/YLw3i9yX5hCMxzvzDEfZb9qtvms597k+NE6WDNavLr1tWGfIKViaGeuuVKMgP9XX2n5e6EZadUjHpIpclKLV2Oe011rybbysUK/fsXWHPoQDxElJtxf6C3P0++L0sQKAgsNnsSbGRhFDha6H/DG7pdBZmDmZHJGZlcQGFFrtvVEVkW9M1xERFJpaOBw8xOMrNHzexxM7uwyfaXm9nNZvaAmd1mZoOJbRUz2xq/tiTyDzOzX5rZv5nZtWamwUgRkb2oY0NVZpYBvg6cCGwH7jazLe7+UKLYpcB33P1qM3szcAnw5/G2CXff2OTQXwS+7O7XmNk3gA8Bl3fqOkREFpNyWKYUlihWipTCEqVKiWJYrC2T+aWwxKbBTQTW3j5CJ+9xHAc87u7bAMzsGuBdQDJwbAA+GqdvBX7U6oAW3YF9M/C+OOtq4LMocIhIG4Ue1jXO822kk3nz2acYti5b25Y4fuizfOpvFsMfGKYn09PWn08nA8chwFOJ9e3AHzeUuR/YDHwVOBUYMLPV7v48UDCzYaLvyX7B3X8ErAZ2uns5ccxDmp3czM4BzgF42cte1p4rEtnHlcMyY6UxJsoTlMMyoYd1r4pXcDxaev2ysWyzfUPC6FvWtC7XeGzHqYSVlvs13T/er1rnlnWbKxgkGumyL2z6mEZZy5LL5MgFOfKZfN2yms5n8vTn+8kH+TnL1vZJlM1lcuSD/IztWdu3vjne7HOLjR+4Px+4zMzOBu4AnmZ6QoWXufszZvYK4BYz+zXQ7Js+TT/E7+5XAFcADA0N7VtfxxZJCD1kvDTOWGmMsdIYo6XRWdPjpfH6vOIo4+VxRotR3mRlstuXMy+BBQQE0XIer4xlMIxMEC8t+hJlxjJ15XJBjt5sL/tl9osa11ka6boGOdGwz7lP3IA37tPuoaJu62Tg2A4cmlgfBJ5JFnD3Z4DTAMysH9js7rsS23D3bWZ2G/Ba4AfAKjPLxr2OGccUWQzcnYnyRF2jPa+GvjTKWHGMsfIYY8W4THl8XufMB3lW5FbUvdb2rWVFdgUr8ivoz/XTl+uLltk+skF29saYoGkj3KwxbtwvCOL957FfszKG6UuTi1wnA8fdwOFmdhhRT+Ispu9NAGBma4AX3D0EPgFcFefvD4y7+1Rc5g3Al9zdzexW4D3ANcB/An7cwWvY69ydclhuOiZarBRr2xq727N1u5P7zGeMtVgpYmZkLUvGMmSCDBnLkA2m17OWreU3rmeD7Kxla/mtts3j+HPlNys3n4bI3SmGxagRL9Y38o2N/nyCwHzGojOWYUWuvlHfr7AfB2cPpj/fXwsA/bn+GUGhMS+f0QcMZe/oWOBw97KZnQf8jGimiqvc/UEzuwgYdvctwAnAJWbmRENVH4l3fyXwD2YWEn1k+AuJT2P9DXCNmX0euA+4slPX8MCOBxgpjkw3wuH0Tapkw1y9aVW7gdXYqLfY1qzxbqfAguZd6Li7nQ/yZINsXfcdoBJWKHuZSlih4hXKYZkpn5peb9hW8UrT9bKXKYftHS/eE9V3tzMCjGUJgoDJ8iSjpdF51dWwpg34gb0HRuv56B19f76/LiAky1fzejI9enct+xzz+Uw1vY8bGhry4eHh1Pu960fvYtuubfMqm7wpNaOBTt7cyuTqGu3ZboLNZxy1sWyzfbJB9ycHcPfaDcrGoFINNjMCVYrA1I5yvdneGY1+s3f2/bl+CtnCkhuzFmnGzO5x96HG/O63KovYxX9yMeWw3LTBTwaC+Q6FLFfVsesMGQ2niCwBChwtvGrNq7pdBRGRRUf9bRERSUWBQ0REUlHgEBGRVBQ4REQkFQUOERFJRYFDRERSUeAQEZFUFDhERCQVBQ4REUlFgUNERFJR4BARkVQUOEREJBUFDhERSUWBQ0REUlHgEBGRVBQ4REQkFQUOERFJRYFDRERSUeAQEZFU9MxxkeWgPAVjO+LXczD+AmRykOuDXG9i2Vufl8mBWbdrL4uMAofIvsgdJnfC6I5EQGh4jSYCxdSuPTuPZaIAkm8MMLMEmpZ5K5pvyxYg0OBHU+4QViAsQaUIleSymi5CWJ5OV5LpErxqc9t/vgocIotFY69gbAeMPlu/PvbsdDosNzmIQd8BsGJt9Hrp0dPpFWug/8Ao3bt/tH9pHEoT8Wt8lmWcLo7X540+23xfPP21ZxsDTouAlO+bo1yL3lMYxg1tQ8ObbIhnbGtoiMNmjXeafefT+Ce27cnPM+mV74CgsLBjNFDgEOkUd5h4MdHo72GvIFuAFQdGDf/KQxqCQZxfXe9bDZku/lm7RwEwTUCqLZuUK45GP5/GbZXiHlTOWHAj3EqQhUw+ClCZPAS56XQmmc5H/6c9K5tsy8X75Wfft3aefPR/XUvPtm++7ZeqwCGSRmOvoNYjqPYKGnoIaXsF/WvrewgrDoT8in3nPoMZ5ArRq5MqZSjPFphmCVblqbhhbdXYNjS4QUOjndw3aDjOvvJ/1AYKHLJ4uSeWPr2ckUf99mZ5TfeJ8zyMewYNQ0G1oaLn5t8r6F/b0Cs4MBEE1kZDRb0HdLdXsBRkspAZgJ6BbtdkWdJv72LhHo9pTkXL8tTMdLk4PQY6Y3txejmfdHlq+nzlYnysYpT2kHQNMTO3z6vxbhEEuq7aK4iHgqqBoK5HkHjtS70CkQVS4GjlyX+CiZ3pG+gZebM00HXBYKq9dQ+ykOmBbLUrnUznIdsTLXOrptOZfFTGMtExzACbXs7IYx7b0+4zj/PUDp1mnxbntgAKqxpuHqtXIDIb/WW0csNfw3OPti5THR/NNjbODQ11fv8oL5ObbqjrGux4W6anSX6+4RzJso3ni8vr440i0iEKHK2c/o/RZ6hna9TVQIvIMqTA0cpBR3W7BiIii47eLouISCoKHCIikooCh4iIpNLRwGFmJ5nZo2b2uJld2GT7y83sZjN7wMxuM7PBhu0rzexpM7sskfdeM/t1vM9PzWxNJ69BRETqdSxwmFkG+DpwMrABeK+ZbWgodinwHXd/DXARcEnD9s8BtyeOmQW+Crwp3ucB4LzOXIGIiDTTyR7HccDj7r7N3YvANcC7GspsAG6O07cmt5vZscBBwE2J8tVvbq0wMwNWAs90pvoiItJMJwPHIcBTifXtcV7S/cDmOH0qMGBmq80sAP4O+HiysLuXgL8Efk0UMDYAVzY7uZmdY2bDZja8Y8eOhV6LiIjEOhk4mk3c0zgJ0fnA8WZ2H3A88DRQBs4FbnT3ZODBzHJEgeO1wMFEQ1WfaHZyd7/C3YfcfWjt2rULuhAREZnWyS8AbgcOTawP0jCs5O7PAKcBmFk/sNndd5nZ64FNZnYu0A/kzWwU+EG83xPxPtcBM266i4hI53QycNwNHG5mhxH1JM4C3pcsEH8i6gV3D4l6DlcBuPv7E2XOBobc/UIzOxjYYGZr3X0HcCLwcAevQUREGnQscLh72czOA34GZICr3P1BM7sIGHb3LcAJwCVm5sAdwEfmOOYzZva3wB1mVgJ+B5zdqWsQkX1PqVRi+/btTE5Odrsq+4xCocDg4CC5XG5e5c19MTz7oLOGhoZ8eHi429UQkb3gt7/9LQMDA6xevRrTM1Lm5O48//zzjIyMcNhhh9VtM7N73H2ocR99c1xElpTJyUkFjRTMjNWrV6fqoSlwiMiSo6CRTtqflwKHiIikosAhItJBn/3sZ7n00kv3aN+tW7dy4403LuhYH//4xznyyCN5zWtew6mnnsrOnTv3qC5JChwiIotUY+DYEyeeeCK/+c1veOCBBzjiiCO45JLGKQHTU+AQEWmziy++mPXr1/OWt7yFRx99tJb/xBNPcNJJJ3HssceyadMmHnnkEQDOPvtsPvzhD7Np0yaOOOIIbrjhBorFIp/+9Ke59tpr2bhxI9deey0ADz30ECeccAKveMUr+NrXvjZnXd761reSzUbfvHjd617H9u3bF3x9enSsiCxZf/uTB3nomd1tPeaGg1fymXfM/ljpe+65h2uuuYb77ruPcrnMMcccw7HHHgvAOeecwze+8Q0OP/xwfvnLX3Luuedyyy23APDkk09y++2388QTT/CmN72Jxx9/nIsuuojh4WEuuyx6ssRnP/tZHnnkEW699VZGRkZYv349f/mXf0kul+OUU07hW9/6FgcffPCsdbvqqqs488wzF/wzUOAQEWmjO++8k1NPPZW+vj4A3vnOdwIwOjrKXXfdxemnn14rOzU1VUufccYZBEHA4Ycfzite8Ypab6TR2972Nnp6eujp6eHAAw/kD3/4A4ODg3MOaV188cVks1ne//73tyw3HwocIrJkteoZdFKzj7eGYciqVavYunXrvPaZ7SOyPT09tXQmk6FcLs9Zn6uvvpobbriBm2++uS0fVdY9DhGRNnrjG9/I9ddfz8TEBCMjI/zkJz8BYOXKlRx22GF8//vfB6JvbN9///21/b7//e8ThiFPPPEE27ZtY/369QwMDDAyMrKg+vz0pz/li1/8Ilu2bKn1ghZKgUNEpI2OOeYYzjzzTDZu3MjmzZvZtGlTbdt3v/tdrrzySo4++miOOuoofvzjH9e2rV+/nuOPP56TTz6Zb3zjGxQKBd70pjfx0EMP1d0cn80pp5zCM8/MfK7deeedx8jICCeeeCIbN27kwx/+8IKvUXNViciS8vDDD/PKV76y29VI5eyzz+btb38773nPe7pWh2Y/N81VJSIibaGb4yIiXfbtb3+721VIRT0OERFJRYFDRERSUeAQEZFUFDhERCQVBQ4RkQ7q9rTqVZdeeilmxnPPPbdH+ycpcIiILFLtmFYd4KmnnuLnP/85L3vZy9pQKwUOEZG2W0zTqgN89KMf5Utf+lLbHqmr73GIyNL1/18Iv/91e4/5klfDyV+YdfNim1Z9y5YtHHLIIRx99NFt+xEocIiItNFimlZ9fHyciy++mJtuuqmdl6jAISJLWIueQSctlmnVn3jiCX7729/Wehvbt2/nmGOO4Ve/+hUveclL5ryO2egeh4hIGy2madVf/epX8+yzz/Lkk0/y5JNPMjg4yL333rugoAEKHCIibbXYplXvBE2rLiJLiqZV3zOaVl1ERDpGN8dFRLpsyUyrbmZ/ZmYz+k1m9n4zO7Gz1RIRkcWq1VDV3wK3N8m/GbioM9UREZHFrlXg6HP3HY2Z7v57YEXnqiQiIotZq8BRMLMZ90DMLAf0dq5KIiKymLUKHD8Evmlmtd5FnP5GvE1EROawGKZV//u//3vWr1/PUUcdxQUXXLBHdUlq9amq/wZ8Hvidmf0OMOBQ4Ergvy/4zCIi0tLWrVsZHh7mlFNO2eNj3Hrrrfz4xz/mgQceoKenh2effXbB9Zq1x+HuZXe/kChYnA38J+Bl7n6hu5fmc3AzO8nMHjWzx83swibbX25mN5vZA2Z2m5kNNmxfaWZPm9lliby8mV1hZo+Z2SNmtnme1yoislcspmnVL7/8ci688MLaHFcHHnjggq9v1h6HmZ3WkOXAKjPb6u5zTp5iZhng68CJwHbgbjPb4u4PJYpdCnzH3a82szcDlwB/ntj+OWZ+sutTwLPufoSZBcABc9VFRJanL/7qizzyQvNZZvfUkQccyd8c9zezbl9s06o/9thj3HnnnXzqU5+iUChw6aWX8kd/9EcL+hm0Gqp6R5O8A4DXmNmH3P2WOY59HPC4u28DMLNrgHcBycCxAfhonL4V+FF1g5kdCxwE/BRIfuX9g8CRAO4eAgt/DqKISJsspmnVAcrlMi+++CL/+q//yt13380ZZ5zBtm3bFvRQp1kDh7v/52b5ZvZy4Drgj+c49iHAU4n17U32uR/YDHwVOBUYMLPVwIvA3xH1Pv40ce5VcfJzZnYC8ARwnrv/oUk9zwHOAdr2uEQR2be06hl00mKZVh1gcHCQ0047DTPjuOOOIwgCnnvuOdauXTvXZcwq9VxV7v47IDePos2uunFGxfOB483sPuB44GmgDJwL3OjuTzWUzwKDwD+7+zHAvxANdzWr5xXuPuTuQwv5AYmIpLGYplUHePe7310bDnvssccoFousWbNmQcdMHTjM7Ehgas6CUQ/j0MT6IFA356+7P+Pup7n7a4nuXeDuu4DXA+eZ2ZNEgeE/mtkXgOeBceD6+BDfB45Jew0iIp2y2KZV/+AHP8i2bdt41atexVlnncXVV1+94GePzzqtupn9hJk9hAOAlwIfcPd/aXng6MuDjxENNT0N3A28z90fTJRZA7zg7qGZXQxU3P3TDcc5Gxhy9/Pi9WuAK9z9lnjb29z9dFrQtOoiy4emVd8zaaZVb3VzvHEIyIEXiILHB4iGiWbl7mUzOw/4GZABrnL3B83sImDY3bcAJwCXmJkDdwAfaXXM2N8A/9PMvgLsAJreixERkc6Y14OczGwj8D7gDOC3wA/c/bLWey0e6nGILB/7Yo9jMWhLj8PMjgDOAt5LdG/hWqJA86b2VldERPYlrYaqHgHuBN7h7o8DmNlHW5QXEZFloNWnqjYDvwduNbNvmtmf0vwjtiIisoy0mqvqenc/k+hb2rcRfcP7IDO73MzeupfqJyIii8yc3+Nw9zF3/667v53ouxhbgRkTFoqIyEzdnla9+p2SjRs3sm7dOjZu3LhHdUlqdY9jBnd/AfiH+CUiIh3UjmnVk18c/NjHPsZ+++234Hql/ua4iIi0tpimVa9yd6677jre+973Lvj6UvU4RET2Jb//H/+DqYfbO616zyuP5CWf/OSs2xfbtOpVd955JwcddBCHH374gn8GChwiIm202KZVr/re977Xlt4GKHCIyBLWqmfQSYtpWnWInsnxwx/+kHvuuWfOsvOhexwiIm202KZVB/jFL37BkUceyeDg4NyF50GBQ0SkjRbbtOoA11xzTduGqWCekxzu6zTJocjysS9OcrivTauuHoeIiKSim+MiIl327W9/u9tVSEU9DhERSUWBQ0REUlHgEBGRVBQ4REQkFQUOEZEO6va06lu3buV1r3sdGzduZGhoiF/96ld7VJckBQ4RkUWqMXDsiQsuuIDPfOYzbN26lYsuuogLLrhgwfVS4BARabPFNK26mbF7924Adu3aNevsuWnoexwismTded1jPPfUaFuPuebQfjadccSs2xfbtOpf+cpX+LM/+zPOP/98wjDkrrvuWvDPQIFDRKSNFtu06pdffjlf/vKX2bx5M9dddx0f+tCH+MUvfrGga1TgEJElq1XPoJMW07TqV199NV/96lcBOP300/mLv/iLluXnQ/c4RETaaLFNq37wwQdz++23A3DLLbe05QmAChwiIm202KZV/+Y3v8nHPvYxjj76aD75yU9yxRVXLPgaNa26iCwpmlZ9z2hadRER6RjdHBcR6TJNqy4i0mXLYQi+ndL+vBQ4RGRJKRQKPP/88woe8+TuPP/88xQKhXnvo6EqEVlSBgcH2b59Ozt27Oh2VfYZhUKBwcHBeZdX4BCRJSWXy3HYYYd1uxpLmoaqREQklY4GDjM7ycweNbPHzezCJttfbmY3m9kDZnabmQ02bF9pZk+b2WVN9t1iZr/pZP1FRGSmjgUOM8sAXwdOBjYA7zWzDQ3FLgW+4+6vAS4CLmnY/jng9ibHPg1o75SXIiIyL53scRwHPO7u29y9CFwDvKuhzAbg5jh9a3K7mR0LHATclNzBzPqBvwY+36F6i4hIC50MHIcATyXWt8d5SfcDm+P0qcCAma02swD4O+DjTY77uXjbeKuTm9k5ZjZsZsP6dIWISPt0MnA0mxO48YPV5wPHm9l9wPHA00AZOBe40d2TgQcz2wj8B3e/fq6Tu/sV7j7k7kNr167dowsQEZGZOvlx3O3AoYn1QaBu6kZ3fwY4DWpDUJvdfZeZvR7YZGbnAv1A3sxGgd8Bx5rZk3HdDzSz29z9hA5eh4iIJHQycNwNHG5mhxH1JM4C3pcsYGZrgBfcPQQ+AVwF4O7vT5Q5Gxhy9+qnsi6P89cBNyhoiIjsXR0bqnL3MnAe8DPgYeA6d3/QzC4ys3fGxU4AHjWzx4huhF/cqfqIiEh76HkcIiLSlJ7HISIibaHAISIiqShwiIhIKgocIiKSigKHiIikosAhIiKpKHCIiEgqChwiIpKKAoeIiKSiwCEiIqkocIiISCoKHCIikooCh4iIpKLAISIiqShwiIhIKgocIiKSigKHiIikosAhIiKpKHCIiEgqChwiIpKKAoeIiKSiwCEiIqkocIiISCoKHCIikooCh4iIpKLAISIiqShwiIhIKgocIiKSigKHiIikosAhIiKpKHCIiEgqChwiIpKKAoeIiKSiwCEiIqkocIiISCodDRxmdpKZPWpmj5vZhU22v9zMbjazB8zsNjMbbNi+0syeNrPL4vU+M/v/zOwRM3vQzL7QyfqLiMhMHQscZpYBvg6cDGwA3mtmGxqKXQp8x1GyR7AAAAr9SURBVN1fA1wEXNKw/XPA7Y37uPuRwGuBN5jZyW2vvIiIzKqTPY7jgMfdfZu7F4FrgHc1lNkA3Bynb01uN7NjgYOAm6p57j7u7rfG6SJwL1DXSxERkc7qZOA4BHgqsb49zku6H9gcp08FBsxstZkFwN8BH5/t4Ga2CngH04Gncfs5ZjZsZsM7duzYw0sQEZFGnQwc1iTPG9bPB443s/uA44GngTJwLnCjuz9FE2aWBb4HfM3dtzUr4+5XuPuQuw+tXbt2T69BREQaZDt47O3AoYn1QeCZZAF3fwY4DcDM+oHN7r7LzF4PbDKzc4F+IG9mo+5evcF+BfBv7v6VDtZfRESa6GTguBs43MwOI+pJnAW8L1nAzNYAL7h7CHwCuArA3d+fKHM2MFQNGmb2eWA/4C86WHcREZlFxwKHu5fN7DzgZ0AGuMrdHzSzi4Bhd98CnABcYmYO3AF8pNUx44/rfgp4BLjXzAAuc/dvdeo6REQWC3dnZKrMrvESO8dL7JwoxssSu8aj9K6J6nqUvvGvNpEJmt052HPm3njbYekZGhry4eHhbldDRASAciWsNfBRYx8HgLix3zVRYud4MbE9Wt81USJs0WT35TPs15tjv94cq/pyrOrN8+UzN9Kbz+xRPc3sHncfaszv5FCViMiSNlmq1L/zHy+xe6KxJzBz+8hUueVxVxayrOrLs6ovCgKHHtDHqjgYTAeGfBwccuwX5/dk9yxApKXAIbJIhKFTcSd0JwxJpJ1K6IQOoVfTUZnQ433i7bVtyXLV/GqZeHtgRsaMIICMGZnACAKbTsfL2qtatpZOLIOZx4qHkhe95PDPron6IaDaO//x+uGf6vapcjjrcbOB1Rr6VX15XrKywPqXDETrvXGjn9heDQwDhVzbh5baTYFjL3Ov/wMvVxuFcLoBqLhTrtT/8VfCaJ9KvH063+vypxuZxLHj/cMwzmsoV4nzq+d2jz83HQ9j+nQSxxPp+vxqpieudbayyRFST3Eekvk+fV6vO2+iTnXHaHae6XrObHRnNtR1jXmy8W5spJNlnLr/3+YN/Hx+e/YtZtQCS7YhyERBiSaBJ1mOGXnZoD6o1Y6TDHSzHAdg92S5NgS0qxoMJkpUWvwH9OYyiQY+x7o1fazqXRXl9SWCQOKd/6q+PCvymX0meKalwNHCx79/P//+wnitEa7+8Vcb4WoDXgmbNOTVBiHRwJTDcJ9rIKq/9wa1PwKry7faN3YssY/Fa2bJfJv+ck9j/hzngcYy0+dvtm+y/nOVrR6v2vjUGqXACBKNXxBALgjqy1hcplbeyBiJdOI4dcefpYwljhMwe30SjWJ9faivW7UBjetULWNmeOINSe13d8abCuLf5XDG733y97325mRGXvKYxH9L9cdqfkwa3jhFr1IlZLLkVJwZdQqb/N0ljwPUjf8fsqq3dh8g+c5/+v5AjpW9OQq5vTP8sy9R4GihHEbvZ7NBQCE3811ONgjiP0zquviZoP5dUX1Xv1qO2juoZLnA4rzkkMGMY1N/7FnL1b8Ly2SaD01kG8oFi7ybLCLdpcDRwpfP3NjtKoiILDp6HoeIiKSiwCEiIqkocIiISCoKHCIikooCh4iIpKLAISIiqShwiIhIKgocIiKSyrKYVt3MdgC/28Pd1wDPtbE6+wJd8/Kw3K55uV0vLPyaX+7uM569vSwCx0KY2XCz+eiXMl3z8rDcrnm5XS907po1VCUiIqkocIiISCoKHHO7otsV6AJd8/Kw3K55uV0vdOiadY9DRERSUY9DRERSUeAQEZFUFDhmYWZXmdmzZvabbtdlbzCzQ83sVjN72MweNLO/6nadOs3MCmb2KzO7P77mv+12nfYWM8uY2X1mdkO367I3mNmTZvZrM9tqZsPdrs/eYGarzOx/mdkj8d/169t2bN3jaM7M3giMAt9x91d1uz6dZmYvBV7q7vea2QBwD/Bud3+oy1XrGIseOr7C3UfNLAf8E/BX7v6vXa5ax5nZXwNDwEp3f3u369NpZvYkMOTuy+YLgGZ2NXCnu3/LzPJAn7vvbMex1eOYhbvfAbzQ7XrsLe7+v9393jg9AjwMHNLdWnWWR0bj1Vz8WvLvpMxsEHgb8K1u10U6w8xWAm8ErgRw92K7ggYocEgTZrYOeC3wy+7WpPPiIZutwLPAz919yV8z8BXgAiDsdkX2IgduMrN7zOycbldmL3gFsAP4x3hI8ltmtqJdB1fgkDpm1g/8APiv7r672/XpNHevuPtGYBA4zsyW9LCkmb0deNbd7+l2XfayN7j7McDJwEfioeilLAscA1zu7q8FxoAL23VwBQ6picf5fwB8191/2O367E1xN/424KQuV6XT3gC8Mx7zvwZ4s5n9P92tUue5+zPx8lngeuC47tao47YD2xM96P9FFEjaQoFDgNqN4iuBh939/+p2ffYGM1trZqvidC/wFuCR7taqs9z9E+4+6O7rgLOAW9z9A12uVkeZ2Yr4Ax/EwzVvBZb0pyXd/ffAU2a2Ps76U6BtH3TJtutAS42ZfQ84AVhjZtuBz7j7ld2tVUe9Afhz4NfxmD/AJ939xi7WqdNeClxtZhmiN1HXufuy+HjqMnMQcH303ogs8P+6+0+7W6W94r8A340/UbUN+M/tOrA+jisiIqloqEpERFJR4BARkVQUOEREJBUFDhERSUWBQ0REUlHgEBGRVBQ4ZNkys3eaWdumYTCz28xsqEn+kJl9LU6fbWaXzbL/aLP8NtTrxuoXHVuUma3uG83slE7US/Zd+gKgLFvuvgXYshfOMwx07RkQ7r6Qhn8j0fTrS/mLoJKSehyyJJnZuvgBNt8ys9+Y2XfN7C1m9s9m9m9mdlzy3b+ZfdvMvmZmd5nZNjN7zxzHvyB+MND9ZvaFxKbT44dDPWZmm+KyJzR7YJKZHWZm/2Jmd5vZ5+Y43/9tZu+M09eb2VVx+kNm9vk4/YH43FvN7B/ib8RXH2K0Jk7/9/jn8nMz+56ZnT9b3eNvHF8EnBkf88zWP3VZLhQ4ZCn7D8BXgdcARwLvA/4EOB/4ZJPyL423vx34QpPtAJjZycC7gT9296OBLyU2Z939OOC/Ap+Zo35fJZq99I+A389R9g5gU5w+BNgQp/8EuNPMXgmcSTQL7EagAry/od5DwGaiKfNPI+pJJNXV3d2LwKeBa919o7tfO0cdZZlQ4JCl7Lfu/mt3D4EHgZs9mmPn18C6JuV/5O5h/NTDg1oc9y3AP7r7OIC7Jx/4VZ1V+J5ZzpH0BuB7cfp/zlH2TmCTmW0gmqzuD/FTG18P3EU0id2xwN3xXGN/SvRMhqQ/AX7s7hPxw7p+0rA9Td1lGdM9DlnKphLpMLEe0vx3P1neWhzXmP1JgdVjVGY5R6N5TRbn7k+b2f5E077fARwAnAGMuvtIPLvx1e7+iTnq3UraussypR6HSHo3AR80sz4AMztgD4/zz0RTm0PDsNIs/oVoGOkOoh7I+fES4GbgPWZ2YLVOZvbyhv3/CXiHmRXiB3a9bR7nHAEG5lFOlhEFDpGU4im5twDD8bDQ+XPsMpu/Inoa3d3AfvMofyfRfYjHgXuJeh13xnV6CPhvRI9HfQD4OdE9m2S9747rfT/RsNQwsGuOc94KbNDNcUnStOoiy4iZ9bv7aNxbugM4x93v7Xa9ZN+icUyR5eWK+AZ7geieiIKGpKYeh8gszOzVzPy005S7//FSOqdIWgocIiKSim6Oi4hIKgocIiKSigKHiIikosAhIiKp/B+uss1ohTOYPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "max_depth=[2,4,6,7,8]\n",
    "min_child_weight=[1,2,3,4,5,6]\n",
    "scores = np.array(means).reshape(len(max_depth), len(min_child_weight))\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, scores[i], label='depth: ' + str(value))\n",
    "pyplot.legend()\n",
    "pyplot.xlabel('min_child_weight')\n",
    "pyplot.ylabel('AUC')\n",
    "pyplot.savefig('min_child_weight vs_max_depth.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This graphical visualization shows that increasing depth generally increases the auc(more tree better results).But peak was for second highest depth of 7 for min child weight of 4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Modelo 6 neural net 0.8790437490105019\n",
      "AUC: 0.9515\n"
     ]
    }
   ],
   "source": [
    "#seed(2)\n",
    "mlp = XGBClassifier(scale_pos_weight=2.61, seed=42,n_estimators=1700,learning_rate=0.05,min_child_weight=4,max_depth=7)\n",
    "mlp.fit(X_train,y_train)\n",
    "pred6=mlp.predict(X_test)\n",
    "print(\"Modelo 6 neural net\",mlp.score(X_test,y_test))\n",
    "\n",
    "probs = mlp.predict_proba(X_test)\n",
    "probs = probs[:, 1]\n",
    "auc = roc_auc_score(y_test, probs)\n",
    "print('AUC: %.4f' % auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Auc on test data is 95.15 with above tunned parameters. Parameter tunning for the project was done on entire train data, but for the purpose of this notebook I have split the train data into train and test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_test1 = {\n",
    " 'gamma':[i/10.0 for i in range(0,5)]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( max_depth=7,\n",
    " min_child_weight=4,\n",
    " objective= 'binary:logistic', scale_pos_weight=2.61, seed=42,n_estimators=1700,learning_rate=0.05,min_child_weight=4,max_depth=7), \n",
    " param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=3)\n",
    "gsearch1.fit(X_train,y_train)\n",
    "#gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'mean_fit_time': array([2118.74570791, 2122.09651065, 2120.21854258, 2121.21690075,\n",
       "         1716.43201033]),\n",
       "  'std_fit_time': array([ 2.54991761,  0.70104673,  0.71823438,  0.3402881 , 29.58762467]),\n",
       "  'mean_score_time': array([4.71210988, 5.53630535, 5.41697152, 5.54980834, 2.99957371]),\n",
       "  'std_score_time': array([0.83900077, 0.29805903, 0.61813024, 0.40167704, 0.32573392]),\n",
       "  'param_gamma': masked_array(data=[0.0, 0.1, 0.2, 0.3, 0.4],\n",
       "               mask=[False, False, False, False, False],\n",
       "         fill_value='?',\n",
       "              dtype=object),\n",
       "  'params': [{'gamma': 0.0},\n",
       "   {'gamma': 0.1},\n",
       "   {'gamma': 0.2},\n",
       "   {'gamma': 0.3},\n",
       "   {'gamma': 0.4}],\n",
       "  'split0_test_score': array([0.9502011 , 0.95016912, 0.95027793, 0.94980543, 0.95007811]),\n",
       "  'split1_test_score': array([0.95099434, 0.95100261, 0.95090898, 0.95102608, 0.95072542]),\n",
       "  'split2_test_score': array([0.95039512, 0.95024394, 0.95038152, 0.95065724, 0.95050122]),\n",
       "  'mean_test_score': array([0.95053019, 0.95047189, 0.95052281, 0.95049625, 0.95043492]),\n",
       "  'std_test_score': array([0.00033763, 0.00037652, 0.00027632, 0.00051117, 0.00026839]),\n",
       "  'rank_test_score': array([1, 4, 2, 3, 5])},\n",
       " {'gamma': 0.0},\n",
       " 0.9505301854659093)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch1.cv_results_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "gamma 0 ,which is default parameter value is the best."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### subsample and colsample_bytree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score=nan,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None, gamma=0,\n",
       "                                     gpu_id=None, importance_type='gain',\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=0.05, max_delta_step=None,\n",
       "                                     max_depth=7, min_child_weight=4,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=170...\n",
       "                                     objective='binary:logistic',\n",
       "                                     random_state=None, reg_alpha=None,\n",
       "                                     reg_lambda=None, scale_pos_weight=2.61,\n",
       "                                     seed=42, subsample=None, tree_method=None,\n",
       "                                     validate_parameters=False,\n",
       "                                     verbosity=None),\n",
       "             iid=False, n_jobs=4,\n",
       "             param_grid={'colsample_bytree': [0.6, 0.7, 0.8, 0.9],\n",
       "                         'subsample': [0.6, 0.7, 0.8, 0.9]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {\n",
    " 'subsample':[i/10.0 for i in range(6,10)],\n",
    " 'colsample_bytree':[i/10.0 for i in range(6,10)]\n",
    "}\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( max_depth=7,\n",
    " min_child_weight=4,\n",
    " objective= 'binary:logistic', scale_pos_weight=2.61, seed=42,n_estimators=1700,learning_rate=0.05,gamma=0), \n",
    " param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=3)\n",
    "gsearch1.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'mean_fit_time': array([1517.80121072, 1560.57705386, 1571.3845017 , 1516.35370533,\n",
       "         1673.68825897, 1692.81899587, 1685.97389325, 1656.96202342,\n",
       "         1811.32009037, 1821.33343053, 1815.21176561, 1806.06085531,\n",
       "         1988.8742667 , 2002.78725608, 1999.04388698, 1957.87839691]),\n",
       "  'std_fit_time': array([17.08843233, 35.17996665, 33.56513422,  1.57314081,  2.05426337,\n",
       "          3.69779463, 18.71620904,  0.73856065,  0.91031828, 11.93001935,\n",
       "          6.36262953,  2.13509743,  1.99860922, 12.60677229, 11.00723791,\n",
       "         25.68344156]),\n",
       "  'mean_score_time': array([4.90152947, 4.57490571, 4.55126254, 4.15608557, 4.20296653,\n",
       "         4.97545036, 4.64245621, 4.42094024, 5.38116558, 5.14954797,\n",
       "         4.55159728, 4.61241015, 4.29149   , 4.74690652, 4.75749707,\n",
       "         3.90016373]),\n",
       "  'std_score_time': array([0.43520426, 0.08048311, 0.25211498, 0.01780867, 0.07438681,\n",
       "         0.52324615, 0.69139043, 0.58337526, 0.84065648, 0.69918281,\n",
       "         0.72527648, 0.77605558, 0.06644559, 0.37906788, 0.83835872,\n",
       "         1.55729249]),\n",
       "  'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.8, 0.8, 0.8,\n",
       "                     0.8, 0.9, 0.9, 0.9, 0.9],\n",
       "               mask=[False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False],\n",
       "         fill_value='?',\n",
       "              dtype=object),\n",
       "  'param_subsample': masked_array(data=[0.6, 0.7, 0.8, 0.9, 0.6, 0.7, 0.8, 0.9, 0.6, 0.7, 0.8,\n",
       "                     0.9, 0.6, 0.7, 0.8, 0.9],\n",
       "               mask=[False, False, False, False, False, False, False, False,\n",
       "                     False, False, False, False, False, False, False, False],\n",
       "         fill_value='?',\n",
       "              dtype=object),\n",
       "  'params': [{'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "   {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "   {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "   {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "   {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "   {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "   {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "   {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "   {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "   {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "   {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "   {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "   {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "   {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "   {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       "   {'colsample_bytree': 0.9, 'subsample': 0.9}],\n",
       "  'split0_test_score': array([0.95028235, 0.95016318, 0.95092753, 0.95127142, 0.95004276,\n",
       "         0.95023574, 0.95097062, 0.95098771, 0.94988647, 0.95034822,\n",
       "         0.9505583 , 0.95101743, 0.94973585, 0.95034408, 0.95066747,\n",
       "         0.95087721]),\n",
       "  'split1_test_score': array([0.951109  , 0.95149484, 0.95174263, 0.9520301 , 0.95109248,\n",
       "         0.95159888, 0.95155779, 0.95200317, 0.95122311, 0.95137054,\n",
       "         0.95188633, 0.95192016, 0.95107185, 0.95185093, 0.95160819,\n",
       "         0.95172183]),\n",
       "  'split2_test_score': array([0.95036746, 0.95142585, 0.95117077, 0.951277  , 0.95077832,\n",
       "         0.95103754, 0.9509543 , 0.9512968 , 0.95058118, 0.95071231,\n",
       "         0.95104321, 0.95089408, 0.95082765, 0.95095074, 0.95100569,\n",
       "         0.95083404]),\n",
       "  'mean_test_score': array([0.95058627, 0.95102795, 0.95128031, 0.95152618, 0.95063785,\n",
       "         0.95095739, 0.9511609 , 0.95142923, 0.95056359, 0.95081036,\n",
       "         0.95116261, 0.95127722, 0.95054511, 0.95104858, 0.95109378,\n",
       "         0.95114436]),\n",
       "  'std_test_score': array([0.00037126, 0.00061214, 0.00034166, 0.00035634, 0.00043991,\n",
       "         0.00055938, 0.00028072, 0.000425  , 0.00054583, 0.00042308,\n",
       "         0.0005487 , 0.0004574 , 0.00058086, 0.00061905, 0.00038907,\n",
       "         0.00040872]),\n",
       "  'rank_test_score': array([14, 10,  3,  1, 13, 11,  6,  2, 15, 12,  5,  4, 16,  9,  8,  7])},\n",
       " {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       " 0.9515261762124528)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch1.cv_results_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After three fold cross validation and gridsearch best colsample_bytree is 0.6 and subsample value is 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Modelo 6 neural net 0.8803894664626102\n",
      "AUC: 0.9527\n"
     ]
    }
   ],
   "source": [
    "#seed(2)\n",
    "mlp = XGBClassifier(scale_pos_weight=2.61, seed=42,n_estimators=1700,learning_rate=0.05,min_child_weight=4,max_depth=7,subsample=0.9,colsample_bytree=0.6)\n",
    "mlp.fit(X_train,y_train)\n",
    "pred6=mlp.predict(X_test)\n",
    "print(\"Modelo 6 neural net\",mlp.score(X_test,y_test))\n",
    "\n",
    "probs = mlp.predict_proba(X_test)\n",
    "probs = probs[:, 1]\n",
    "auc = roc_auc_score(y_test, probs)\n",
    "print('AUC: %.4f' % auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After using all the tunned parameter values final auc obtained was 0.9527. It can be increased furthur by increasing n_estimators and reducing learning rate but it can be very time consuming to train the model."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}