Prelim9_Encoding+Scaling_Ft.select_FeatureShuffling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score, mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Administrative</th>\n",
       "      <th>Administrative_Duration</th>\n",
       "      <th>Informational</th>\n",
       "      <th>Informational_Duration</th>\n",
       "      <th>ProductRelated</th>\n",
       "      <th>ProductRelated_Duration</th>\n",
       "      <th>BounceRates</th>\n",
       "      <th>ExitRates</th>\n",
       "      <th>PageValues</th>\n",
       "      <th>SpecialDay</th>\n",
       "      <th>Month</th>\n",
       "      <th>OperatingSystems</th>\n",
       "      <th>Browser</th>\n",
       "      <th>Region</th>\n",
       "      <th>TrafficType</th>\n",
       "      <th>VisitorType</th>\n",
       "      <th>Weekend</th>\n",
       "      <th>Revenue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>627.500000</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Administrative  Administrative_Duration  Informational  \\\n",
       "0               0                      0.0              0   \n",
       "1               0                      0.0              0   \n",
       "2               0                      0.0              0   \n",
       "3               0                      0.0              0   \n",
       "4               0                      0.0              0   \n",
       "\n",
       "   Informational_Duration  ProductRelated  ProductRelated_Duration  \\\n",
       "0                     0.0               1                 0.000000   \n",
       "1                     0.0               2                64.000000   \n",
       "2                     0.0               1                 0.000000   \n",
       "3                     0.0               2                 2.666667   \n",
       "4                     0.0              10               627.500000   \n",
       "\n",
       "   BounceRates  ExitRates  PageValues  SpecialDay Month  OperatingSystems  \\\n",
       "0         0.20       0.20         0.0         0.0   Feb                 1   \n",
       "1         0.00       0.10         0.0         0.0   Feb                 2   \n",
       "2         0.20       0.20         0.0         0.0   Feb                 4   \n",
       "3         0.05       0.14         0.0         0.0   Feb                 3   \n",
       "4         0.02       0.05         0.0         0.0   Feb                 3   \n",
       "\n",
       "   Browser  Region  TrafficType        VisitorType  Weekend  Revenue  \n",
       "0        1       1            1  Returning_Visitor    False    False  \n",
       "1        2       1            2  Returning_Visitor    False    False  \n",
       "2        1       9            3  Returning_Visitor    False    False  \n",
       "3        2       2            4  Returning_Visitor    False    False  \n",
       "4        3       1            4  Returning_Visitor     True    False  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_columns', 60)\n",
    "myData=pd.read_csv('imputedData.csv')\n",
    "myData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 12330 entries, 0 to 12329\n",
      "Data columns (total 57 columns):\n",
      "Administrative                   12330 non-null int64\n",
      "Administrative_Duration          12330 non-null float64\n",
      "Informational                    12330 non-null int64\n",
      "Informational_Duration           12330 non-null float64\n",
      "ProductRelated                   12330 non-null int64\n",
      "ProductRelated_Duration          12330 non-null float64\n",
      "BounceRates                      12330 non-null float64\n",
      "ExitRates                        12330 non-null float64\n",
      "PageValues                       12330 non-null float64\n",
      "TrafficType                      12330 non-null int64\n",
      "SpecialDay                       12330 non-null float64\n",
      "Month_Aug                        12330 non-null uint8\n",
      "Month_Dec                        12330 non-null uint8\n",
      "Month_Feb                        12330 non-null uint8\n",
      "Month_Jul                        12330 non-null uint8\n",
      "Month_June                       12330 non-null uint8\n",
      "Month_Mar                        12330 non-null uint8\n",
      "Month_May                        12330 non-null uint8\n",
      "Month_Nov                        12330 non-null uint8\n",
      "Month_Oct                        12330 non-null uint8\n",
      "Month_Sep                        12330 non-null uint8\n",
      "OperatingSystems_1               12330 non-null uint8\n",
      "OperatingSystems_2               12330 non-null uint8\n",
      "OperatingSystems_3               12330 non-null uint8\n",
      "OperatingSystems_4               12330 non-null uint8\n",
      "OperatingSystems_5               12330 non-null uint8\n",
      "OperatingSystems_6               12330 non-null uint8\n",
      "OperatingSystems_7               12330 non-null uint8\n",
      "OperatingSystems_8               12330 non-null uint8\n",
      "Browser_1                        12330 non-null uint8\n",
      "Browser_2                        12330 non-null uint8\n",
      "Browser_3                        12330 non-null uint8\n",
      "Browser_4                        12330 non-null uint8\n",
      "Browser_5                        12330 non-null uint8\n",
      "Browser_6                        12330 non-null uint8\n",
      "Browser_7                        12330 non-null uint8\n",
      "Browser_8                        12330 non-null uint8\n",
      "Browser_9                        12330 non-null uint8\n",
      "Browser_10                       12330 non-null uint8\n",
      "Browser_11                       12330 non-null uint8\n",
      "Browser_12                       12330 non-null uint8\n",
      "Browser_13                       12330 non-null uint8\n",
      "Region_1                         12330 non-null uint8\n",
      "Region_2                         12330 non-null uint8\n",
      "Region_3                         12330 non-null uint8\n",
      "Region_4                         12330 non-null uint8\n",
      "Region_5                         12330 non-null uint8\n",
      "Region_6                         12330 non-null uint8\n",
      "Region_7                         12330 non-null uint8\n",
      "Region_8                         12330 non-null uint8\n",
      "Region_9                         12330 non-null uint8\n",
      "VisitorType_New_Visitor          12330 non-null uint8\n",
      "VisitorType_Other                12330 non-null uint8\n",
      "VisitorType_Returning_Visitor    12330 non-null uint8\n",
      "Weekend_False                    12330 non-null uint8\n",
      "Weekend_True                     12330 non-null uint8\n",
      "Revenue                          12330 non-null bool\n",
      "dtypes: bool(1), float64(7), int64(4), uint8(45)\n",
      "memory usage: 1.6 MB\n"
     ]
    }
   ],
   "source": [
    "#One hot encoding of categorical variables\n",
    "\n",
    "#Create list with features to be dummified cols.\n",
    "nonum_feats_names = ['Month','OperatingSystems','Browser','Region','VisitorType','Weekend']\n",
    "\n",
    "\n",
    "dataModel = pd.concat([myData[['Administrative', 'Administrative_Duration', 'Informational','Informational_Duration', \n",
    "                              'ProductRelated','ProductRelated_Duration','BounceRates','ExitRates','PageValues',\n",
    "                               'TrafficType','SpecialDay']],\n",
    "                       pd.get_dummies(myData[nonum_feats_names].astype('category')),myData['Revenue']],axis=1)\n",
    "                      \n",
    "dataModel.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of x: (12330, 56)\n",
      "Shape of y: (12330,)\n"
     ]
    }
   ],
   "source": [
    "#Label Encoding of revenue\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "le = LabelEncoder()\n",
    "myData['Revenue'] = le.fit_transform(myData['Revenue'])\n",
    "myData['Revenue'].value_counts()\n",
    "\n",
    "myData['Revenue'].head()\n",
    "\n",
    "# getting dependent and independent variables\n",
    "\n",
    "x = dataModel\n",
    "# removing the target column revenue from x\n",
    "x = x.drop(['Revenue'], axis = 1)\n",
    "\n",
    "y = myData['Revenue']\n",
    "\n",
    "# checking the shapes\n",
    "print(\"Shape of x:\", x.shape)\n",
    "print(\"Shape of y:\", y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of x_train : (8631, 56)\n",
      "Shape of y_train : (8631,)\n",
      "Shape of x_test : (3699, 56)\n",
      "Shape of y_test : (3699,)\n"
     ]
    }
   ],
   "source": [
    "# splitting the data\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_baseTrain, x_baseTest, y_baseTrain, y_baseTest = train_test_split(x, y, test_size = 0.3, random_state = 42)\n",
    "\n",
    "# checking the shapes\n",
    "\n",
    "print(\"Shape of x_train :\", x_baseTrain.shape)\n",
    "print(\"Shape of y_train :\", y_baseTrain.shape)\n",
    "print(\"Shape of x_test :\", x_baseTest.shape)\n",
    "print(\"Shape of y_test :\", y_baseTest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\neera\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\neera\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:25: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "#SCALING WITH STANDARD Z SCORE SCALER\n",
    "#Scaling the data first fitting it and transforming the training set\n",
    "#to later apply the fit to transform the test set.\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "num_cols_names = ['Administrative', 'Administrative_Duration', 'Informational','Informational_Duration', \n",
    "                              'ProductRelated','ProductRelated_Duration','BounceRates','ExitRates','PageValues',\n",
    "                               'TrafficType','SpecialDay']\n",
    "\n",
    "#Instantiate Satandard Scaler\n",
    "scaler = StandardScaler()\n",
    "#Fit transform the numerical features in the training dataset to a new dataframe\n",
    "scaled_numfeats_train = pd.DataFrame(scaler.fit_transform(x_baseTrain[num_cols_names]), \n",
    "                                     columns=num_cols_names, index= x_baseTrain.index)\n",
    "#Integrate scaled values to the training set\n",
    "for col in num_cols_names:\n",
    "    x_baseTrain[col] = scaled_numfeats_train[col]\n",
    "    \n",
    "    \n",
    "#Transform the numerical features inthe training dataset to a new dataframe\n",
    "scaled_numfeats_test = pd.DataFrame(scaler.transform(x_baseTest[num_cols_names]),\n",
    "                                    columns=num_cols_names, index= x_baseTest.index)\n",
    "#Integrate scaled values to the test set\n",
    "for col in num_cols_names:\n",
    "    x_baseTest[col] = scaled_numfeats_test[col]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In all feature selection procedures, it is good practice to select the features by examining only the training set. And this is to avoid overfit. applying this feature selection on corr fixed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\neera\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:4102: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((8631, 52), (3699, 52))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_baseTrain.drop(labels=['Browser_1','BounceRates','ProductRelated','VisitorType_Returning_Visitor'], \n",
    "                 axis=1, inplace=True)\n",
    "x_baseTest.drop(labels=['Browser_1','BounceRates','ProductRelated','VisitorType_Returning_Visitor'], \n",
    "                axis=1, inplace=True)\n",
    "\n",
    "x_baseTrain.shape, x_baseTest.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train auc score:  0.8967925216437023\n",
      "test auc score:  0.8932953849579692\n"
     ]
    }
   ],
   "source": [
    "# The first step to determine feature importance by feature shuffling\n",
    "# is to build the machine learning model for which we want to \n",
    "# select features\n",
    "\n",
    "# In this case, I will build Random Forests, but remember that \n",
    "# you can use this procedure for any other machine learning algorithm\n",
    "\n",
    "# I build few and shallow trees to avoid overfitting\n",
    "rf = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=2909, n_jobs=4)\n",
    "\n",
    "rf.fit(x_baseTrain, y_baseTrain)\n",
    "\n",
    "# print roc-auc in train and testing sets\n",
    "print('train auc score: ',roc_auc_score(y_baseTrain, (rf.predict_proba(x_baseTrain)[:, 1])))\n",
    "print('test auc score: ',roc_auc_score(y_baseTest, (rf.predict_proba(x_baseTest)[:, 1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in this cell, I will shuffle one by one, each feature of the dataset\n",
    "# and then use the dataset with the shuffled variable to make predictions\n",
    "# using the random forests I trained in the previous cell\n",
    "\n",
    "# overall train roc-auc: using all the features\n",
    "train_auc = roc_auc_score(y_baseTrain, (rf.predict_proba(x_baseTrain)[:, 1]))\n",
    "\n",
    "# dictionary to capture the features and the drop in auc that they\n",
    "# cause when shuffled\n",
    "feature_dict = {}\n",
    "\n",
    "# selection  logic\n",
    "for feature in x_baseTrain.columns:\n",
    "    X_train_c = x_baseTrain.copy()\n",
    "    \n",
    "    # shuffle individual feature\n",
    "    X_train_c[feature] = X_train_c[feature].sample(frac=1).reset_index(\n",
    "        drop=True)\n",
    "    \n",
    "    # make prediction with shuffled feature and calculate roc-auc\n",
    "    shuff_auc = roc_auc_score(y_baseTrain,\n",
    "                              (rf.predict_proba(X_train_c.fillna(0)))[:, 1])\n",
    "    \n",
    "    # save the drop in roc-auc\n",
    "    feature_dict[feature] = (train_auc - shuff_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Administrative': -0.0018167223362430818,\n",
       " 'Administrative_Duration': -0.005170722661482108,\n",
       " 'Informational': -0.002948890826433681,\n",
       " 'Informational_Duration': -0.003193231166108812,\n",
       " 'ProductRelated_Duration': -0.002490534253185017,\n",
       " 'ExitRates': 0.013109933416486474,\n",
       " 'PageValues': 0.22948111651097203,\n",
       " 'TrafficType': 0.0005185936111323786,\n",
       " 'SpecialDay': 0.0005714295112556611,\n",
       " 'Month_Aug': 3.844479892245456e-05,\n",
       " 'Month_Dec': 4.928952161309397e-05,\n",
       " 'Month_Feb': 2.0044748101133436e-05,\n",
       " 'Month_Jul': 1.4236910830911498e-05,\n",
       " 'Month_June': 2.353972982149788e-05,\n",
       " 'Month_Mar': 0.00013830876189868668,\n",
       " 'Month_May': 0.000831394475091729,\n",
       " 'Month_Nov': 0.017324829974279088,\n",
       " 'Month_Oct': 0.0002629459776564147,\n",
       " 'Month_Sep': 0.00015557808333965806,\n",
       " 'OperatingSystems_1': 0.0,\n",
       " 'OperatingSystems_2': 0.0003160388617297061,\n",
       " 'OperatingSystems_3': 0.00022989784168436778,\n",
       " 'OperatingSystems_4': 0.0,\n",
       " 'OperatingSystems_5': -2.6726330801807308e-06,\n",
       " 'OperatingSystems_6': -3.5977753002347512e-06,\n",
       " 'OperatingSystems_7': 0.0,\n",
       " 'OperatingSystems_8': 0.0,\n",
       " 'Browser_2': 1.0279358009235295e-07,\n",
       " 'Browser_3': 0.0,\n",
       " 'Browser_4': 3.217439054203908e-05,\n",
       " 'Browser_5': 0.0,\n",
       " 'Browser_6': 1.7680495761118742e-05,\n",
       " 'Browser_7': 0.0,\n",
       " 'Browser_8': 0.0,\n",
       " 'Browser_9': 0.0,\n",
       " 'Browser_10': 0.0,\n",
       " 'Browser_11': 4.111743201473672e-07,\n",
       " 'Browser_12': 1.716652786121209e-05,\n",
       " 'Browser_13': -1.757770218102639e-05,\n",
       " 'Region_1': 2.528722068162459e-05,\n",
       " 'Region_2': 0.0,\n",
       " 'Region_3': 0.0,\n",
       " 'Region_4': 2.282017476140652e-05,\n",
       " 'Region_5': 3.5360991522237484e-05,\n",
       " 'Region_6': 5.139679004617648e-08,\n",
       " 'Region_7': 0.0,\n",
       " 'Region_8': 0.0,\n",
       " 'Region_9': -1.747490860015688e-06,\n",
       " 'VisitorType_New_Visitor': 0.0031066789717434373,\n",
       " 'VisitorType_Other': 0.0,\n",
       " 'Weekend_False': -0.00016189988851011972,\n",
       " 'Weekend_True': 6.78437628054418e-06}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's have a look at the generated dictionary\n",
    "feature_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>auc_drop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Administrative</td>\n",
       "      <td>-0.00182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Administrative_Duration</td>\n",
       "      <td>-0.00517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Informational</td>\n",
       "      <td>-0.00295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Informational_Duration</td>\n",
       "      <td>-0.00319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>ProductRelated_Duration</td>\n",
       "      <td>-0.00249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>ExitRates</td>\n",
       "      <td>0.01311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>PageValues</td>\n",
       "      <td>0.22948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>TrafficType</td>\n",
       "      <td>0.00052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>SpecialDay</td>\n",
       "      <td>0.00057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>Month_Aug</td>\n",
       "      <td>0.00004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>Month_Dec</td>\n",
       "      <td>0.00005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>Month_Feb</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>Month_Jul</td>\n",
       "      <td>0.00001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>Month_June</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>Month_Mar</td>\n",
       "      <td>0.00014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>Month_May</td>\n",
       "      <td>0.00083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>Month_Nov</td>\n",
       "      <td>0.01732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>Month_Oct</td>\n",
       "      <td>0.00026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>Month_Sep</td>\n",
       "      <td>0.00016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>OperatingSystems_1</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>OperatingSystems_2</td>\n",
       "      <td>0.00032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>OperatingSystems_3</td>\n",
       "      <td>0.00023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>OperatingSystems_4</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>OperatingSystems_5</td>\n",
       "      <td>-0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>OperatingSystems_6</td>\n",
       "      <td>-0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>OperatingSystems_7</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>OperatingSystems_8</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>Browser_2</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>Browser_3</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>Browser_4</td>\n",
       "      <td>0.00003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>Browser_5</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>Browser_6</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>Browser_7</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>Browser_8</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>Browser_9</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>Browser_10</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>Browser_11</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>Browser_12</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>Browser_13</td>\n",
       "      <td>-0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>Region_1</td>\n",
       "      <td>0.00003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>Region_2</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>Region_3</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>Region_4</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>Region_5</td>\n",
       "      <td>0.00004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>Region_6</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>Region_7</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>Region_8</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>Region_9</td>\n",
       "      <td>-0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>VisitorType_New_Visitor</td>\n",
       "      <td>0.00311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>VisitorType_Other</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>Weekend_False</td>\n",
       "      <td>-0.00016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>Weekend_True</td>\n",
       "      <td>0.00001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    feature  auc_drop\n",
       "0            Administrative  -0.00182\n",
       "1   Administrative_Duration  -0.00517\n",
       "2             Informational  -0.00295\n",
       "3    Informational_Duration  -0.00319\n",
       "4   ProductRelated_Duration  -0.00249\n",
       "5                 ExitRates   0.01311\n",
       "6                PageValues   0.22948\n",
       "7               TrafficType   0.00052\n",
       "8                SpecialDay   0.00057\n",
       "9                 Month_Aug   0.00004\n",
       "10                Month_Dec   0.00005\n",
       "11                Month_Feb   0.00002\n",
       "12                Month_Jul   0.00001\n",
       "13               Month_June   0.00002\n",
       "14                Month_Mar   0.00014\n",
       "15                Month_May   0.00083\n",
       "16                Month_Nov   0.01732\n",
       "17                Month_Oct   0.00026\n",
       "18                Month_Sep   0.00016\n",
       "19       OperatingSystems_1   0.00000\n",
       "20       OperatingSystems_2   0.00032\n",
       "21       OperatingSystems_3   0.00023\n",
       "22       OperatingSystems_4   0.00000\n",
       "23       OperatingSystems_5  -0.00000\n",
       "24       OperatingSystems_6  -0.00000\n",
       "25       OperatingSystems_7   0.00000\n",
       "26       OperatingSystems_8   0.00000\n",
       "27                Browser_2   0.00000\n",
       "28                Browser_3   0.00000\n",
       "29                Browser_4   0.00003\n",
       "30                Browser_5   0.00000\n",
       "31                Browser_6   0.00002\n",
       "32                Browser_7   0.00000\n",
       "33                Browser_8   0.00000\n",
       "34                Browser_9   0.00000\n",
       "35               Browser_10   0.00000\n",
       "36               Browser_11   0.00000\n",
       "37               Browser_12   0.00002\n",
       "38               Browser_13  -0.00002\n",
       "39                 Region_1   0.00003\n",
       "40                 Region_2   0.00000\n",
       "41                 Region_3   0.00000\n",
       "42                 Region_4   0.00002\n",
       "43                 Region_5   0.00004\n",
       "44                 Region_6   0.00000\n",
       "45                 Region_7   0.00000\n",
       "46                 Region_8   0.00000\n",
       "47                 Region_9  -0.00000\n",
       "48  VisitorType_New_Visitor   0.00311\n",
       "49        VisitorType_Other   0.00000\n",
       "50            Weekend_False  -0.00016\n",
       "51             Weekend_True   0.00001"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now I will transform the dictionary into a pandas dataframe\n",
    "# for easy manipulation\n",
    "pd.set_option('display.float_format', lambda x: '%.5f' % x)\n",
    "\n",
    "feature_importance = pd.Series(feature_dict).reset_index()\n",
    "feature_importance.columns = ['feature', 'auc_drop']\n",
    "feature_importance.head(52)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now I will sort the dataframe according to the drop in performance\n",
    "# caused by feature shuffling\n",
    "\n",
    "feature_importance.sort_values(by=['auc_drop'], ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>auc_drop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>PageValues</td>\n",
       "      <td>0.22948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>Month_Nov</td>\n",
       "      <td>0.01732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>ExitRates</td>\n",
       "      <td>0.01311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>VisitorType_New_Visitor</td>\n",
       "      <td>0.00311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>Month_May</td>\n",
       "      <td>0.00083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>SpecialDay</td>\n",
       "      <td>0.00057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>TrafficType</td>\n",
       "      <td>0.00052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>OperatingSystems_2</td>\n",
       "      <td>0.00032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>Month_Oct</td>\n",
       "      <td>0.00026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>OperatingSystems_3</td>\n",
       "      <td>0.00023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>Month_Sep</td>\n",
       "      <td>0.00016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>Month_Mar</td>\n",
       "      <td>0.00014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>Month_Dec</td>\n",
       "      <td>0.00005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>Month_Aug</td>\n",
       "      <td>0.00004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>Region_5</td>\n",
       "      <td>0.00004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>Browser_4</td>\n",
       "      <td>0.00003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>Region_1</td>\n",
       "      <td>0.00003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>Month_June</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>Region_4</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>Month_Feb</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>Browser_6</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>Browser_12</td>\n",
       "      <td>0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>Month_Jul</td>\n",
       "      <td>0.00001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>Weekend_True</td>\n",
       "      <td>0.00001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>Browser_11</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    feature  auc_drop\n",
       "6                PageValues   0.22948\n",
       "16                Month_Nov   0.01732\n",
       "5                 ExitRates   0.01311\n",
       "48  VisitorType_New_Visitor   0.00311\n",
       "15                Month_May   0.00083\n",
       "8                SpecialDay   0.00057\n",
       "7               TrafficType   0.00052\n",
       "20       OperatingSystems_2   0.00032\n",
       "17                Month_Oct   0.00026\n",
       "21       OperatingSystems_3   0.00023\n",
       "18                Month_Sep   0.00016\n",
       "14                Month_Mar   0.00014\n",
       "10                Month_Dec   0.00005\n",
       "9                 Month_Aug   0.00004\n",
       "43                 Region_5   0.00004\n",
       "29                Browser_4   0.00003\n",
       "39                 Region_1   0.00003\n",
       "13               Month_June   0.00002\n",
       "42                 Region_4   0.00002\n",
       "11                Month_Feb   0.00002\n",
       "31                Browser_6   0.00002\n",
       "37               Browser_12   0.00002\n",
       "12                Month_Jul   0.00001\n",
       "51             Weekend_True   0.00001\n",
       "36               Browser_11   0.00000"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualise the top 25 features that caused the major drop\n",
    "# in roc-auc (aka model performance)\n",
    "\n",
    "feature_importance.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>auc_drop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>Browser_2</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>Region_6</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>Region_3</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>Region_2</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>Region_7</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>Browser_9</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>Region_8</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>VisitorType_Other</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>Browser_10</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>OperatingSystems_8</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>Browser_8</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>Browser_7</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>Browser_5</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>Browser_3</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>OperatingSystems_1</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>OperatingSystems_7</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>OperatingSystems_4</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>Region_9</td>\n",
       "      <td>-0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>OperatingSystems_5</td>\n",
       "      <td>-0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>OperatingSystems_6</td>\n",
       "      <td>-0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>Browser_13</td>\n",
       "      <td>-0.00002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>Weekend_False</td>\n",
       "      <td>-0.00016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Administrative</td>\n",
       "      <td>-0.00182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>ProductRelated_Duration</td>\n",
       "      <td>-0.00249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Informational</td>\n",
       "      <td>-0.00295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Informational_Duration</td>\n",
       "      <td>-0.00319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Administrative_Duration</td>\n",
       "      <td>-0.00517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    feature  auc_drop\n",
       "27                Browser_2   0.00000\n",
       "44                 Region_6   0.00000\n",
       "41                 Region_3   0.00000\n",
       "40                 Region_2   0.00000\n",
       "45                 Region_7   0.00000\n",
       "34                Browser_9   0.00000\n",
       "46                 Region_8   0.00000\n",
       "49        VisitorType_Other   0.00000\n",
       "35               Browser_10   0.00000\n",
       "26       OperatingSystems_8   0.00000\n",
       "33                Browser_8   0.00000\n",
       "32                Browser_7   0.00000\n",
       "30                Browser_5   0.00000\n",
       "28                Browser_3   0.00000\n",
       "19       OperatingSystems_1   0.00000\n",
       "25       OperatingSystems_7   0.00000\n",
       "22       OperatingSystems_4   0.00000\n",
       "47                 Region_9  -0.00000\n",
       "23       OperatingSystems_5  -0.00000\n",
       "24       OperatingSystems_6  -0.00000\n",
       "38               Browser_13  -0.00002\n",
       "50            Weekend_False  -0.00016\n",
       "0            Administrative  -0.00182\n",
       "4   ProductRelated_Duration  -0.00249\n",
       "2             Informational  -0.00295\n",
       "3    Informational_Duration  -0.00319\n",
       "1   Administrative_Duration  -0.00517"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# visualise the features that caused virtually \n",
    "# no drop in performance\n",
    "\n",
    "feature_importance.tail(27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# original number of features (rows in this case)\n",
    "feature_importance.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of features that cause a drop in performance\n",
    "# when shuffled\n",
    "\n",
    "feature_importance[feature_importance.auc_drop>0].shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only 27 out of the 52 features caused a drop in the performance of the random forests when their values were permuted. This means that we could select those features and discard the rest(25), and should keep the original random forest performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3YAAAHfCAYAAAAcIUqCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzde5glZXnv/e8PEMQDEzZOouEg6KAJnqKOx5hsDR5w64hRUNBs0bBlx0jUuNUXE6OGSKLGaNSoCQZP2SqgRgUh4lnjAWVUPCCbOAEURCNGHYkKiN7vH1UNa5pe0wvoqupa/f1c11yzVq1a665aXV1ddz3Pcz+pKiRJkiRJ47XD0BsgSZIkSbphTOwkSZIkaeRM7CRJkiRp5EzsJEmSJGnkTOwkSZIkaeRM7CRJkiRp5HYaegOui1vc4ha17777Dr0ZkiRJkjSIz3/+89+rqvWLl48qsdt3333ZvHnz0JshSZIkSYNI8o2lltsVU5IkSZJGzsROkiRJkkZuFIldkk1Jjt+6devQmyJJkiRJq84oEruqOrWqjlq3bt3QmyJJkiRJq84oEjtJkiRJ0nQmdpIkSZI0ciZ2kiRJkjRyJnaSJEmSNHKjSOysiilJkiRJ040isbMqpiRJkiRNN4rETpIkSZI0nYmdJEmSJI2ciZ0kSZIkjdxOQ2/AStn3mNNu0PsvfPHDVmhLJEmSJKlftthJkiRJ0siNIrFzugNJkiRJmm4UiZ3THUiSJEnSdKNI7CRJkiRJ05nYSZIkSdLImdhJkiRJ0siZ2EmSJEnSyJnYSZIkSdLImdhJkiRJ0siZ2EmSJEnSyJnYSZIkSdLIzZTYJTkoyXlJtiQ5ZonXfzvJF5JcleSQRa8dkeTr7b8jJpbfPclX2s98VZLc8N2RJEmSpLVn2cQuyY7Aa4CHAgcAhyc5YNFq3wSeCLxt0Xv/G/AC4F7APYEXJNm9ffl1wFHA/u2/g673XkiSJEnSGjZLi909gS1VdX5VXQmcCBw8uUJVXVhVXwZ+sei9DwE+WFXfr6ofAB8EDkpyK2C3qvpMVRXwFuCRN3RnJEmSJGktmiWx2xO4aOL5xe2yWUx7757t4+vzmZIkSZKkCbMkdkuNfasZP3/ae2f+zCRHJdmcZPOll146Y1hJkiRJWjtmSewuBvaeeL4XcMmMnz/tvRe3j5f9zKo6vqo2VtXG9evXzxhWkiRJktaOWRK7s4D9k+yXZGfgMOCUGT//DODBSXZvi6Y8GDijqr4NXJbk3m01zCcA7532IUk2JTl+69atM4aVJEmSpLVj2cSuqq4CjqZJ0s4FTq6qc5Icm+QRAEnukeRi4FDgH5Kc0773+8Bf0CSHZwHHtssAngL8I7AF+HfgX7azDadW1VHr1q27nrspSZIkSfNrp1lWqqrTgdMXLXv+xOOz2LZr5eR6bwDesMTyzcAdZ4mfZBOwacOGDbOsLkmSJElrykwTlA/NFjtJkiRJmm4UiZ0kSZIkabpRJHYWT5EkSZKk6UaR2NkVU5IkSZKmG0ViJ0mSJEmabhSJnV0xJUmSJGm6USR2dsWUJEmSpOlGkdhJkiRJkqYbRWJnV0xJkiRJmm4UiZ1dMSVJkiRpulEkdpIkSZKk6UzsJEmSJGnkTOwkSZIkaeRGkdhZPEWSJEmSphtFYmfxFEmSJEmabhSJnSRJkiRpOhM7SZIkSRo5EztJkiRJGrlRJHYWT5EkSZKk6UaR2Fk8RZIkSZKmG0ViJ0mSJEmazsROkiRJkkbOxE6SJEmSRs7ETpIkSZJGzsROkiRJkkbOxE6SJEmSRm4UiZ3z2EmSJEnSdKNI7JzHTpIkSZKmG0ViJ0mSJEmabqehN2Ae7HvMaTfo/Re++GErtCWSJEmS1iJb7CRJkiRp5EzsJEmSJGnkTOwkSZIkaeRM7CRJkiRp5CyeMnIWbpEkSZJki50kSZIkjdxMiV2Sg5Kcl2RLkmOWeH2XJCe1r382yb7t8j2SfDTJfyX5u0XvuXuSr7TveVWSrMQOSZIkSdJas2xil2RH4DXAQ4EDgMOTHLBotSOBH1TVBuAVwEva5ZcDfwY8a4mPfh1wFLB/+++g67MDkiRJkrTWzdJid09gS1WdX1VXAicCBy9a52Dgze3jdwIHJklV/biqPkmT4F0tya2A3arqM1VVwFuAR96QHZEkSZKktWqWxG5P4KKJ5xe3y5Zcp6quArYCeyzzmRcv85kAJDkqyeYkmy+99NIZNleSJEmS1pZZErulxr7V9Vjneq1fVcdX1caq2rh+/frtfKQkSZIkrU2zJHYXA3tPPN8LuGTaOkl2AtYB31/mM/da5jOvlmRTkuO3bt06w+ZKkiRJ0toyS2J3FrB/kv2S7AwcBpyyaJ1TgCPax4cAH2nHzi2pqr4NXJbk3m01zCcA793O+qdW1VHr1q2bYXMlSZIkaW1ZdoLyqroqydHAGcCOwBuq6pwkxwKbq+oU4ATgn5JsoWmpO2zh/UkuBHYDdk7ySODBVfU14CnAm4BdgX9p/0mSJEmSrqNlEzuAqjodOH3RsudPPL4cOHTKe/edsnwzcMdZ4ifZBGzasGHDLKtLkiRJ0poy0wTlQ7MrpiRJkiRNN4rEzuIpkiRJkjTdKBI7W+wkSZIkabpRJHaSJEmSpOlGkdjZFVOSJEmSphtFYmdXTEmSJEmabhSJnSRJkiRpOhM7SZIkSRq5USR2jrGTJEmSpOlGkdg5xk6SJEmSphtFYidJkiRJms7ETpIkSZJGbhSJnWPsJEmSJGm6USR2jrGTJEmSpOlGkdhJkiRJkqYzsZMkSZKkkTOxkyRJkqSR22noDdB47XvMaTfo/Re++GErtCWSJEnS2jaKFjurYkqSJEnSdKNI7KyKKUmSJEnTjSKxkyRJkiRNZ2InSZIkSSNnYidJkiRJI2diJ0mSJEkjZ2InSZIkSSNnYidJkiRJIzeKxM557CRJkiRpulEkds5jJ0mSJEnTjSKxkyRJkiRNZ2InSZIkSSNnYidJkiRJI2diJ0mSJEkjZ2InSZIkSSNnYidJkiRJI2diJ0mSJEkjN1Nil+SgJOcl2ZLkmCVe3yXJSe3rn02y78Rrz22Xn5fkIRPL35Dku0m+uhI7IkmSJElr1bKJXZIdgdcADwUOAA5PcsCi1Y4EflBVG4BXAC9p33sAcBhwB+Ag4LXt5wG8qV0mSZIkSboBZmmxuyewparOr6orgROBgxetczDw5vbxO4EDk6RdfmJVXVFVFwBb2s+jqj4BfH8F9kGSJEmS1rRZErs9gYsmnl/cLltynaq6CtgK7DHjeyVJkiRJN8AsiV2WWFYzrjPLe7cfPDkqyeYkmy+99NLr8lZJkiRJWhNmSewuBvaeeL4XcMm0dZLsBKyj6WY5y3u3q6qOr6qNVbVx/fr11+WtkiRJkrQmzJLYnQXsn2S/JDvTFEM5ZdE6pwBHtI8PAT5SVdUuP6ytmrkfsD/wueu6kUk2JTl+69at1/WtkiRJkjT3lk3s2jFzRwNnAOcCJ1fVOUmOTfKIdrUTgD2SbAGeCRzTvvcc4GTga8D7gadW1c8Bkrwd+Axw+yQXJzlyO9twalUdtW7duuu7n5IkSZI0t3aaZaWqOh04fdGy5088vhw4dMp7jwOOW2L54bNuZJJNwKYNGzbM+hbNuX2POe0Gvf/CFz9shbZEkiRJGt5ME5QPzRY7SZIkSZpuFImdJEmSJGm6USR2Fk+RJEmSpOlGkdjZFVOSJEmSphtFYidJkiRJmm4UiZ1dMSVJkiRpupmmOxhaVZ0KnLpx48YnD70tklMtSJIkabUZRYudJEmSJGm6UbTYOUG51LC1UJIkSUsZRYudVTElSZIkabpRJHaSJEmSpOlM7CRJkiRp5EzsJEmSJGnkRpHYOY+dJEmSJE03isTO4imSJEmSNN0oEjtJkiRJ0nQmdpIkSZI0cqOYoFzS8JwcXZIkafUaRYudxVMkSZIkabpRJHYWT5EkSZKk6UaR2EmSJEmSpnOMnaRVz/F9kiRJ22eLnSRJkiSNnC12krQdthZKkqQxsMVOkiRJkkbOxE6SJEmSRm4UiZ3z2EmSJEnSdKNI7JzHTpIkSZKmG0ViJ0mSJEmazsROkiRJkkbOxE6SJEmSRs557CRplXIOPUmSNCtb7CRJkiRp5GyxkyRdi62FkiSNiy12kiRJkjRyJnaSJEmSNHIzJXZJDkpyXpItSY5Z4vVdkpzUvv7ZJPtOvPbcdvl5SR4y62dKkiRJkmazbGKXZEfgNcBDgQOAw5McsGi1I4EfVNUG4BXAS9r3HgAcBtwBOAh4bZIdZ/xMSZIkSdIMZmmxuyewparOr6orgROBgxetczDw5vbxO4EDk6RdfmJVXVFVFwBb2s+b5TMlSZIkSTOYJbHbE7ho4vnF7bIl16mqq4CtwB7bee8snylJkiRJmsEsiV2WWFYzrnNdl187eHJUks1JNl966aXb3VBJkiRJWotmSewuBvaeeL4XcMm0dZLsBKwDvr+d987ymQBU1fFVtbGqNq5fv36GzZUkSZKktWWWxO4sYP8k+yXZmaYYyimL1jkFOKJ9fAjwkaqqdvlhbdXM/YD9gc/N+JlXS7IpyfFbt269LvsmSZIkSWvCsoldO2buaOAM4Fzg5Ko6J8mxSR7RrnYCsEeSLcAzgWPa954DnAx8DXg/8NSq+vm0z9zONpxaVUetW7fu+u6nJEmSJM2tnWZZqapOB05ftOz5E48vBw6d8t7jgONm+cxpkmwCNm3YsGGW1SVJkiRpTZlpgvKh2WInSZIkSdONIrFzjJ0kSZIkTTeKxM4WO0mSJEmabhSJnSRJkiRpulEkdnbFlCRJkqTpRpHY2RVTkiRJkqYbRWInSZIkSZrOxE6SJEmSRm6mCcqH5gTlkrR27HvMaTfo/Re++GErtCWSJI3HKFrsHGMnSZIkSdONIrGTJEmSJE1nYidJkiRJIzeKxM557CRJkiRpulEUT6mqU4FTN27c+OSht0WSNL8s3CJJGqtRtNhJkiRJkqYbRYudJEnzztZCSdINYYudJEmSJI2cLXaSJK1xthZK0viNosXOqpiSJEmSNN0oEruqOrWqjlq3bt3QmyJJkiRJq84oEjtJkiRJ0nSOsZMkSYNxfJ8krQxb7CRJkiRp5GyxkyRJa9KQrYW2VEpaabbYSZIkSdLImdhJkiRJ0siNoitmkk3Apg0bNgy9KZIkSaNmN1BpPo2ixc557CRJkiRpulG02EmSJGn8bC2UumNiJ0mSpLlnUql5N4qumJIkSZKk6UzsJEmSJGnk7IopSZIkdchuoOqDLXaSJEmSNHImdpIkSZI0cjMldkkOSnJeki1Jjlni9V2SnNS+/tkk+0689tx2+XlJHjLrZ0qSJEmSZrNsYpdkR+A1wEOBA4DDkxywaLUjgR9U1QbgFcBL2vceABwG3AE4CHhtkh1n/ExJkiRJ0gxmabG7J7Clqs6vqiuBE4GDF61zMPDm9vE7gQOTpF1+YlVdUVUXAFvaz5vlMyVJkiRJM5glsdsTuGji+cXtsiXXqaqrgK3AHtt57yyfKUmSJEmawSyJXZZYVjOuc12XXzt4clSSzUk2X3rppdvdUEmSJElai2ZJ7C4G9p54vhdwybR1kuwErAO+v533zvKZAFTV8VW1sao2rl+/fobNlSRJkqS1ZZbE7ixg/yT7JdmZphjKKYvWOQU4on18CPCRqqp2+WFt1cz9gP2Bz834mVdLsinJ8Vu3br0u+yZJkiRJa8KyiV07Zu5o4AzgXODkqjonybFJHtGudgKwR5ItwDOBY9r3ngOcDHwNeD/w1Kr6+bTP3M42nFpVR61bt+767qckSZIkza2dZlmpqk4HTl+07PkTjy8HDp3y3uOA42b5zGmSbAI2bdiwYZbVJUmSJGlNmWmC8qHZYidJkiRJ040isZMkSZIkTTeKxM7iKZIkSZI03Uxj7IZWVacCp27cuPHJQ2+LJEmSNBb7HnPaDXr/hS9+2Aptibo2ihY7SZIkSdJ0o0js7IopSZIkSdONIrGzKqYkSZIkTTeKxE6SJEmSNJ2JnSRJkiSN3CgSO8fYSZIkSdJ0o0jsHGMnSZIkSdONIrGTJEmSJE1nYidJkiRJIzeKxM4xdpIkSZI03SgSO8fYSZIkSdJ0o0jsJEmSJEnTmdhJkiRJ0siZ2EmSJEnSyI0isbN4iiRJkiRNN4rEzuIpkiRJkjTdKBI7SZIkSdJ0JnaSJEmSNHImdpIkSZI0ciZ2kiRJkjRyJnaSJEmSNHImdpIkSZI0cqNI7JzHTpIkSZKmG0Vi5zx2kiRJkjTdKBI7SZIkSdJ0JnaSJEmSNHImdpIkSZI0ciZ2kiRJkjRyJnaSJEmSNHImdpIkSZI0ciZ2kiRJkjRyJnaSJEmSNHIzJXZJDkpyXpItSY5Z4vVdkpzUvv7ZJPtOvPbcdvl5SR4y62dKkiRJkmazbGKXZEfgNcBDgQOAw5McsGi1I4EfVNUG4BXAS9r3HgAcBtwBOAh4bZIdZ/xMSZIkSdIMZmmxuyewparOr6orgROBgxetczDw5vbxO4EDk6RdfmJVXVFVFwBb2s+b5TMlSZIkSTOYJbHbE7ho4vnF7bIl16mqq4CtwB7bee8snylJkiRJmsEsiV2WWFYzrnNdl187eHJUks1JNl966aXb3VBJkiRJWotmSewuBvaeeL4XcMm0dZLsBKwDvr+d987ymQBU1fFVtbGqNq5fv36GzZUkSZKktWWWxO4sYP8k+yXZmaYYyimL1jkFOKJ9fAjwkaqqdvlhbdXM/YD9gc/N+JlXS7IpyfFbt269LvsmSZIkSWvCsoldO2buaOAM4Fzg5Ko6J8mxSR7RrnYCsEeSLcAzgWPa954DnAx8DXg/8NSq+vm0z9zONpxaVUetW7fu+u6nJEmSJM2tnWZZqapOB05ftOz5E48vBw6d8t7jgONm+cxpkmwCNm3YsGGW1SVJkiRpTZlpgvKh2WInSZIkSdONIrGTJEmSJE03isTO4imSJEmSNN0oEju7YkqSJEnSdKNI7CRJkiRJ040isbMrpiRJkiRNN4rEzq6YkiRJkjTdKBI7SZIkSdJ0o0js7IopSZIkSdONIrGzK6YkSZIkTTeKxE6SJEmSNJ2JnSRJkiSN3E5Db8AskmwCNm3YsGHoTZEkSZI0g32POe0Gvf/CFz9shbZkbRhFi51j7CRJkiRpulEkdpIkSZKk6UzsJEmSJGnkTOwkSZIkaeRM7CRJkiRp5EaR2CXZlOT4rVu3Dr0pkiRJkrTqjCKxsyqmJEmSJE03isROkiRJkjSdiZ0kSZIkjZyJnSRJkiSNnImdJEmSJI2ciZ0kSZIkjZyJnSRJkiSN3CgSO+exkyRJkqTpRpHYOY+dJEmSJE03isROkiRJkjSdiZ0kSZIkjZyJnSRJkiSNnImdJEmSJI2ciZ0kSZIkjZyJnSRJkiSNnImdJEmSJI2ciZ0kSZIkjdxMiV2Sg5Kcl2RLkmOWeH2XJCe1r382yb4Trz23XX5ekofM+pmSJEmSpNksm9gl2RF4DfBQ4ADg8CQHLFrtSOAHVbUBeAXwkva9BwCHAXcADgJem2THGT9TkiRJkjSDWVrs7glsqarzq+pK4ETg4EXrHAy8uX38TuDAJGmXn1hVV1TVBcCW9vNm+UxJkiRJ0gx2mmGdPYGLJp5fDNxr2jpVdVWSrcAe7fIzF713z/bxcp8JQJKjgKMA9tlnnxk2V5IkSdJatu8xp92g91/44oet0Jb0Z5YWuyyxrGZc57ouv/bCquOramNVbVy/fv12N1SSJEmS1qJZEruLgb0nnu8FXDJtnSQ7AeuA72/nvbN85tWSbEpy/NatW2fYXEmSJElaW2ZJ7M4C9k+yX5KdaYqhnLJonVOAI9rHhwAfqapqlx/WVs3cD9gf+NyMn3m1qjq1qo5at27dddk3SZIkSVoTlh1j146ZOxo4A9gReENVnZPkWGBzVZ0CnAD8U5ItNC11h7XvPSfJycDXgKuAp1bVzwGW+syV3z1JkiRJmn+zFE+hqk4HTl+07PkTjy8HDp3y3uOA42b5zGmSbAI2bdiwYZbVJUmSJGlNmWmC8qHZFVOSJEmSphtFYmfxFEmSJEmabhSJnS12kiRJkjTdKBI7SZIkSdJ0o0js7IopSZIkSdONIrGzK6YkSZIkTTeKxE6SJEmSNN1M89hJkiRJkpa37zGn3aD3X/jih12v942ixc4xdpIkSZI03SgSO8fYSZIkSdJ0o0jsJEmSJEnTmdhJkiRJ0siNIrFzjJ0kSZIkTTeKxM4xdpIkSZI03SgSO0mSJEnSdCZ2kiRJkjRyJnaSJEmSNHImdpIkSZI0cqNI7KyKKUmSJEnTjSKxsyqmJEmSJE03isROkiRJkjSdiZ0kSZIkjVyqauhtmFmSS4FvXM+33wL43gpujrGNbWxjG9vYxja2sY1tbGP3HfvWVbV+8cJRJXY3RJLNVbXR2MY2trGNbWxjG9vYxja2secttl0xJUmSJGnkTOwkSZIkaeTWUmJ3vLGNbWxjG9vYxja2sY1tbGPPY+w1M8ZOkiRJkubVWmqxkyRJkqS5ZGInSZIkSSNnYieNWBq/l+T57fN9ktxz6O3SfGqPt72H3g5JknRtc53YJbltkl3ax/dP8rQkv9Rj/Lv2FWtR3Jsm2aF9fLskj0hyox7i7pjk/3YdZ5ltODrJ7gPEffjCd96z1wL3AQ5vn18GvGaA7RhEkv1mWTaPktyy/d3elOSWfcSsZlD2e/qIpdUnyS8PvQ19SnK/JE9qH6/33NLrNuzRc7xrXSMluUWf2zCEJLdO8sD28a5Jbj7w9vzakPHnwVwndsC7gJ8n2QCcAOwHvK3H+C9P8v+S/EWSO/QY9xPAjZPsCXwYeBLwpq6DVtXPgfVJdu461nbcEjgryclJDkqSnuIeBnw9yUuT/HpPMQHuVVVPBS4HqKofAL18/0kekuTIJPsuWv77fcRvvWuJZe/sKljbYvWYJIe2jw9M8qokf9hnYp/kfwGfAx4FHAKc2eP3fmaSe/QUaxtJPjh5cy7J7knO6Cn2ZUl+tOjfRUneneQ2HcW8xaLnv9ceb0d1fW5L8t8W/dsD+Fz7nf+3LmMvs13P7ynOC4D/D3huu+hGQO83LpN8pOd4vZ9bkrx44VhPsjHJ+cBnk3wjyX/vOPYDklwMXJLkA4v+nn2g49jP3N6/LmO38Z9M8/fyH9pFezH8jbuuv/PdkvxVkn9K8rhFr72249i/u3DubG8UvSXJV5KclGSvFYszz1Uxk3yhqu6W5NnA5VX16iRfrKreWtLau12PAR4L7AacVFUv6jjmwn7/EbBrVb20r/1O8g/A3YBTgB8vLK+ql3cde2IbAjyYJqHdCJwMnFBV/95x3N1oWs6eBBTwRuDtVXVZhzE/C9wXOKv9ma8HPtD1zzrJXwL3A74AbAL+tqpe3b72haq6W8fxfw24A/BS4NkTL+0GPLuqOrmR0p74f5kmef4RsAtwKvA/gP+oqqd3EXeJ7TgPuG9V/Wf7fA/g01V1+x5ifw24HfANmt/x0DTm3bmH2Nc6j/V4bvtz4BKam4OhuZlzS+A84ClVdf8OYl79u5TkecBvtfEfDlxcVX+80jEnYv+C5mc8aS/gYpqfdyfJ7HKSfLOq9ukhztnAXYEvLBxfSb7c5XGe5MuLF9H8rp0H0NPvWO/nliRfqao7tY8/Cjynqs5KcjvgbVW1scPYZwFPrKpzkhwC/BXwP6vqzK7PLe3Ng6mq6s+7it3GPxu4J/DZiWP86p9Fh3FfNe0l4Iiq2q3D2O8Cvg6cCfw+8DPgcVV1RdfXLkm+VlUHtI9ParfhHcADgcdX1YNWIs5OK/Ehq9jPkhwOHEFz8QnNXbfeVNV3gFctnKyA5wOdJnY0uc19gMcDR7bL+vpZX9L+2wEYpEm/qirJd4DvAFcBuwPvTPLBqnpOh3F/1J40dgWeAfwu8Owkr1pIejrwKuDdwK8kOY7mDuvzOoo1aRNw16q6KskLgbcluU17odlHK+ntaS5uf4lrfreh6Yr65A7j/lZV3SlNt53vALeqqiuTvA34YodxF7uYZl8XXAZc1FPsh/YUZym/SLJPVX0Tmm5ENDdR+nBQVd1r4vnxSc6sqmOT/ElHMSd/lx5Fc/z9uD3evtBRzAXPobngeHZVfQUgyQVV1Xl3xCQ/mvYSzfm1D1e2f0uq3aab9hDzQpobRi8Cfkqzv//Ktue4rg1xbrlRkp2q6iqam9FnAVTVv6UdTtOhnavqnDbeO5OcC/xzkmPo+NzSdeI2gyvav18AJNmJfs6nTwL+D3DFEq8dvsSylXTbqnp0+/g9Sf4U+EiSR3QcF2DHiccbquqx7eM3JXnGSgWZ98TuScAfAMdV1QVp+sf31pUiTZe8xwKHAt8DTqQ5mLv2DJruI+9u70LdBvhoD3GvPlGl6addVfVffcRdkORpNIn894B/pLko+VmabnJfp7lY6SLuJpq7P7cF/gm4Z1V9N8lNgHOBThK7qnprks8DB7aLHllV53YRa5GFP8JU1Q/b/T8+yTvooStoVb0XeG+S+1TVZ7qON2Fhn3+W5KyqurJ9flWSn/e4Hd+i6ar0Xpo/xAfTdJN7Zrs9nbWQV9U34OrxVjfuKs4Ufwp8MsnH2+e/DRzVU+xfJHkM13T1PWTita4uhnZNM1Z7B2DHqvoxXH38dXq8VdXLkpwIvCLJRcAL6C+J/iFwj6r6j8UvtNvSh5PbHii/1HZZ+33g9V0GrKpHJPldmomLX1ZVpyT52cLvXJcmuv4teW7pOPxrgNOTvBh4f5K/Bf6Z5u/a2R3H/lmSW7Y34WmvmQ4E3kfz97xzSd7IEr9bVdV19/qPtzeldk3yIOAPaXqgdO0s4KtV9enFL7Q3iru0S5IdquoXAFV1XJquuJ8AbtZx7I8lOZamVfhjSR5ZVe9J8gBg60oFmeuumNAMBgX2qarzBoh9JvB24B1VdckA8W+6cCHQY8w70iQ2C2Mwvgc8YeGOWA/xj6XpdnmtP4RJfr2rpCfJW4B/rKpPLPHagVX14S7itp9/N5pukQV8qqq6vpNPkvcBf11VH1+0/EXAn1RVL+PNkrwZeHpV/bB9vjvwN139QUzyL3avJQcAACAASURBVMChi29YpOlyfUpV9VKRdMguPO2dzb8BfhX4LnBr4Nyuur8uEf8WwL1pWjM+U1Xf6ynubYBX0hQrKppuNH9McyF896r6ZAcxF9+Qe1xVfbvtHndGl13UFm3HJpqket+q6ryYRnseOaWqrpVQJHlJVf1/XW9DG+tBNN36Q/N9f7CnuDcF/gLYANytqlZs/M12Yg7dLfD+wFNoup7uRNNK+B7gjVX1sw7jPhC4tKq+tGj5OuDoqjquq9gTsR498fTGNL19Lqmqp3UcdweaXl1XH+M01zGdJgZpxpldXlU/6TLOlNgvpRmu8qFFyw8CXl1V+3cY+0Y059GF65O9aIYznAocs9AT5QbHmefErv1j9DKapvb9kvwGcGxV9dHkurANO9OcqADO6/IENRHzPjTFYm5WVfskuQvwv6vqD3uI/WngT6vqo+3z+wN/WVX37Tr2ou3YpjVhpX5hVps0hQQOpSkiEuCRNDcSuh7HuStAVf10idf2rKpvtY/v0GVSnwHHXC2KeVPgplX13fZ5p/s9GXeAmzdfAn4H+FBV3bW923h4VXXecpamz9Djgdu0XSD3AW65VAIwz5LsCOyycGHUx/HW/s7ftqq+umj5EVX15i5jb2ebevk9G0r7t/s+VfX3i5b38fPu/dyynCGPtT61CdeHqup3ht6WtarrY629cbBTtWNZF712g36/570q5gtpBob+EKCqzgZ6K1ecpprT12m6GbwW+Lckv91D6L8FHgL8J0B7J6qPuNBc3F59l7mqPgb0MTYBaJL5JF8HLgA+TjNm4V96iHvvJGcl+a8kVyb5eaaPE1lJh9N0WXphVb2ApiXj8V0HraqfLpXUta99a+LpP3W8KTtkYnqL9k5g713Mq+rHC0ldq9P9TnKfNEVMzm2f3yUdV/Sa8LP2j9EObZeWjwK/0VPswab3SPLGJG9Y/K+P2ItV1c8X3e3u+vds4Xf+q0u81EvBoCk62+8kj0ry9SRb01RAvaync/rVqupLi5O6Vpf7PeS5ZTm9HmttD40h7A/0USDogiTnL/7Xddxltmmo73yxTo+1qtq6VFLXukG/3/M+xu6qqtqabatC99lE+XLgwQvdQNNUeHo7cPeuA1fVRYv2u6/xP+cn+TOuOTB/jybJ6suLaJKbbVoTeoj7dzRV8t5BU4nzCTTdaLp2IU3L5OXt812ATqt/XkddF1L5G+Azacb2QdN62XnXmRl0vd8LN29OgeYCsKebRgA/THIzmjEJb03yXdqxhz24VzXVX78IzfQe6W96lfdNPL66u1RPsZfT17Quayn2S4FNXXXfv4G63O8hzy3LWfH9bocyTIvVyw2rJJfRXJum/f87NFNtdG2yK/eNaf5+dj6VyWr4zmcw2vPavCd2X00zT8WOSfYHngZca7Bmh240ObavmgpPfVTlvCjJfYFqL3qeRnv3rQe/D/w5zcBnaC7+nthTbGhbE5Jc3ZqQ5CV9BK6qLUl2rGY+vze23VK7dgVwTpIP0vxBeBBNcYlXtdvUaR/9GXRdVewtSTbTdA0M8Kiq+lqXMWfU+Q2kAW/eHExzI+GPaVqH1wHH9hT7Z203xIVKheuBX/QRuKq2mTMxyduBD01ZvW9DjqmY19j/sUqTOuj+vDrUuWU5Xez3WTS9e5a6mP6lJZatuKoaqoL44hajv03ySZrq7V0a/DufwWjPa/Oe2P0RzUDFK2hays6gGZDcl81JTuCa1qvHA5/vIe4f0Azy35OmdPEHaKod9eGBi5OJJIfStGT1YajWhJ+0SfTZ7eDcb9NPF9R3t/8WfKyHmKtG2yJ7B5oT4TmrJKnrw2A3bxaNu+l7vMvC9B6/nH6n91hKL92lRmDIO9td2pxmrqn3MFGWvar+efpb5sKQN4aX08Wxdi5NDYKvXytYxxVYt9NyBc0x983qdh7cyfg70LTg9ZFkDvadXwejPa/NdWLXjkH40/bfEJ4CPJXmxBiaZKOPvuq3r6ptxlkl+U3gUz3Efi7XTuKWWtaVoVoT/ifNifHoNvbewKO3+46V8Z/A6dWW7l2FruziQ5PsSdMqfDnNzZIAj2lbZ3930Ti/IXSy3xMmb958i+am1VO7DDjRXejqRUx0H6oOJ5VdUNtO7xH6m95jqf3vq7vULLr6PdsBOKSqTt7Oan38XZmmy9+z3YCf0FQMXFBc0xtlSF3u91I3hjs9t8Cgx9oLmV5v4o86iDfpb7bz2k7APkleU1Uv7SH+VTTDZh7TUaxJL2S473zuz2vzXhXzoyw9N8hcVxpK8oWquttyy1Y45kOB/0FzUjhp4qXdgAOqpzLwQ2q7hVFVl/YY8//SFJN4F01Z6F7vrLY3DM6uZtLk3wPuBryyOp53Kcm7gfdW1ZsWLX8C8OiqOrjj+IPs95CSvAe4Jc2F7YlDVJpNcifg19qn504p5jGX2psZt2bihmwtMb1KB3E/UVWDjbEaar+H1vd+J3nUQmtkkt2r6gddxdrONgx6rG3PEBU500zO/kXgabWoPP8KfPYONFP3nLTsygPp8jsf+ljr8vd73hO7ySIlN6ZpQbmqqjqZpHoi7pIJZauq6sApr93QuPcB7kszQfkrJl7ajaYV4y5dxG1j34Vm0OuxbNs/+zLgo13/kRiqNSHNQIQX0LTUheYu1FU086H0Mu4oyW40BWKeRLPPbwTe3mUXjonYXwbuAtyZpsvxCTTj3P57x3HPq6rbX9fXVjD+UPv9ZOBjVfX19tg7gea89g3gidXxHIZpSjQ/iqZQ0I1pbuKcWFXf7yHue2lawr9M87t2J+CbwMFV1Wu1wiS3pfmdO6yq7thDvJcAjwW+xjXjnap6mLqnLYb1U5qf9dXdcLv+mbexh9zvoSaNHmS/J2/+dn0jeDvbMNixtpwBv5ONwPFdxB46uVlOl9/5PJ/X5r0r5uLxbJ9K8vElV15Zz1pi2b2B59BM6NuVnYGb0fxcJ/tJ/4hmLEpnqplS4UtJ3lpVfVXIm/RhhmlNeAbwmzRTDlwAkGYi49cl+eOqesV2370CqupHSd4F7Npuz+8Cz07yqqp6dcfhr6qqSnIwTYvVCUmO6DgmwI5LLWzvQi752gobar+fDrypfXw4TXJ5G+CuNN2nfqvL4FW1laYw0Jtp/jC9mibBe3mXcWnGRm8Gfmeh23FbROWvaKqg9tF951Y0Ce3hNAn9X9FPxV1o5qe8fVVdseyaK28hkZnsjlc0x13XhtzvIaugDrHfmfK4T0Mea8sZ5Dupqs1ZVMlmBX0wybNYhYl0q8vvfG7Pa3Od2KWZ02rBDjTTDNyy67iTCWWauez+jKYM/R9UVWdzdFTVx4GPJ3lT313CkpxcVY8BvpjkWi1nVXXnLuNX1SMnWhNen6Sv1oQnAA+qqu9NbMv5bfe8D7Bty+mKWeg2k2QTzQnqtjQtR/esqu8muQnNAOWuE7vLkjyXZlqL324vtvuo/HpqktcDz1go5pFmkvBXAKf3EH+o/b6qqn7WPn448Ja2stmH2qI9nWqLKhxOk0B+kqYnwL92HRd4IHDnybGkVfXzJH8CfKXLwG0r6eHAXsDJwP+i6Qb8513GXeR8muOr9wSnqnqb+3UJQ+73kFVQh9jvXZPcleZa6cbt46svrLvuDdDGGPJYW85oqyQu1l4jPpHVnUhDh9/5PJ/X5jqxoymqsNAdb2Fg6JF9BE7yEJqE7nLguJqYtLsHP0ny1zTVAm+8sLDjsYULkzk+vMMY2zVQa8KNJpO6iW25NN1ObfE8mtbJQ4FXLO6bXVU/SdJ5lyGa7/lxwJFV9Z0k+wB/3UPc59C0mHwjyTdofs9vTVOl8U96iD/Ufv+ibTn6AU0Bkck5+3btMnCSC4EfAicCR9FWm12orNbxhd+VS/UEqKqrknR98fsa4DPA46pqM8Cim1d9+AlNxd0Ps22Fxs6nM2lvEj0T2KeqjkozddDtq+p9y7x1JQy230voswrqEPv9ba75W/kdtv27WTRTynRq4GNtOaOtkriEO8Pgyc0sOvvO5/m8NteJ3VAHbZKzgPU0F3qfaZdd3U+4hztfb6VprXo4TYWrI4BOC3pU1bfbh98DflpVv0gzIfuvAZ21Uk4aqDVhe9WLuq6MSFU9YTuvfbiH+N8BXp5kt7aF/L/YtgtTV3F/Bjyr7Se/geYPwJZqKuFeLcmDquqDHcQfZL9pxq9upuluekpVnQNX9ww4v+PYF9Jc4D2EplLg5B/dri/8rtWC0ApNb4gu/SrNDZSXJ/kVmla7PlpnJ53S/hvCG2lukt63fX4xTZXjPo73wfY7w00aDQPsd1U9YJb1ujqntgY51rL6qyReuMKfd5Mp51Ogn9bZVfCdz+15bS6LpyR51PZer47noUnyMa5pQl74wzARvtuqnEk+X1V3T/LlhS6QST7edWGHhdg0idXuwJk0F6E/qUXTL3QQ90KuaU34CIvmruvqRJXk50z0TZ98CbhxVXVyAZjkJ8CWKXE77/o6sR3/m6Zgzk+ZOOaralV05ehq8PWQ+51kJ+DmNVGQqO2Gmqr6r/Z5lxdfy23fisdOU5BqqlkvSldgO/bimnF2NwHeXVV9tBCTZk6x27VPz5voktt13M1VtTHJF6vqru2yL1WHxbgWxR9kv4e2Wve744IWgx1rQxcSaW9M78u2VRLf0lGsy2gmCV8qsev8GnViOwb7zuf5vDavLXabtvNa5/PQVNX9Z1mvw4uvhQPk20keRjPge68O4iwlbTfAI2kqQ740yRd7iHshA7QmVFUfhTqWcgHbP8778izgDkt1R10luurKMdh+t10Sf7Bo2eKbCy8BBknsuoi9SloTqKqLgZcBL0tye5okr/PYSe5P0834Qppjeu80pcD7KPt/ZZJdaW9gpKkI2svYryH2OwNPGt1uw/0Z7ue9nC67JA52rDFgIZEk/0QzTv5sJqokAp0kdjS9W1bDtF9DFm+Z2/PaXCZ2VfWkobdhRl1dfL2oLSTyf2jGme1GUy2xD0kz7cLjuWY8Y+fH2VDJdLYt0HMtHZ6grqzVMWfav9P0F1+tuuqSsNr3e8jxIEPG7i2hrarzgMkCKl3G/hvgwW1M2m7ub6cpCNa1FwDvp7n4eCtNFeAn9hAXhtnvoSeNXtiGoX7ey+mym9eQx9qQhUQ20sz3O39d6LZvyO98bs9rc5nYTWpbrBYXEellfrEZdHIBNDH4cyvwAGBhQuU+PAN4Lk0XpXPSlP7vs3DMclb64muyQM9iXZ6ghuzvP+m5wKeTfJbhixv0abXv99xUcLuO5jWhvdHCRQBAVf1bx8WZrlZVH0zyBZopewI8vceW6t73e7nW4bSTRrddEruqkjnYz3tIQx5rAxcS+SpNxfZvL7fiCplprGiSd1XVo7vaiCG/83k+r811Ypfk72nGQTwA+Eeaudw+N+hGbWulS9juCDwG2BN4f1V9NcnDaaoE7koz11Wnqp1yYeL5+cBqudiFFb74GurEVFVHAyT5S+ClVfXD9vnuwP+pquf1tCn/QDOm8SvAL5ZZdwgXdvS5q32/16p5TWg3JzmBZkoTaHpELJ6ntRPtTcGzq+q0NNO4/EmSV/bUY2Cw/Z6mqq5I8gTgeKCrCatX3X5PuLCrDx7yWBuiSmKSU2nOGzcHvpbkc2x7o7CTCemr6gMzrtppy9kQ3/lE7Lk9r81l8ZQFC8VDJv6/GfDPVfXgobcNVn4QcpI3AXvTJK/3Ar4B3Ac4pqres1JxpsT+26p6xsSJahtdnaCuq44Hfu9OUxJ7snW40zERkwN/J5Z1to9LxP90Vd13+TU73YbeBp1PxBx8v7cnyT9X1XaLSM1p7N6O/T5jt61ETwXuR3Nz6hPAa6uHCayTfBm4C02J9LcAbwAeVf0U4xpsv5ez1Ll3BT970P0e4pzaxh3yWDuJ5uL6CVV1x3b81Weq6jc6jLnd/WpvlA+m6/PpEN/5ROy5Pa/NdYsdTcU6aOZ1+1XgP4HVNG/HhSv8eRtpJ/FNM0H394AN1ZRm79rCSf9lPcRadZL8L5q5/PaiGQB9b5qpLroeoLxjkl0WTgjtibHr8u+TPprkKOBUtr3T2Mfg5yEGnS8YdL9h+xdfXSdWQ8ZexoXzGLv9/X453c7JOc1VVVVJDgZeVVUnJDmij8AD7/dyupw8ebD9HvCcCgMea8Btq+qxSQ4HqKqfJum0a/dC4pbkJVW1TffIJC9hovfTnOr9O58wt+e1eU/s3pfkl2jmk/sCzcnp9X1uQM8XQFdW1S/az748yb/1lNQB/EWStwHvqWtX6FtNLuzoc58O3AM4s6oekOTX2LawQlf+L/DhJG+kOb5/n6baUl8e1/7/3IllfQ1+huEGnQ+630NefA184bemEtokJ1fVY5J8haV7QvQxrcllSZ4L/B7w222X/07He62S/e7dKtnvIQt59H6sTRiyIueDuPa4t4cusaxvXSdZQ37nc3tem8uumElOA7ZJMtqmzxtX1dYet2PJC6DqqMBCtp3bLG3sLe3j6vKPQnvX4zDgQJpiKW8HTq+qzifpXmJbhuiad1ZV3SPJ2cC92nEYZ/fUpeChNN97gA9U1Rldx1wtkrwDeFpV9TXofFVIci4DXXwNHLvXc+rQsZPcqqq+neTWS73e09ijW9LcyDirqv41yT7A/bs8p66G/V5OF12OV8N+D3lOHeJYm4j9IOB5wAHAB2irJFbVxzqM+RTgD2luCP77xEs3Bz5dHc//u5wkD74O4/Guz+f3/p1PxJ7b89q8JnarIsno+wJo2sGyoKc/CrsCj6D5/u8DnA68vXqaLHmoC78k7waeRFMV9Hdo5hm7UVX9jy7jDi0DDX7OtoPOf4NmXGnng84n4g826LuNP+TF15Cx12RCO6Qkvw/8a1V9feht6dtQY82GMPQ5td2GQY+1JHtwTZXEM6vjKolppqXaHfgr4JiJly7rslv/tBajBX22iPf9nU/Endvz2lwmdgtWQZIxyAXQtP7ai5f1sB13pukWeOfqaSLv1XDx1Q6IXkdTmbSTmwlJPllV90tyGdueoBdaZ3frIu4S2zHI4OcMPOh8wP0e7OJrlVz4rdWEdvHvOTTT2WymqYJ7foexj6UZ5H9rmmP+X4FPVNWXuoo5EXvI/R6ydbj3/R76nNpuw5DH2kKVxB+nqZJ4N6CvKokLVc1/hW1vInyzo1gLjQAL88dNVmf8SfU0JdiQ3/k8n9fmOrGb1GeSMfQFUJaoZJS2MmiXcds4v0Iz5cJhwK2Ad9Ak02d3HbuN3+vFV5LdqupHmTJReVd33ZLcpsuLmuuwHZuramMmKsQl+VJV3aWn+IPcxBhqv4e8+Bo49lpPaP8cuIRmiEFozq+3BM4DnlJV9+9hG3YFngw8C9izj5t1Q+73wK3DQ+734DeGBzrWhqySeDTwQuA/uGb6nOr6mi3Jp6rqN5db1mH8wb7ziW2Yu/PaXCd2QyUZQ10ALdNf+1NV9XtdxG1jPxk4HLg98M/AiVXV2yTaQ118JXlfVT08yQVcM1H51f9XVSfFNJJ8vqrunuTDVXVgFzFm3I5P03R5/lRV3S3N4Oe3V9U9e4o/yE2MVbDfg118DRF7rSa0E9vw2aq616JlZ1bVvbu+oZDkeTRjX24GfBH4JE0Xps5vng2830O20A6530PeGB7yWPtCey5/PvCtaqok9jJ9SpItNGPz/7PrWIving0cXVWfbJ/fl6bsfue1Adp4Q37nc3tem8uqmEskGc/pM8mo4UrYvg34F3rur926L/Bi4EPVVuZcSpI7VNU5HcQfZJqFqnp4+3/f02jskOQFwO2SPHOJ7eqrTPYLgfcDeyd5K82J8kldB528idHe9Vtwc+DTXcdnoP2eMGQVtd5jD3hOXS0lyX+R5DHAO9vnh0xuYsexHwVcBZxGs69nVtXlHcdc0Pt+Z6BJoxcZYr+HPqfCsMfakBU5L6Lpite3I4E3pBnrB/BDmsrafRnyO5/b89pcttilKf3+dpZJMnrYjl7vfA3VLfC66PpuzIBd834X+Ei1VVfTTLNx/+poYvgktwceSVOs5e8Xv15VfUy1sLAtvQ9+zkCDzhdtwxD7PVgVtSFjT2zDkK0JQ8a+DfBKmrHiBZwJ/DHwLeDuC3fcO4x/c5rxKPej6QXzH1V1vy5jtnF73+9V0kI7xH4Pfk5tt2OoY23Iipwn0DREnMa2NxF6uUGbZDeafKDX5HLI77yNP5fntblM7BYkCc1g0NtU1bHtQXPLqvpcx3EHuQCa0i1wQWfdAq+LTIxJ6ujzh+qad62pDbrc1yRPr6pXJnl+9TTQecp2XKsraN/dQ9PjoPOJmIPs95AXXwPHXtMJ7ZCS3BH4LeC/08xxdhFNl6XnD7phHRuyu/PQhjintnEHO9YyYJXEtvfNtXR9gzbNNGCP5tqVX/sqnjLkdz6357V5T+xeRzMQ9Xeq6teT7E4zz9c9Oo67Ku58rUZdtdgNffG1VPKY5CtVdaeO4p1dVb/RV3/0JeLfGLgJzXQi9+eamwi7Af9SVb/e03b0Ouh8tex3uy2DXHwNEXutJrQT23A74HXAr1RThfXOwCOq6kU9xD4N+ARN1bizqupnXceciD3kfg/ZQjvkfg9SyKONPeSxNliVxIltuDnNd/1fPcV7P00X0M9zTeVXqupveoo/ZGXKuT2vzXtitzAwc5CKfW28IVoTjqyqExZtw/P67J43TYeJ3aAXX0neQNM//TU0raV/BOxeVU/sKN7baZrx17NtItv5ZPRt/KfTdAP9VZruAwsJzo+A11fV33UZf2I7eh10vor2e8iLr8Fit/HXTEI7EffjwLOBf5j4W/bVqrpj17HbWDsDt2ufntfXRdAQ+z30TcJ2Gwb7efd9Tl0i/iDH2kT8Iaok3pFmyoGFYTTfo5lKp4t6BJNxezuHLLMdvX/nbdy5PK/NZfGUCT9r/xAXQJL1XHMh0rlpF0A0pV27dGCSR9MMjL0FTQnZPgb4z6KTed3avuFbgcMXXXzdLMnNerj4+iPgz4CT2ucfAJ7XVbCqOrztn34GzVyNvaqqVwKvTPJHVfXqvuNP6HXQ+Sra72fQTIg+xMXXYLEHPKcOGhu4SVV9rhldcLWreoi7MObsLcCFNDcy9k5yRFV9oofwQ+z3kEXIFgz282a4Qh6DHmu5dpXEZ9G05vTheOCZVfXRdlvuD7yepihdlz6d5E5V9ZWO4yxpyO98ns9r857YvQp4N/DLSY6jqTzT2cX2Ega5AKqqxyV5LPAV4CfA4dVTVdA0k7p+gqav8v9bYtvu3XH8QS6+qurHwDFtEtlLN4qq+g7NHDBD+k6Sm1fVZe1J+m7Ai6rqCz3FPx/4WNutos9B50Pv92AXXwPHXpMJLfC9NFNqLNykPAToqwz/y4EHV9V5bezb0RQnu3sPsXvf71VwkxCG/XkPdU6FYY+1Iask3nQhqQOoqo8luWkPce8HPDFNXYYr6KnHz4Qhv/O5Pa/NdWJXVW9N8nma+aYCPLKqzu1xEwa5AEqyP/B04F3ArwP/s+2O+pMewr+R5mTx6jSVf86m6TP9yh5iw0AXX2nmf/lHmjtP+yS5C/C/q+oPO467P82d5QOAGy8sr/4K5fxZVb0jyf2Ah9BMO/E64F7bf9uK+Wb7b+f2X1+G3u8hL76GjL1WE9qn0tzV/7Uk3wIuoCkM1ocbLVz8AFTVvyXpqyT5YPs9cAvtkD/voc6pMOCx1g7bWaiS+CDg9Ul6qZIInJ/kz2i6Y0JT/v+CHuI+tIcYUw38nc/teW2uE7s0Zf+/S5OFLyy7UY99toe6ADoVeGpVfThNW+8zgbOAO3Qcl6r6SNt/+B7AA4A/aOP2ldgNdfH1CpoL/FMAqupLSX67h7hvBF7Qxn8AzVxq2e47VtbCgOuHAa+rqvcmeWFfwRfGjfY96JyB95thL76GjL0mE9qqOh94YHsXfwfgp8BjgW90HRv4fJpy7AsXnY+nKXTQuYH3e7AW2iH3e8BzKgx4rGVKlcQ+YtPMHffnNPMuh6bXU+fzolbVN9qbk/tX1Rvb4Uo36zrugoG/87k9r8178ZQLgb2BH9D8svwSTXPnd4EnV1WnP8QMV8J2t6r60aJl+1cPJWWTfBi4KfAZml/QT1bVd7uOOxF/kPlgkny2qu6Vngv1JPl8Vd09ExU4k/xrVf1Wl3En4r+PpojIA2m6MPwU+FzX+z0Rf6hB54Pu98R2DHHxNVjsoc6pQ8VOM7/UU4E9gfcCH2qfPwv4UlUd3FXsiW3YpY15P6656HxtVV2x3TfesJirYb8/Cjyoqvoa27Za9nuQc2obu/djbSL2YFUSh9Ke0zbS3MC4XZJfBd5RVb/ZU/whK1PO7Xlt3hO7vwfeXVVntM8fDBwEnAy8sqp66TbV1wVQkudU1Uvbx4dW1TsmXvvLqvqTLuO3cV5Bc6F7BfApml+Wz1TVT7uO3cYfKpl+J02f7b+jmbT6acDGqjqs47ifornj9U7gIzTJxour6vZdxp2IfxOa36mvVNXXk9wKuFNVfaCn+J8G/nTRoPO/rKpOB52vgv0e8uJrsNgT27AmEtok76W5MfkZmiEFu9O0kj69qs7uIf4OwJer58p5Q+93uw293yRcJfs91Dl1kGNt0Tb0WiUxySnbe72qOi2MluRs4K7AFyZuSPcypcfENvRemXLez2vznthtrqqNSy3LEhNKdxC/1wugTEwlkEXTCix+3rUkN6PpSvAsmknhd+krdhu/7/lgbkHT3fSBNE3rZ9D8snbajSfJPYBzaVqj/4JmPrW/rqozu4w7Ef9lwBv7vKhfFP9araI9tZQOvd+DXHytgthrKqFd1BK/Yxtzn6q6rKuYS2zDW4HnVk9TSrQxV8N+D9FCuxr2e5Bzahun92NtIvb/3969h9tR1Wcc/74QJBEFRRCjBIEqoFIgoEJAq4hYkYsXEEFQFOu9UqgVW7AVsbW2WLyAdyGgT4lgwVLtg0C9UEBC0AARIXip0cYZEgAAE9xJREFUYEXqjYsxioXk7R8zJ9kcTrg0M7P2mXk/z3OenJmdk/VbOzuT+c1a67fuUyUROMItVkmU9Auq6YcLgCuZtIzCdqvVzCUtsv1Mrd4abAOqB/FdbV3T+Xs+0nZvr2u9XmMH3CbpncDn6+NXALfXb2gX2x50XcJWa/h+quN2AqgWnD+batTuZqqtFrqaM32fmy9Jndz42f4l3S1uB1ZdGA62/Q7gN3QwJ38KS4FPSZpBtd5vgauqcl0ptei8dL9LVVEr3XapsuCl2l719Nr2Ckk/6vImvzYb+K6kRcDykXjaHE0o3m+XWWtWvN+Uu6ZCmc/ahBJVEh9HVTTkUOCVVKPDCzp8YHiOpE8Cj5L0eqq1fp/pqG0oW5myt9e1vo/YbUJVWGJiDu1lVAtU76TKkn/QcvudPvkahxE7Se+gmn75bXe4NmGk/VLTSLamGrHbjapy2hXAMa4WybbR3gzb90j6GrCXC/9DlrQtVWJ5KNUU3E+P3vy32O6jqf5Nj86TP8H27W23Xbdfqt9fBBZz75uvp9t+Sc/bLjma0Hnbklaw+qZDwCyqLWwmypJv2FbbIzE8Z6rzbY4mjEm/S4zQjkO/i11TS3zWRtq+zxTELqclqlrzdShwEnCiO9onVdLewAuo/q4vpKpi3vqaxrrtYu95n69rvU7sSuv6BmjkQzP6gaE+nmm7k1KumqLKku1OnvgVnJq3EPgoqyuwHgK8zS2t4xyZOvFPwJOBL3Dvp07ntdHuGmJZF9iPKsGZQ7WG9VnAcre8xrCkkv0ufPNVsu1BJrTjon5Y+qvSD5K6UHLKcXT/WZM0n2om12iVxBm2W50JUyd0+1IldVtSVdY+3fYtbbZbt3267SNHjh8BnG97r7bbrtsr8p5PEUevrmu9TuzqpOJYqnL7o3t8Pa+j9ouOJpSg8lWWitx8qa6KOencQre0IftIYjd/5LRZ/eTnyDX8aNNxnAzsT1W45TTbi0Zeu9EtFXFR+UXnRfo9dENNaEuQtBvwfuA2qvW7nwM2oVpD/GrbXykYXutKjg6XUPKaOg6fNZWpkngmsD1wAfB529e11dYa2n8vsIntN9fXt3+nmnUy/wF+tKn2S7znxT9rbet7YncRcDZVAY83AUcAv7D9zqKB9ZgKV1kqdfMl6f3AHVTrOU21nnN9qlE8bN/WcHs/oZqfLlYndBPsbvb1QtKRVP8h/XaK1zZyS+vOVH7Real+l7z5KppMR7ckfQs4DtiIan3hPrYXStqOah3Q3KIBtmxoI7Qlr6mlP2sqVyVxJatn2ozejHc59fYfqN73Xagqap/bdpt1u6Xe895f1/qe2E3s8bUqsZB0ie0p59Y22O5gb4BUuMpSKZImpppO/IOanGht3XB7twIfn9TOaHsnNtneFO0/EbhjIoGRtCfwEqqCOafa/t+W21+X1YvOd6CjRedj0O+SN18l205C2zGNVI6WdIPtp4y8dnUfboDuzwBHaItcU+u2i3/WVLAiZ9ckvWz0EPhrYBHwFehuKUeJ93wcPmtt63tVzIkKNLdK2hf4KbB5B+3O435ugHpuqipLn2670VI3X6q2G/hv21vVx0cAB1KV7z2h6ZG6Ebe2nbw9gHOAlwJ3StqJao3f3wM7Ah8D/qTNxm2voPpP6Csji86/IantRedF+03ZKmol2y55TR3q9Xy0cvTkfUj7+0S4VidwR5WOoysFr6kwHp+1khU5u7b/pOOrgfXq8wa6WqNf4j0fh89aq/o+YrcfVan9OcApVHt8vcf2/SYBDbRb7MnXONCkKku2L+6gzSKjCZIWA8+3fZukP6Kaivk2YCfgKbYPaqndok+WJo2CfwBYafvYenrFNV2M0JZYdD4O/R6JpUgVtRJtFx5NGOT1XGNSjKtrQx2hhTLX1Lrd4p81FazIOVQl3vNx+Ky1rZeJnaSZVGvqngR8h6q4Qeel9+tYit18DUnBqXmrFtNL+ijVGs4T6uNVQ/4ttLtxi6OBD6b90Y02F1NNp7iwPm59TWWpReel+123U7KKWrG2J8UwiIQ2uld6/W4ppQt5jBP1rErimqjaN+7jwGa2t5e0A3CA7b8tEMsg3vMu9DWxO5tqGualwD7Azbb/rOMYit8AdaleY7amD5Nt/0GHsXR28yXpOmAnV3vKLQXeYPs/J17remFwVyR9mGoaxa3AAcA2tu+WNBv4ku2nt9x+kUXnY9DvYjdfpW/8hp7QRjcGPEJbvJBHCRpAlcQ1kXQJ8A7gk15d7K71+5Yhv+dd6GtiN/pUfQawyB1szj3S/uCefEl6zKRT6wAHU1UkXWz7wA5iKDE173jgRVSb124B7Gzbkp4EnOmOtnnomiRRVf6cDZwz8R5Lmgs8dmIUq29K97vkzVfhtgeb0EY5GaHtPw2gSuKaSLrK9jNGl3a0OdNopN3Bvudd6Gtit3g0kZt83EH7g3zyBUyUsH0V1VOga6g2dL2+g3ZL3vjtRnWjf5Ht5fW5bag2Zl/cVRwRfTXUhDbKyAjtcAyhSuKaSLoA+FOqvYZ3lnQQ8Drb+7Tc7mDf8y70tSrmjpJ+XX8vYFZ93Ml/xLbXafPPH0eS1qOqgHkMcBnwYts/7DCEV1HdfG0DHFUNrFSh0fLfue2FU5z7XlvtjRNJy7jvFNw7gW8Bb7f9X91H1b6h9ruUktfUIV7Ph2zSQ8L3ZIS293pfJfF+vJVqxGw7SbcAP6Lar7FtQ37PW9fLEbvonqoNs+8BPgTcZ08Sd7QvSnRL0nuothE5iyqJPoSqLP6NwJttP7dcdO0Zar8j+i4jtMMyhCqJD0TVfsPr2F7WUXuDf8/blMQuGiHpDO6/eMqRHYYTHZF0pe1dJ51baHu30YqhfTPUfkdERD9I2gx4H/B42/tIeiowz/ZphUOLtdDXqZjRMduveTC/T9IRts9sOZzozkpJBwP/Uh+P7tvX56dGQ+13RET0wxnAfOD4+vh7wNlAErtpLGsHomudbjsRrTuMan3jz4Gf1d8fLmkW1aLsvhpqvyMioh82sX0O9Zo3V/s9rygbUqytjNhF1/TAvyWmi7pIyP5rePmyLmPp0lD7HRERvbG83qrKsKrC951lQ4q1lcQuupZpaj0iaVPg9VQlwVddT/q+pnKo/Y6IiOlN0tHA5cCxwPnA1pIuBzYFXl4ytlh7Seyiaxmx65fzgUuB/2BYUziG2u+IiJjeNgc+DGwHLAUuBr4BnG37lwXjigakKmZ0StKptrMGqSdGNxodkqH2OyIi+kHSw4CnA7sD8+qvO2w/tWhgsVZSPCUaJWkzSadJuqA+fqqk1028nqSud74s6UWlgyhgqP2OiIh+mAVsCGxUf/0UuLJoRLHWMmIXjaoTuvnA8bZ3lDQDuNr2HxYOLVogaRmwAfB74G4GsonvUPsdERHTm6RPAU8DllElcguBhbZvLxpYNCIjdtG0lM8dENuPtL2O7Vm2N6yPe5/cDLXfEREx7W0BrA/8D3AL8BPgjqIRRWNSPCWalvK5AyBpO9tLJe081eu2F3cdUxeG2u+IiOgH2y+UJKpRu92BtwPbS7oNuML2u4sGGGslUzGjUfUN7ynA9sB1VOVzD7K9pGhg0ShJn7L9Bklfn+Jl235e50F1YKj9joiI/pG0ObAHVYK3H/AY248qG1WsjSR20bh6Xd22VOuObrR9d+GQoiWSZtq+64HO9c1Q+x0REdObpKOoErk9qNaIXw5cUf/6HdsrC4YXaymJXTRK0kzgLcCzqKZjXgp8Ije8/SRpse2dH+hc3wy13xERMb1JOhn4JnC57VtLxxPNyhq7aNpnqSotnVIfHwp8Dnh5sYiicZIeBzwBmCVpLqs3nt8QeHixwFo21H5HREQ/2P7z0jFEe5LYRdO2tb3jyPHXJV1bLJpoyx8DrwE2B04eOb8MOK5EQB0Zar8jIiJizGUqZjRK0hlUUy8X1se7AkfYfkvRwKIVkg60fW7pOLo21H5HRETE+EpiF42SdANV4ZQf16e2AG6g2tfOtncoFVu0Q9K+VGWTZ06cs31iuYi6MdR+R0RExHjKVMxo2gtLBxDdkfQJqrVlewKfAQ4CFhUNqgND7XdERESMr4zYRaMkfQA43fb1pWOJ9klaYnuHkV8fAZxn+wWlY2vTUPsdERER42ud0gFE7ywFPi3pSklvkrRR6YCiVRPbWPxW0uOp9sTZqmA8XRlqvyMiImJMJbGLRtn+jO09gFcDWwJLJJ0lac+ykUVLviTpUcBJwGLgJmBB0Yi6MdR+R0RExJjKVMxonKR1gf2A1wJzgHOoNixfbvuQkrFFcyStA+xm+5v18frATNt3lo2sXUPtd0RERIy3JHbRCEnvs32cpJOBA4CvAqfZXjTye260vW2xIKNxkq6wPa90HF0bar8jIiJifGUqZjRlohrmdcAOtt84mtTVntlxTNG+iyQdKEmlA+nYUPsdERERYyojdtEISdcCzwWmvNG1fVunAUUnJC0DNgBWAL+j+vu37Q2LBtayofY7IiIixlcSu2iEpN8Dt0wcTnrZtrfuOKSIiIiIiMHIBuXRlOttzy0dRHSrnop4GLCV7fdKmgPMnmIabq8Mtd8RERExvrLGLiLWxseAecAr6+PfAB8tF05nhtrviIiIGFNJ7KIpH5a0rqSTSgcSndrV9lupN+y2fTvwsLIhdWKo/Y6IiIgxlcQuGmH7DNsrgF1SKXBQ7q73LTSApE2BlWVD6sRQ+x0RERFjKmvsomlXA+dL+gKwfOKk7fPKhRQt+gjwRWAzSX8HHAS8q2xInRhqvyMiImJMpSpmNErS/ClO2/aRnQcTnZC0HbBXffg12zeUjKcrQ+13REREjKeM2EWjbL+2dAzRuYcDE9MSZxWOpUtD7XdERESMoayxi0ZJ2lzSFyX9XNLPJJ0rafPScUU7JP0NcCawMbAJMF9S76ckDrXfERERMb4yFTMaJeli4Czgc/Wpw4HDbO9dLqpoi6QbgLm276qPZwGLbT+lbGTtGmq/IyIiYnxlxC6atqnt+bbvqb/OADYtHVS05iZg5sjx+sAPy4TSqZsYZr8jIiJiTGWNXTTtl5IOBxbUx4cCvyoYT7Tr98B365FagOcDl0n6CIDto4pF1q6h9jsiIiLGVKZiRqMkbQGcCsyjKirxTeAo2z8uGli0QtKbqR4QrQRWAL8bfd32mSXiattQ+x0RERHjKyN20bQ5tg8YPSFpDyCJXY9ImgG8DzgSuJlqWvccYD5wnO27C4bXmqH2OyIiIsZf1thF0055kOdiejuJqiLkVrZ3sT0X2BrYqH6tr4ba74iIiBhzmYoZjZA0D9gdOBr44MhLGwIvtb1jkcCiFZK+D2zjSRcQSesCS20/uUxk7RpqvyMiImL8ZSpmNOVhwCOoPlOPHDn/a+CgIhFFmzw5ualPrpDU56dFQ+13REREjLkkdtEI25cAl0g6w/bNpeOJ1l0v6dW2Pzt6sq6IurRQTF0Yar8jIiJizGUqZjRC0odsHy3pS1TVMO9lckGVmN4kPQE4j6oa5Lep/s6fAcyimnp7S8HwWjPUfkdERMT4S2IXjZC0i+1vS3rOVK/XI3rRM5KeBzwNEPBd218tHFInhtrviIiIGF9J7KI1kh5Ntf3BktKxRERERET0WbY7iEZJ+oakDSVtDFwLzJd0cum4IiIiIiL6LIldNG0j278GXgbMt70L8PzCMUVERERE9FoSu2jaDEmzgYOBL5cOJiIiIiJiCJLYRdNOBC4EfmD7KklbA98vHFNERERERK+leEpERERERMQ0lw3KoxGSjrX9j5JOYep97I4qEFZERERExCAksYumvFHS5cC3SgcSERERETE0mYoZjZB0NPAKYDZwNrDA9jVlo4qIiIiIGIYkdtEoSU8EDqm/ZgILqJK8FFCJiIiIiGhJErtojaS5wOnADrbXLR1PRERERERfZbuDaJSk9STtL+mfgQuA7wEHFg4rIiIiIqLXMmIXjZC0N3AosC+wCPg88K+2lxcNLCIiIiJiAJLYRSMkfR04CzjX9m2l44mIiIiIGJIkdhEREREREdNc1thFRERERERMc0nsIiIiIiIiprkkdhERMQiSjpJ0Q12196H83JaSXtlWXBEREU1IYhcREUPxFuBFtg97iD+3JfCQEztJ2b8zIiI6k8QuIiJ6T9IngK2Bf5N0vKTTJV0l6WpJL65/z5aSLpW0uP7avf7x9wPPlnSNpGMkvUbSqSN/9pclPbf+/jeSTpR0JTBP0uGSFtU/+8kkexER0ZYkdhER0Xu23wT8FNgT2AD4mu1n1McnSdoA+Dmwt+2dgVcAH6l//C+BS23vZPuDD9DUBsB1tncFflX/OXvY3glYATzU0cKIiIgHZUbpACIiIjr2AuAASX9RH88EtqBK/E6VNJGEbfP/+LNXAOfW3+8F7AJcJQlgFlXyGBER0bgkdhERMTQCDrR9471OSicAPwN2pJrRctcafv4e7j3jZebI93fZXjHSzpm2/6qJoCMiIu5PpmJGRMTQXAi8TfUwmqS59fmNgFttrwReBUysh1sGPHLk528CdpK0jqQ5wDPX0M5XgYMkPbZuZ2NJT2y0JxEREbUkdhERMTTvBdYDlki6rj4G+BhwhKSFVNMwl9fnlwD3SLpW0jHA5cCPgO8AHwAWT9WI7euBdwEXSVoCXAzMbqdLERExdLJdOoaIiIiIiIhYCxmxi4iIiIiImOaS2EVERERERExzSewiIiIiIiKmuSR2ERERERER01wSu4iIiIiIiGkuiV1ERERERMQ0l8QuIiIiIiJimktiFxERERERMc39H3I7SfMuFXH9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# and now let's plot the distribution of importances\n",
    "# for those features that are actually important\n",
    "\n",
    "# capture the important features in a series for visualisation\n",
    "temp = pd.Series(feature_importance[feature_importance.auc_drop>0]['auc_drop'])\n",
    "temp.index = pd.Series(feature_importance[feature_importance.auc_drop>0]['feature'])\n",
    "\n",
    "pd.Series(temp).plot.bar(figsize=(15,6))\n",
    "plt.yscale(\"log\")\n",
    "from matplotlib.ticker import ScalarFormatter\n",
    "plt.gca().yaxis.set_major_formatter(ScalarFormatter())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train auc score:  0.9164767212630782\n",
      "test auc score:  0.9128686744975785\n"
     ]
    }
   ],
   "source": [
    "# Now let's build a random forests only with the 26 selected features\n",
    "\n",
    "# capture the 27 selected features\n",
    "selected_features = feature_importance[feature_importance.auc_drop>0]['feature']\n",
    "\n",
    "# train a new random forests using only the selected features\n",
    "rf = RandomForestClassifier(\n",
    "    n_estimators=100, max_depth=2, random_state=2909, n_jobs=4)\n",
    "\n",
    "rf.fit(x_baseTrain[selected_features], y_baseTrain)\n",
    "\n",
    "# print roc-auc in train and testing sets\n",
    "print('train auc score: ',\n",
    "      roc_auc_score(y_baseTrain, (rf.predict_proba(x_baseTrain[selected_features])[:, 1])))\n",
    "print('test auc score: ',\n",
    "      roc_auc_score(y_baseTest, (rf.predict_proba(x_baseTest[selected_features])[:, 1])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, the random forests with the selected features show a better(from 89 to 91) performance to the random forests built using all of the features. And yet, provides a simpler, faster and more reliable model, for implementation in a live scenario."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}