(3)manual_feature_scaling_selection_encoding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# (3) Manual Feature Scaling, Selection, and Encoding\n",
    "\n",
    "**Feature scaling** and **feature_selection** are yet two components of a classical data science pipeline that isn't fully adressed in neither `featuretools` and `h2o`. `featuretools` has no mention of these, and `h2o` includes them in some models as hyperparameters to learn and doesn't include them in others. Whereas **data encoding** for categorical variables are handled intrinsically in `h2o` as we shall discuss in the next notebook,  it does so by applying different encodings for different algorithms. For example, XGBoost models perform an internal *one-hot encoding* and Gradient Boosting Machine (GBM) models perform *enum encoding*. For this reason, we will both demonstrate the case where we handle encoding ourselves and the case where we completely leave it to `h2o`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas.api.types import is_numeric_dtype\n",
    "from sklearn.feature_selection import SelectKBest, chi2, SelectFromModel, RFE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder, MinMaxScaler\n",
    "from utils import categorical_to_onehot_columns\n",
    "\n",
    "SEED = 42\n",
    "pd.options.mode.chained_assignment = None  # suppress SettingWithCopyWarning() for chained assignments\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "sns.set_style(\"white\", {'ytick.major.size': 8.0})\n",
    "sns.set_context(\"poster\", font_scale=0.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "      <th>...</th>\n",
       "      <th>buckets.CUM_SUM(CA)</th>\n",
       "      <th>buckets.CUM_SUM(DE)</th>\n",
       "      <th>buckets.CUM_SUM(FR)</th>\n",
       "      <th>buckets.CUM_SUM(GB)</th>\n",
       "      <th>buckets.CUM_SUM(AU)</th>\n",
       "      <th>buckets.CUM_SUM(NL)</th>\n",
       "      <th>buckets.CUM_SUM(US)</th>\n",
       "      <th>buckets.CUM_SUM(IT)</th>\n",
       "      <th>buckets.CUM_SUM(PT)</th>\n",
       "      <th>buckets.CUM_SUM(ES)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>31.000000</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>omg</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>12114.0</td>\n",
       "      <td>23369.0</td>\n",
       "      <td>21961.0</td>\n",
       "      <td>21458.0</td>\n",
       "      <td>8783.0</td>\n",
       "      <td>5457.0</td>\n",
       "      <td>119834.0</td>\n",
       "      <td>16818.0</td>\n",
       "      <td>3092.0</td>\n",
       "      <td>13463.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>43.857143</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>iPad</td>\n",
       "      <td>Mobile Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>16926.0</td>\n",
       "      <td>33405.0</td>\n",
       "      <td>29925.0</td>\n",
       "      <td>29738.0</td>\n",
       "      <td>12100.0</td>\n",
       "      <td>7523.0</td>\n",
       "      <td>161807.0</td>\n",
       "      <td>25250.0</td>\n",
       "      <td>4714.0</td>\n",
       "      <td>21218.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>45.857143</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>Firefox</td>\n",
       "      <td>...</td>\n",
       "      <td>19322.0</td>\n",
       "      <td>39395.0</td>\n",
       "      <td>34357.0</td>\n",
       "      <td>34188.0</td>\n",
       "      <td>13713.0</td>\n",
       "      <td>8742.0</td>\n",
       "      <td>182625.0</td>\n",
       "      <td>30158.0</td>\n",
       "      <td>5525.0</td>\n",
       "      <td>25122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MALE</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>18105.0</td>\n",
       "      <td>36038.0</td>\n",
       "      <td>32137.0</td>\n",
       "      <td>31839.0</td>\n",
       "      <td>12920.0</td>\n",
       "      <td>8105.0</td>\n",
       "      <td>171966.0</td>\n",
       "      <td>27678.0</td>\n",
       "      <td>5130.0</td>\n",
       "      <td>23270.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>33.714286</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Moweb</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>...</td>\n",
       "      <td>12114.0</td>\n",
       "      <td>23369.0</td>\n",
       "      <td>21961.0</td>\n",
       "      <td>21458.0</td>\n",
       "      <td>8783.0</td>\n",
       "      <td>5457.0</td>\n",
       "      <td>119834.0</td>\n",
       "      <td>16818.0</td>\n",
       "      <td>3092.0</td>\n",
       "      <td>13463.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender        age signup_method language affiliate_channel  \\\n",
       "0  FEMALE  31.000000         basic       en            direct   \n",
       "1  FEMALE  43.857143         basic       en            direct   \n",
       "2  FEMALE  45.857143         basic       en            direct   \n",
       "3    MALE  40.000000         basic       en            direct   \n",
       "4  FEMALE  33.714286         basic       en            direct   \n",
       "\n",
       "  affiliate_provider first_affiliate_tracked signup_app first_device_type  \\\n",
       "0             direct                     omg        Web       Mac Desktop   \n",
       "1             direct               untracked        Web              iPad   \n",
       "2             direct               untracked        Web   Windows Desktop   \n",
       "3             direct               untracked        Web       Mac Desktop   \n",
       "4             direct               untracked      Moweb   Windows Desktop   \n",
       "\n",
       "   first_browser         ...          buckets.CUM_SUM(CA)  \\\n",
       "0         Safari         ...                      12114.0   \n",
       "1  Mobile Safari         ...                      16926.0   \n",
       "2        Firefox         ...                      19322.0   \n",
       "3         Safari         ...                      18105.0   \n",
       "4         Chrome         ...                      12114.0   \n",
       "\n",
       "   buckets.CUM_SUM(DE) buckets.CUM_SUM(FR) buckets.CUM_SUM(GB)  \\\n",
       "0              23369.0             21961.0             21458.0   \n",
       "1              33405.0             29925.0             29738.0   \n",
       "2              39395.0             34357.0             34188.0   \n",
       "3              36038.0             32137.0             31839.0   \n",
       "4              23369.0             21961.0             21458.0   \n",
       "\n",
       "  buckets.CUM_SUM(AU) buckets.CUM_SUM(NL)  buckets.CUM_SUM(US)  \\\n",
       "0              8783.0              5457.0             119834.0   \n",
       "1             12100.0              7523.0             161807.0   \n",
       "2             13713.0              8742.0             182625.0   \n",
       "3             12920.0              8105.0             171966.0   \n",
       "4              8783.0              5457.0             119834.0   \n",
       "\n",
       "   buckets.CUM_SUM(IT)  buckets.CUM_SUM(PT)  buckets.CUM_SUM(ES)  \n",
       "0              16818.0               3092.0              13463.0  \n",
       "1              25250.0               4714.0              21218.0  \n",
       "2              30158.0               5525.0              25122.0  \n",
       "3              27678.0               5130.0              23270.0  \n",
       "4              16818.0               3092.0              13463.0  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = pd.read_csv('(2)data_automated_ops/train_users.csv')\n",
    "X_train, Y_train = X_train.drop('country_destination', axis=1), X_train['country_destination']\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "      <th>...</th>\n",
       "      <th>buckets.CUM_SUM(CA)</th>\n",
       "      <th>buckets.CUM_SUM(DE)</th>\n",
       "      <th>buckets.CUM_SUM(FR)</th>\n",
       "      <th>buckets.CUM_SUM(GB)</th>\n",
       "      <th>buckets.CUM_SUM(AU)</th>\n",
       "      <th>buckets.CUM_SUM(NL)</th>\n",
       "      <th>buckets.CUM_SUM(US)</th>\n",
       "      <th>buckets.CUM_SUM(IT)</th>\n",
       "      <th>buckets.CUM_SUM(PT)</th>\n",
       "      <th>buckets.CUM_SUM(ES)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MALE</td>\n",
       "      <td>34.428571</td>\n",
       "      <td>basic</td>\n",
       "      <td>zh</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>linked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>...</td>\n",
       "      <td>13376.0</td>\n",
       "      <td>25996.0</td>\n",
       "      <td>23996.0</td>\n",
       "      <td>23648.0</td>\n",
       "      <td>9664.0</td>\n",
       "      <td>5960.0</td>\n",
       "      <td>130818.0</td>\n",
       "      <td>18632.0</td>\n",
       "      <td>3463.0</td>\n",
       "      <td>15211.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>facebook</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>facebook</td>\n",
       "      <td>linked</td>\n",
       "      <td>Web</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>Mobile Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>7178.0</td>\n",
       "      <td>13444.0</td>\n",
       "      <td>13753.0</td>\n",
       "      <td>12950.0</td>\n",
       "      <td>5352.0</td>\n",
       "      <td>3410.0</td>\n",
       "      <td>74974.0</td>\n",
       "      <td>10143.0</td>\n",
       "      <td>1837.0</td>\n",
       "      <td>8015.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MALE</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>google</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>6019.0</td>\n",
       "      <td>11283.0</td>\n",
       "      <td>11806.0</td>\n",
       "      <td>10993.0</td>\n",
       "      <td>4565.0</td>\n",
       "      <td>2906.0</td>\n",
       "      <td>63880.0</td>\n",
       "      <td>8629.0</td>\n",
       "      <td>1562.0</td>\n",
       "      <td>6909.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>46.714286</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>iOS</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>Mobile Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>19322.0</td>\n",
       "      <td>39395.0</td>\n",
       "      <td>34357.0</td>\n",
       "      <td>34188.0</td>\n",
       "      <td>13713.0</td>\n",
       "      <td>8742.0</td>\n",
       "      <td>182625.0</td>\n",
       "      <td>30158.0</td>\n",
       "      <td>5525.0</td>\n",
       "      <td>25122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MALE</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>facebook</td>\n",
       "      <td>en</td>\n",
       "      <td>seo</td>\n",
       "      <td>facebook</td>\n",
       "      <td>linked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Android Phone</td>\n",
       "      <td>Silk</td>\n",
       "      <td>...</td>\n",
       "      <td>18105.0</td>\n",
       "      <td>36038.0</td>\n",
       "      <td>32137.0</td>\n",
       "      <td>31839.0</td>\n",
       "      <td>12920.0</td>\n",
       "      <td>8105.0</td>\n",
       "      <td>171966.0</td>\n",
       "      <td>27678.0</td>\n",
       "      <td>5130.0</td>\n",
       "      <td>23270.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender        age signup_method language affiliate_channel  \\\n",
       "0    MALE  34.428571         basic       zh               seo   \n",
       "1  FEMALE  23.000000      facebook       en               seo   \n",
       "2    MALE  19.000000         basic       en               seo   \n",
       "3  FEMALE  46.714286         basic       en            direct   \n",
       "4    MALE  41.000000      facebook       en               seo   \n",
       "\n",
       "  affiliate_provider first_affiliate_tracked signup_app first_device_type  \\\n",
       "0             google                  linked        Web       Mac Desktop   \n",
       "1           facebook                  linked        Web            iPhone   \n",
       "2             google               untracked        Web       Mac Desktop   \n",
       "3             direct               untracked        iOS            iPhone   \n",
       "4           facebook                  linked        Web     Android Phone   \n",
       "\n",
       "   first_browser         ...          buckets.CUM_SUM(CA)  \\\n",
       "0         Chrome         ...                      13376.0   \n",
       "1  Mobile Safari         ...                       7178.0   \n",
       "2         Safari         ...                       6019.0   \n",
       "3  Mobile Safari         ...                      19322.0   \n",
       "4           Silk         ...                      18105.0   \n",
       "\n",
       "   buckets.CUM_SUM(DE) buckets.CUM_SUM(FR) buckets.CUM_SUM(GB)  \\\n",
       "0              25996.0             23996.0             23648.0   \n",
       "1              13444.0             13753.0             12950.0   \n",
       "2              11283.0             11806.0             10993.0   \n",
       "3              39395.0             34357.0             34188.0   \n",
       "4              36038.0             32137.0             31839.0   \n",
       "\n",
       "  buckets.CUM_SUM(AU) buckets.CUM_SUM(NL)  buckets.CUM_SUM(US)  \\\n",
       "0              9664.0              5960.0             130818.0   \n",
       "1              5352.0              3410.0              74974.0   \n",
       "2              4565.0              2906.0              63880.0   \n",
       "3             13713.0              8742.0             182625.0   \n",
       "4             12920.0              8105.0             171966.0   \n",
       "\n",
       "   buckets.CUM_SUM(IT)  buckets.CUM_SUM(PT)  buckets.CUM_SUM(ES)  \n",
       "0              18632.0               3463.0              15211.0  \n",
       "1              10143.0               1837.0               8015.0  \n",
       "2               8629.0               1562.0               6909.0  \n",
       "3              30158.0               5525.0              25122.0  \n",
       "4              27678.0               5130.0              23270.0  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = pd.read_csv('(2)data_automated_ops/test_users.csv')\n",
    "X_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Scaling\n",
    "**Feature scaling** is notoriously effective in linear models and neural networks. As it is a rather conventional aspect of the data processing pipeline, we decided to include it here no matter what. An importing thing to note is that, we have to apply scaling (sometimes also referred to as **normalization**) only to *originally numerical* columns at first. Later in the notebook when we encode categorical variables, we will apply another scaling there as well. Let's first observe the different numeric and categorical variables we have."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUMERIC VARIABLES:  ['age', 'signup_flow', 'LAST(sessions.secs_elapsed)', 'NUM_UNIQUE(sessions.action)', 'NUM_UNIQUE(sessions.action_type)', 'NUM_UNIQUE(sessions.action_detail)', 'NUM_UNIQUE(sessions.device_type)', 'SKEW(sessions.secs_elapsed)', 'MIN(sessions.secs_elapsed)', 'MEAN(sessions.secs_elapsed)', 'STD(sessions.secs_elapsed)', 'MAX(sessions.secs_elapsed)', 'MEDIAN(sessions.secs_elapsed)', 'HOUR(date_account_created)', 'HOUR(timestamp_first_active)', 'DAY(date_account_created)', 'DAY(timestamp_first_active)', 'WEEK(date_account_created)', 'WEEK(timestamp_first_active)', 'MONTH(date_account_created)', 'MONTH(timestamp_first_active)', 'YEAR(date_account_created)', 'YEAR(timestamp_first_active)', 'CUM_SUM(age)', 'buckets.CA', 'buckets.DE', 'buckets.FR', 'buckets.GB', 'buckets.AU', 'buckets.NL', 'buckets.US', 'buckets.IT', 'buckets.PT', 'buckets.ES', 'LAST(sessions.CUM_SUM(secs_elapsed))', 'SKEW(sessions.CUM_SUM(secs_elapsed))', 'MIN(sessions.CUM_SUM(secs_elapsed))', 'MEAN(sessions.CUM_SUM(secs_elapsed))', 'STD(sessions.CUM_SUM(secs_elapsed))', 'MAX(sessions.CUM_SUM(secs_elapsed))', 'MEDIAN(sessions.CUM_SUM(secs_elapsed))', 'CUM_SUM(LAST(sessions.secs_elapsed))', 'CUM_SUM(NUM_UNIQUE(sessions.action))', 'CUM_SUM(NUM_UNIQUE(sessions.action_type))', 'CUM_SUM(NUM_UNIQUE(sessions.action_detail))', 'CUM_SUM(NUM_UNIQUE(sessions.device_type))', 'CUM_SUM(SKEW(sessions.secs_elapsed))', 'CUM_SUM(MIN(sessions.secs_elapsed))', 'CUM_SUM(MEAN(sessions.secs_elapsed))', 'CUM_SUM(STD(sessions.secs_elapsed))', 'CUM_SUM(MAX(sessions.secs_elapsed))', 'CUM_SUM(MEDIAN(sessions.secs_elapsed))', 'buckets.LAST(users.age)', 'buckets.LAST(users.signup_flow)', 'buckets.NUM_UNIQUE(users.gender)', 'buckets.NUM_UNIQUE(users.signup_method)', 'buckets.NUM_UNIQUE(users.language)', 'buckets.NUM_UNIQUE(users.affiliate_channel)', 'buckets.NUM_UNIQUE(users.affiliate_provider)', 'buckets.NUM_UNIQUE(users.first_affiliate_tracked)', 'buckets.NUM_UNIQUE(users.signup_app)', 'buckets.NUM_UNIQUE(users.first_device_type)', 'buckets.NUM_UNIQUE(users.first_browser)', 'buckets.NUM_UNIQUE(users.signup_flow)', 'buckets.SKEW(users.age)', 'buckets.MIN(users.age)', 'buckets.MEAN(users.age)', 'buckets.STD(users.age)', 'buckets.MAX(users.age)', 'buckets.MEDIAN(users.age)', 'buckets.MODE(users.signup_flow)', 'buckets.LAST(sessions.secs_elapsed)', 'buckets.NUM_UNIQUE(sessions.action)', 'buckets.NUM_UNIQUE(sessions.action_type)', 'buckets.NUM_UNIQUE(sessions.action_detail)', 'buckets.NUM_UNIQUE(sessions.device_type)', 'buckets.SKEW(sessions.secs_elapsed)', 'buckets.MIN(sessions.secs_elapsed)', 'buckets.MEAN(sessions.secs_elapsed)', 'buckets.STD(sessions.secs_elapsed)', 'buckets.MAX(sessions.secs_elapsed)', 'buckets.MEDIAN(sessions.secs_elapsed)', 'buckets.CUM_SUM(CA)', 'buckets.CUM_SUM(DE)', 'buckets.CUM_SUM(FR)', 'buckets.CUM_SUM(GB)', 'buckets.CUM_SUM(AU)', 'buckets.CUM_SUM(NL)', 'buckets.CUM_SUM(US)', 'buckets.CUM_SUM(IT)', 'buckets.CUM_SUM(PT)', 'buckets.CUM_SUM(ES)']\n",
      "CATEGORICAL VARIABLES:  ['gender', 'signup_method', 'language', 'affiliate_channel', 'affiliate_provider', 'first_affiliate_tracked', 'signup_app', 'first_device_type', 'first_browser', 'age_gender_bucket', 'LAST(sessions.action)', 'LAST(sessions.action_type)', 'LAST(sessions.action_detail)', 'LAST(sessions.device_type)', 'MODE(sessions.action)', 'MODE(sessions.action_type)', 'MODE(sessions.action_detail)', 'MODE(sessions.device_type)', 'IS_WEEKEND(date_account_created)', 'IS_WEEKEND(timestamp_first_active)', 'buckets.LAST(users.gender)', 'buckets.LAST(users.signup_method)', 'buckets.LAST(users.language)', 'buckets.LAST(users.affiliate_channel)', 'buckets.LAST(users.affiliate_provider)', 'buckets.LAST(users.first_affiliate_tracked)', 'buckets.LAST(users.signup_app)', 'buckets.LAST(users.first_device_type)', 'buckets.LAST(users.first_browser)', 'buckets.MODE(users.gender)', 'buckets.MODE(users.signup_method)', 'buckets.MODE(users.language)', 'buckets.MODE(users.affiliate_channel)', 'buckets.MODE(users.affiliate_provider)', 'buckets.MODE(users.first_affiliate_tracked)', 'buckets.MODE(users.signup_app)', 'buckets.MODE(users.first_device_type)', 'buckets.MODE(users.first_browser)', 'buckets.LAST(sessions.action)', 'buckets.LAST(sessions.action_type)', 'buckets.LAST(sessions.action_detail)', 'buckets.LAST(sessions.device_type)', 'buckets.MODE(sessions.action)', 'buckets.MODE(sessions.action_type)', 'buckets.MODE(sessions.action_detail)', 'buckets.MODE(sessions.device_type)']\n"
     ]
    }
   ],
   "source": [
    "numeric_variables = X_train.select_dtypes(include=[np.number]).columns.tolist()\n",
    "categorical_variables = [x for x in X_train.columns.tolist() if x not in numeric_variables]\n",
    "print(\"NUMERIC VARIABLES: \", numeric_variables)\n",
    "print(\"CATEGORICAL VARIABLES: \", categorical_variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaler Choice\n",
    "You can check the most common feature scaling methods from [here](https://en.wikipedia.org/wiki/Feature_scaling). The most effective algorithms are **Min-Max Scaling**, **Mean Normalization**, **Gaussian (Standard) Scaling**, **Unit-Lenth Scaling**, **Robust Scaling**, **Logarithmic Scaling**, and **Exponential Scaling** depending on the application. Here, we went with a fairly safe method: Gaussian (Standard) scaling based on mean and standard deviation of the variable sample. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>signup_method</th>\n",
       "      <th>language</th>\n",
       "      <th>affiliate_channel</th>\n",
       "      <th>affiliate_provider</th>\n",
       "      <th>first_affiliate_tracked</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>first_device_type</th>\n",
       "      <th>first_browser</th>\n",
       "      <th>...</th>\n",
       "      <th>buckets.CUM_SUM(CA)</th>\n",
       "      <th>buckets.CUM_SUM(DE)</th>\n",
       "      <th>buckets.CUM_SUM(FR)</th>\n",
       "      <th>buckets.CUM_SUM(GB)</th>\n",
       "      <th>buckets.CUM_SUM(AU)</th>\n",
       "      <th>buckets.CUM_SUM(NL)</th>\n",
       "      <th>buckets.CUM_SUM(US)</th>\n",
       "      <th>buckets.CUM_SUM(IT)</th>\n",
       "      <th>buckets.CUM_SUM(PT)</th>\n",
       "      <th>buckets.CUM_SUM(ES)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>-0.644729</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>omg</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.632065</td>\n",
       "      <td>-0.622391</td>\n",
       "      <td>-0.619435</td>\n",
       "      <td>-0.629440</td>\n",
       "      <td>-0.637644</td>\n",
       "      <td>-0.620349</td>\n",
       "      <td>-0.623034</td>\n",
       "      <td>-0.664049</td>\n",
       "      <td>-0.687217</td>\n",
       "      <td>-0.710146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>0.504372</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>iPad</td>\n",
       "      <td>Mobile Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>0.296425</td>\n",
       "      <td>0.237997</td>\n",
       "      <td>0.273689</td>\n",
       "      <td>0.285184</td>\n",
       "      <td>0.326004</td>\n",
       "      <td>0.250720</td>\n",
       "      <td>0.295706</td>\n",
       "      <td>0.283107</td>\n",
       "      <td>0.334959</td>\n",
       "      <td>0.342713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>0.683121</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>Firefox</td>\n",
       "      <td>...</td>\n",
       "      <td>0.758740</td>\n",
       "      <td>0.751521</td>\n",
       "      <td>0.770716</td>\n",
       "      <td>0.776740</td>\n",
       "      <td>0.794609</td>\n",
       "      <td>0.764677</td>\n",
       "      <td>0.751388</td>\n",
       "      <td>0.834417</td>\n",
       "      <td>0.846047</td>\n",
       "      <td>0.872740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MALE</td>\n",
       "      <td>0.159642</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Web</td>\n",
       "      <td>Mac Desktop</td>\n",
       "      <td>Safari</td>\n",
       "      <td>...</td>\n",
       "      <td>0.523916</td>\n",
       "      <td>0.463725</td>\n",
       "      <td>0.521754</td>\n",
       "      <td>0.517265</td>\n",
       "      <td>0.564228</td>\n",
       "      <td>0.496104</td>\n",
       "      <td>0.518075</td>\n",
       "      <td>0.555842</td>\n",
       "      <td>0.597120</td>\n",
       "      <td>0.621303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FEMALE</td>\n",
       "      <td>-0.402141</td>\n",
       "      <td>basic</td>\n",
       "      <td>en</td>\n",
       "      <td>direct</td>\n",
       "      <td>direct</td>\n",
       "      <td>untracked</td>\n",
       "      <td>Moweb</td>\n",
       "      <td>Windows Desktop</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.632065</td>\n",
       "      <td>-0.622391</td>\n",
       "      <td>-0.619435</td>\n",
       "      <td>-0.629440</td>\n",
       "      <td>-0.637644</td>\n",
       "      <td>-0.620349</td>\n",
       "      <td>-0.623034</td>\n",
       "      <td>-0.664049</td>\n",
       "      <td>-0.687217</td>\n",
       "      <td>-0.710146</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender       age signup_method language affiliate_channel  \\\n",
       "0  FEMALE -0.644729         basic       en            direct   \n",
       "1  FEMALE  0.504372         basic       en            direct   \n",
       "2  FEMALE  0.683121         basic       en            direct   \n",
       "3    MALE  0.159642         basic       en            direct   \n",
       "4  FEMALE -0.402141         basic       en            direct   \n",
       "\n",
       "  affiliate_provider first_affiliate_tracked signup_app first_device_type  \\\n",
       "0             direct                     omg        Web       Mac Desktop   \n",
       "1             direct               untracked        Web              iPad   \n",
       "2             direct               untracked        Web   Windows Desktop   \n",
       "3             direct               untracked        Web       Mac Desktop   \n",
       "4             direct               untracked      Moweb   Windows Desktop   \n",
       "\n",
       "   first_browser         ...          buckets.CUM_SUM(CA)  \\\n",
       "0         Safari         ...                    -0.632065   \n",
       "1  Mobile Safari         ...                     0.296425   \n",
       "2        Firefox         ...                     0.758740   \n",
       "3         Safari         ...                     0.523916   \n",
       "4         Chrome         ...                    -0.632065   \n",
       "\n",
       "   buckets.CUM_SUM(DE) buckets.CUM_SUM(FR) buckets.CUM_SUM(GB)  \\\n",
       "0            -0.622391           -0.619435           -0.629440   \n",
       "1             0.237997            0.273689            0.285184   \n",
       "2             0.751521            0.770716            0.776740   \n",
       "3             0.463725            0.521754            0.517265   \n",
       "4            -0.622391           -0.619435           -0.629440   \n",
       "\n",
       "  buckets.CUM_SUM(AU) buckets.CUM_SUM(NL)  buckets.CUM_SUM(US)  \\\n",
       "0           -0.637644           -0.620349            -0.623034   \n",
       "1            0.326004            0.250720             0.295706   \n",
       "2            0.794609            0.764677             0.751388   \n",
       "3            0.564228            0.496104             0.518075   \n",
       "4           -0.637644           -0.620349            -0.623034   \n",
       "\n",
       "   buckets.CUM_SUM(IT)  buckets.CUM_SUM(PT)  buckets.CUM_SUM(ES)  \n",
       "0            -0.664049            -0.687217            -0.710146  \n",
       "1             0.283107             0.334959             0.342713  \n",
       "2             0.834417             0.846047             0.872740  \n",
       "3             0.555842             0.597120             0.621303  \n",
       "4            -0.664049            -0.687217            -0.710146  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = StandardScaler()\n",
    "scaler = scaler.fit(X_train[numeric_variables])\n",
    "X_train[numeric_variables] = scaler.transform(X_train[numeric_variables])\n",
    "X_test[numeric_variables] = scaler.transform(X_test[numeric_variables])\n",
    "# Observe scaling effect\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection\n",
    "Not only that feature selection yields smaller training & test sets and decreases training time vastly, it also prevents models from overfitting and allows them to generalize better. Moreover, as different models react to high numbers of feature spaces differently, we decided that feature selection might allow us to compare these models in a more fair way."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setting Back Unknown Categorical Levels & Setting 0's for Numeric NaNs\n",
    "In the previous notebook, we have converted the newly introduced missing values (from automated feature engineering) in all columns to NaNs for uniformity in representation. However, NaNs for numeric variables tell us that the corresponding users didn't have any related session information. For the majority of the numeric features we have generated, plugging in 0s as missing values seems to be logical here. On the other hand, NaNs for categorical variables tell us that corresponding users have untracked information or they used a tool/utility/method that is not recognized. Defining in a new categorical level of 'UNKNOWN' seems to be logical here. We also need these operations to eliminate missing values before applying feature selection algorithms."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[categorical_variables] = X_train[categorical_variables].fillna('UNKNOWN')\n",
    "X_test[categorical_variables] = X_test[categorical_variables].fillna('UNKNOWN')\n",
    "X_train[numeric_variables] = X_train[numeric_variables].fillna(0.0)\n",
    "X_test[numeric_variables] = X_test[numeric_variables].fillna(0.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One-Hot Encoding Categorical Variables\n",
    "Before we proceed any further, we will have to *temporarily* one-hot encode our categorical variables so that they fit in with our feature selection methods. (NOTE: Particulary, we are referring to the estimation of *chi-squared test statistic* and the utilization of *logistic regression* models.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform training set to one-hot encoded representation & get encoded columns\n",
    "X_train_onehot_encoded = categorical_to_onehot_columns(df=X_train)\n",
    "encoded_columns = X_train_onehot_encoded.columns.values.tolist()\n",
    "# Transform test set to one-hot encoded representation\n",
    "X_test_onehot_encoded = categorical_to_onehot_columns(df=X_test)\n",
    "# Add categorical levels that exist in training set to test set\n",
    "for fitted_column in encoded_columns:\n",
    "    if fitted_column not in X_test_onehot_encoded.columns.values.tolist():\n",
    "        X_test_onehot_encoded[fitted_column] = 0\n",
    "# Drop categorical levels that don't exist in training set from test set\n",
    "for column in X_test_onehot_encoded.columns.values.tolist():\n",
    "    if column not in encoded_columns:\n",
    "        X_test_onehot_encoded.drop(column, axis=1, inplace=True)\n",
    "# Ensure that training and test sets have the same column-wise order\n",
    "X_test_onehot_encoded = X_test_onehot_encoded[encoded_columns]\n",
    "assert len(X_train_onehot_encoded.columns) == len(X_test_onehot_encoded.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection Algorithms\n",
    "Common feature selection algorithms include:\n",
    "* **Pearson Correlation Coefficient**: Perphaps the most common and easiest way to perform feature selection. However, we will have to try different methods as our response variables are categorical rather than numerical. \n",
    "* **Chi-Squared Test Statistic**: The chi-square test is a statistical test of independence to determine the dependency of two variables. If the target variable is independent of the feature variable, we can discard that feature variable. If they are dependent, the feature variable is very important. It's a pretty conventional method.\n",
    "* **Recursive Feature Elimination (RFE)**: Feature ranking with recursive feature elimination.\n",
    "* **Variable Importances from a Baseline Model**: Applies common algorithms such as Random Forests, Logistic Regression, and XGBoost.\n",
    "\n",
    "For the problem at hand, we have focused on the last three algorithms."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Chi-Squared Test Statistic\n",
    "The chi-squared test does not apply to negative values,  because it assumes a distribution of frequencies. Hence, we will first check if any negative values exist in the data, and normalize values to [0, 1] if so."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize values to [0, 1] interval, column-wise\n",
    "if len(X_train_onehot_encoded[X_train_onehot_encoded < 0]) > 0:\n",
    "    min_max_scaler = MinMaxScaler()\n",
    "    normalized_values = min_max_scaler.fit_transform(X_train_onehot_encoded.T)\n",
    "    X_train_onehot_encoded_normalized = pd.DataFrame(normalized_values.T,\n",
    "                                                     columns=X_train_onehot_encoded.columns,\n",
    "                                                     index=X_train_onehot_encoded.index)\n",
    "else:\n",
    "    X_train_onehot_encoded_normalized = X_train_onehot_encoded\n",
    "    \n",
    "chi_model = SelectKBest(chi2, k=200)\n",
    "chi_model.fit(X_train_onehot_encoded_normalized, Y_train)\n",
    "chi_support = chi_model.get_support()\n",
    "chi_selected_columns = X_train_onehot_encoded.loc[:, chi_support].columns.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Recursive Feature Elimination (RFE) by Logistic Regression (L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 1031 features.\n",
      "Fitting estimator with 931 features.\n",
      "Fitting estimator with 831 features.\n",
      "Fitting estimator with 731 features.\n",
      "Fitting estimator with 631 features.\n",
      "Fitting estimator with 531 features.\n",
      "Fitting estimator with 431 features.\n",
      "Fitting estimator with 331 features.\n",
      "Fitting estimator with 231 features.\n"
     ]
    }
   ],
   "source": [
    "rfe_model = RFE(estimator=LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=150, \n",
    "                                             penalty='l2', random_state=SEED, n_jobs=-1), \n",
    "                n_features_to_select=200, step=100, verbose=5)\n",
    "rfe_model.fit(X_train_onehot_encoded, Y_train)\n",
    "rfe_support = rfe_model.get_support()\n",
    "rfe_selected_columns = X_train_onehot_encoded.loc[:, rfe_support].columns.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Variable Importances by Logistic Regression (L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_model = SelectFromModel(LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=150,\n",
    "                                              penalty='l2', random_state=SEED, n_jobs=-1), \n",
    "                           threshold='1.25*median')\n",
    "lr_model.fit(X_train_onehot_encoded, Y_train)\n",
    "lr_support = lr_model.get_support()\n",
    "lr_selected_columns = X_train_onehot_encoded.loc[:, lr_support].columns.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combining All Methods\n",
    "By observing the *union* and *intersection* of the different features we have selected with the given methods above, we have decided that limiting the training & test sets to the *intersection* features seems like best option. This was partly because the *union* features were still large in number. It should be noted that the feature selection process adopted here is not optimized in any way. We could have used different *models*, *parameters*, and *numbers of independent features to select* in each method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOTAL NUMBER OF FEATURES SELECTED:  487\n",
      "ALL SELECTED FEATURES:  ['buckets.LAST(users.gender)=UNKNOWN', 'buckets.LAST(users.first_device_type)=Other/Unknown', 'MODE(sessions.action)=update', 'buckets.MODE(users.first_browser)=IE', 'buckets.MODE(users.first_affiliate_tracked)=UNKNOWN', 'LAST(sessions.action)=authenticate', 'buckets.MAX(sessions.secs_elapsed)', 'buckets.LAST(users.affiliate_provider)=facebook', 'LAST(sessions.action)=faq_category', 'LAST(sessions.action_detail)=oauth_login', 'buckets.NUM_UNIQUE(users.affiliate_channel)', 'WEEK(timestamp_first_active)', 'LAST(sessions.action)=edit_verification', 'language=en', 'buckets.LAST(sessions.action_type)=submit', 'LAST(sessions.action)=header_userpic', 'age_gender_bucket=65-69female', 'first_device_type=Desktop (Other)', 'MAX(sessions.secs_elapsed)', 'MODE(sessions.action_type)=data', 'buckets.MODE(sessions.device_type)=Windows Desktop', 'MODE(sessions.action)=faq_category', 'age_gender_bucket=30-34other', 'LAST(sessions.action_detail)=create_user', 'buckets.CUM_SUM(DE)', 'LAST(sessions.action)=recommended_listings', 'buckets.LAST(sessions.action_detail)=user_profile', 'age_gender_bucket=55-59male', 'buckets.MODE(users.signup_method)=facebook', 'MODE(sessions.action_detail)=apply_coupon', 'MODE(sessions.action_detail)=list_your_space', 'LAST(sessions.action)=impressions', 'language=ru', 'buckets.MIN(users.age)', 'buckets.NL', 'first_browser=Chrome Mobile', 'buckets.AU', 'LAST(sessions.action_detail)=view_search_results', 'LAST(sessions.action)=reviews', 'MODE(sessions.device_type)=iPad Tablet', 'LAST(sessions.action_type)=view', 'LAST(sessions.action_detail)=notifications', 'buckets.NUM_UNIQUE(sessions.device_type)', 'gender=FEMALE', 'gender=MALE', 'buckets.CUM_SUM(NL)', 'age_gender_bucket=20-24female', 'buckets.LAST(users.signup_app)=UNKNOWN', 'signup_flow', 'MODE(sessions.action_type)=booking_request', 'language=de', 'CUM_SUM(NUM_UNIQUE(sessions.action_type))', 'affiliate_channel=sem-brand', 'language=zh', 'affiliate_provider=email-marketing', 'buckets.LAST(users.language)=en', 'buckets.LAST(sessions.action)=request_new_confirm_email', 'MODE(sessions.action)=cancellation_policies', 'age_gender_bucket=70-74female', 'first_affiliate_tracked=omg', 'MODE(sessions.device_type)=Chromebook', 'first_browser=AOL Explorer', 'MODE(sessions.action)=UNKNOWN', 'buckets.NUM_UNIQUE(sessions.action)', 'buckets.MODE(users.affiliate_provider)=google', 'buckets.MODE(users.gender)=MALE', 'MODE(sessions.action_detail)=p3', 'SKEW(sessions.CUM_SUM(secs_elapsed))', 'buckets.LAST(users.affiliate_provider)=google', 'buckets.LAST(sessions.action_type)=UNKNOWN', 'signup_app=Android', 'buckets.LAST(users.affiliate_channel)=other', 'LAST(sessions.action)=reviews_new', 'buckets.MODE(users.affiliate_provider)=direct', 'LAST(sessions.action_detail)=create_phone_numbers', 'MODE(sessions.action)=header_userpic', 'affiliate_channel=other', 'LAST(sessions.action_detail)=user_profile_content_update', 'buckets.NUM_UNIQUE(sessions.action_detail)', 'gender=OTHER', 'LAST(sessions.action_detail)=your_trips', 'LAST(sessions.action)=this_hosting_reviews', 'buckets.STD(sessions.secs_elapsed)', 'LAST(sessions.action)=profile_pic', 'LAST(sessions.action_detail)=book_it', 'buckets.MODE(sessions.device_type)=Mac Desktop', 'LAST(sessions.action_type)=UNKNOWN', 'buckets.NUM_UNIQUE(users.language)', 'LAST(sessions.action_detail)=change_trip_characteristics', 'MODE(sessions.action_detail)=confirm_email_link', 'affiliate_provider=facebook', 'language=es', 'buckets.LAST(users.signup_method)=facebook', 'LAST(sessions.action)=update', 'MODE(sessions.action)=index', 'buckets.LAST(sessions.device_type)=Mac Desktop', 'LAST(sessions.action_detail)=update_listing_description', 'buckets.MODE(users.affiliate_provider)=UNKNOWN', 'MODE(sessions.action)=ask_question', 'buckets.LAST(sessions.device_type)=Android Phone', 'buckets.MODE(users.first_affiliate_tracked)=untracked', 'MODE(sessions.device_type)=Android App Unknown Phone/Tablet', 'LAST(sessions.action_type)=message_post', 'first_browser=Chromium', 'first_device_type=iPad', 'buckets.LAST(users.gender)=FEMALE', 'first_device_type=Android Phone', 'LAST(sessions.action)=similar_listings', 'MODE(sessions.action_detail)=account_notification_settings', 'LAST(sessions.action)=ajax_lwlb_contact', 'LAST(sessions.action_detail)=edit_profile', 'LAST(sessions.action_detail)=login', 'LAST(sessions.action)=campaigns', 'MODE(sessions.action_detail)=contact_host', 'buckets.LAST(sessions.action)=header_userpic', 'first_browser=Chrome', 'LAST(sessions.device_type)=Mac Desktop', 'affiliate_channel=remarketing', 'language=ja', 'buckets.LAST(users.first_browser)=UNKNOWN', 'buckets.CUM_SUM(IT)', 'MODE(sessions.action)=requested', 'MIN(sessions.secs_elapsed)', 'MODE(sessions.action)=dashboard', 'CUM_SUM(MEDIAN(sessions.secs_elapsed))', 'CUM_SUM(LAST(sessions.secs_elapsed))', 'first_device_type=Windows Desktop', 'MODE(sessions.action_type)=click', 'affiliate_provider=padmapper', 'MODE(sessions.action_detail)=message_thread', 'first_device_type=iPhone', 'buckets.MODE(sessions.device_type)=UNKNOWN', 'MODE(sessions.action)=ajax_get_referrals_amt', 'age_gender_bucket=25-29female', 'LAST(sessions.action_detail)=listing_reviews_page', 'MODE(sessions.action)=complete_status', 'first_affiliate_tracked=product', 'LAST(sessions.action_detail)=header_userpic', 'LAST(sessions.action_detail)=p4', 'age_gender_bucket=35-39male', 'LAST(sessions.action)=account', 'LAST(sessions.action_type)=data', 'buckets.MIN(sessions.secs_elapsed)', 'LAST(sessions.action_detail)=message_thread', 'LAST(sessions.action_detail)=p5', 'LAST(sessions.action)=active', 'age_gender_bucket=30-34male', 'affiliate_provider=vast', 'LAST(sessions.action_detail)=UNKNOWN', 'LAST(sessions.action)=available', 'buckets.ES', 'LAST(sessions.action)=collections', 'buckets.MODE(users.first_device_type)=iPhone', 'buckets.CUM_SUM(GB)', 'buckets.LAST(users.first_affiliate_tracked)=UNKNOWN', 'buckets.LAST(users.affiliate_provider)=direct', 'MODE(sessions.action_type)=message_post', 'buckets.CUM_SUM(PT)', 'WEEK(date_account_created)', 'MODE(sessions.device_type)=Android Phone', 'MODE(sessions.action_detail)=unavailable_dates', 'buckets.NUM_UNIQUE(sessions.action_type)', 'first_browser=IE', 'MODE(sessions.action)=apply_reservation', 'affiliate_provider=direct', 'LAST(sessions.action)=unavailabilities', 'MODE(sessions.action)=click', 'LAST(sessions.action)=delete', 'LAST(sessions.action)=new', 'MODE(sessions.device_type)=Tablet', 'MODE(sessions.action_detail)=manage_listing', 'buckets.LAST(sessions.action_type)=click', 'LAST(sessions.action)=hosting_social_proof', 'LAST(sessions.device_type)=iPhone', 'MODE(sessions.action_detail)=message_post', 'MODE(sessions.device_type)=Windows Desktop', 'NUM_UNIQUE(sessions.device_type)', 'buckets.NUM_UNIQUE(users.first_device_type)', 'buckets.STD(users.age)', 'signup_method=facebook', 'buckets.LAST(users.first_affiliate_tracked)=omg', 'LAST(sessions.action_detail)=listing_reviews', 'LAST(sessions.action)=ajax_refresh_subtotal', 'LAST(sessions.device_type)=Windows Desktop', 'MODE(sessions.action_detail)=header_userpic', 'buckets.LAST(users.first_affiliate_tracked)=untracked', 'buckets.US', 'CUM_SUM(SKEW(sessions.secs_elapsed))', 'buckets.CUM_SUM(AU)', 'buckets.CUM_SUM(FR)', 'MODE(sessions.action_detail)=wishlist_content_update', 'LAST(sessions.action_type)=submit', 'MODE(sessions.action_detail)=create_phone_numbers', 'LAST(sessions.device_type)=iPad Tablet', 'LAST(sessions.action_type)=booking_request', 'LAST(sessions.action)=qt2', 'buckets.LAST(users.first_browser)=Chrome', 'DAY(date_account_created)', 'buckets.CUM_SUM(ES)', 'LAST(sessions.action)=complete_status', 'LAST(sessions.action_detail)=change_contact_host_dates', 'affiliate_provider=yahoo', 'LAST(sessions.action_type)=click', 'MODE(sessions.action)=search', 'LAST(sessions.action)=identity', 'LAST(sessions.action)=languages_multiselect', 'MODE(sessions.action)=lookup', 'CUM_SUM(NUM_UNIQUE(sessions.device_type))', 'LAST(sessions.action_detail)=wishlist_content_update', 'buckets.LAST(sessions.action)=ask_question', 'MODE(sessions.action)=ajax_photo_widget_form_iframe', 'buckets.MODE(users.first_device_type)=UNKNOWN', 'LAST(sessions.action_detail)=contact_host', 'MODE(sessions.action)=search_results', 'MODE(sessions.action_detail)=dashboard', 'buckets.LAST(users.language)=UNKNOWN', 'buckets.PT', 'LAST(sessions.action_detail)=similar_listings', 'CUM_SUM(age)', 'LAST(sessions.action)=search', 'MODE(sessions.action)=tos_confirm', 'LAST(sessions.action)=verify', 'age_gender_bucket=40-44female', 'SKEW(sessions.secs_elapsed)', 'affiliate_channel=seo', 'MODE(sessions.action)=payment_methods', 'LAST(sessions.action)=manage_listing', 'MODE(sessions.action)=similar_listings', 'LAST(sessions.action)=ajax_get_referrals_amt', 'MODE(sessions.action)=message_to_host_focus', 'buckets.NUM_UNIQUE(users.first_browser)', 'affiliate_channel=direct', 'MODE(sessions.device_type)=UNKNOWN', 'buckets.MODE(users.first_browser)=Safari', 'MONTH(timestamp_first_active)', 'age_gender_bucket=40-44male', 'MODE(sessions.action)=manage_listing', 'LAST(sessions.action_detail)=dashboard', 'LAST(sessions.action_detail)=manage_listing', 'buckets.MODE(users.first_browser)=Chrome', 'signup_app=iOS', 'buckets.MODE(sessions.action)=show', 'signup_app=Moweb', 'buckets.MODE(sessions.action_type)=view', 'buckets.MODE(sessions.action_detail)=view_search_results', 'MODE(sessions.action)=confirm_email', 'first_affiliate_tracked=marketing', 'LAST(sessions.action)=create', 'STD(sessions.CUM_SUM(secs_elapsed))', 'buckets.MODE(users.first_browser)=Mobile Safari', 'NUM_UNIQUE(sessions.action)', 'LAST(sessions.action)=pending', 'buckets.LAST(users.first_device_type)=Mac Desktop', 'LAST(sessions.CUM_SUM(secs_elapsed))', 'MODE(sessions.action)=show', 'LAST(sessions.action)=social_connections', 'MODE(sessions.action)=references', 'buckets.LAST(sessions.action_detail)=p3', 'LAST(sessions.action_detail)=signup', 'LAST(sessions.action_detail)=your_listings', 'buckets.MODE(users.first_device_type)=Windows Desktop', 'buckets.LAST(users.first_device_type)=Windows Desktop', 'buckets.LAST(sessions.action_detail)=request_new_confirm_email', 'buckets.LAST(users.affiliate_provider)=UNKNOWN', 'MEAN(sessions.secs_elapsed)', 'MODE(sessions.action_detail)=signup', 'LAST(sessions.action)=personalize', 'buckets.LAST(users.first_browser)=Mobile Safari', 'LAST(sessions.action)=ajax_referral_banner_type', 'LAST(sessions.action_detail)=pending', 'LAST(sessions.action)=connect', 'LAST(sessions.action_detail)=trip_availability', 'buckets.IT', 'LAST(sessions.action_detail)=p3', 'MODE(sessions.action_detail)=p5', 'first_device_type=Mac Desktop', 'MODE(sessions.action_detail)=UNKNOWN', 'buckets.LAST(sessions.action)=index', 'buckets.LAST(sessions.action)=search', 'LAST(sessions.action_detail)=message_post', 'LAST(sessions.action)=populate_help_dropdown', 'buckets.MODE(users.affiliate_channel)=sem-non-brand', 'buckets.LAST(users.gender)=MALE', 'first_affiliate_tracked=untracked', 'LAST(sessions.action_detail)=cancellation_policies', 'MODE(sessions.action)=calendar_tab_inner2', 'LAST(sessions.action)=faq', 'buckets.LAST(sessions.action_detail)=view_search_results', 'LAST(sessions.action)=dashboard', 'IS_WEEKEND(date_account_created)', 'first_browser=Firefox', 'LAST(sessions.action)=kba', 'MODE(sessions.action_detail)=login', 'affiliate_channel=api', 'buckets.LAST(sessions.action_detail)=contact_host', 'buckets.MODE(users.affiliate_channel)=UNKNOWN', 'buckets.MODE(sessions.action_type)=UNKNOWN', 'MODE(sessions.action)=pending', 'LAST(sessions.action)=message_to_host_focus', 'LAST(sessions.action_detail)=unavailable_dates', 'MODE(sessions.action_detail)=view_search_results', 'buckets.LAST(sessions.action)=ajax_refresh_subtotal', 'CUM_SUM(MIN(sessions.secs_elapsed))', 'MODE(sessions.action_detail)=pending', 'LAST(sessions.secs_elapsed)', 'signup_app=Web', 'buckets.LAST(users.affiliate_channel)=seo', 'buckets.CUM_SUM(US)', 'affiliate_provider=bing', 'LAST(sessions.device_type)=Tablet', 'LAST(sessions.action)=confirm_email', 'MODE(sessions.action_detail)=message_to_host_focus', 'buckets.MEDIAN(sessions.secs_elapsed)', 'MODE(sessions.action_detail)=user_profile', 'buckets.LAST(sessions.action)=collections', 'LAST(sessions.action)=notifications', 'MODE(sessions.action)=create', 'MODE(sessions.action_detail)=update_listing', 'LAST(sessions.action)=other_hosting_reviews_first', 'LAST(sessions.action_type)=partner_callback', 'LAST(sessions.action)=requested', 'buckets.LAST(sessions.action_detail)=header_userpic', 'LAST(sessions.action)=show', 'LAST(sessions.action)=travel_plans_current', 'LAST(sessions.action_detail)=post_checkout_action', 'MODE(sessions.action_detail)=user_wishlists', 'first_browser=Mobile Safari', 'buckets.MEAN(sessions.secs_elapsed)', 'MODE(sessions.action)=edit', 'buckets.MODE(users.signup_method)=basic', 'MODE(sessions.action_type)=UNKNOWN', 'buckets.NUM_UNIQUE(users.signup_app)', 'MODE(sessions.action_detail)=similar_listings', 'buckets.LAST(users.first_browser)=Safari', 'MIN(sessions.CUM_SUM(secs_elapsed))', 'buckets.LAST(sessions.device_type)=Windows Desktop', 'MODE(sessions.action)=track_page_view', 'buckets.MODE(users.gender)=UNKNOWN', 'first_browser=Android Browser', 'LAST(sessions.action)=ajax_check_dates', 'LAST(sessions.action)=index', 'MODE(sessions.action)=authenticate', 'first_affiliate_tracked=linked', 'LAST(sessions.action)=edit', 'first_affiliate_tracked=tracked-other', 'buckets.LAST(users.first_browser)=Firefox', 'buckets.LAST(users.signup_app)=iOS', 'LAST(sessions.action)=click', 'language=ko', 'affiliate_provider=gsp', 'buckets.LAST(users.age)', 'MODE(sessions.action_detail)=cancellation_policies', 'LAST(sessions.action)=phone_number_widget', 'buckets.MODE(users.first_browser)=UNKNOWN', 'age_gender_bucket=15-19female', 'MEAN(sessions.CUM_SUM(secs_elapsed))', 'MODE(sessions.device_type)=Mac Desktop', 'LAST(sessions.action)=show_personalize', 'buckets.LAST(sessions.device_type)=iPhone', 'YEAR(timestamp_first_active)', 'HOUR(timestamp_first_active)', 'buckets.MEAN(users.age)', 'signup_method=google', 'buckets.LAST(users.signup_app)=Web', 'buckets.DE', 'buckets.LAST(sessions.action_detail)=UNKNOWN', 'age', 'buckets.LAST(sessions.action_detail)=change_trip_characteristics', 'age_gender_bucket=50-54male', 'language=fr', 'affiliate_provider=meetup', 'MODE(sessions.action)=personalize', 'MODE(sessions.action)=qt2', 'buckets.LAST(sessions.action_type)=view', 'STD(sessions.secs_elapsed)', 'LAST(sessions.action_detail)=update_listing', 'buckets.NUM_UNIQUE(users.first_affiliate_tracked)', 'buckets.LAST(users.first_affiliate_tracked)=linked', 'CUM_SUM(STD(sessions.secs_elapsed))', 'MODE(sessions.action_detail)=edit_profile', 'affiliate_provider=google', 'CUM_SUM(NUM_UNIQUE(sessions.action))', 'first_browser=Safari', 'LAST(sessions.action_detail)=user_profile', 'buckets.MODE(users.gender)=FEMALE', 'buckets.NUM_UNIQUE(users.signup_method)', 'LAST(sessions.action)=set_user', 'LAST(sessions.action)=callback', 'NUM_UNIQUE(sessions.action_type)', 'buckets.LAST(sessions.action)=UNKNOWN', 'MODE(sessions.device_type)=iPhone', 'buckets.MODE(sessions.device_type)=iPhone', 'signup_method=basic', 'DAY(timestamp_first_active)', 'LAST(sessions.action_detail)=list_your_space', 'LAST(sessions.device_type)=UNKNOWN', 'buckets.MODE(users.affiliate_channel)=direct', 'CUM_SUM(MAX(sessions.secs_elapsed))', 'LAST(sessions.action_detail)=oauth_response', 'MODE(sessions.action_detail)=book_it', 'buckets.MODE(users.first_affiliate_tracked)=omg', 'buckets.MEDIAN(users.age)', 'LAST(sessions.device_type)=Android App Unknown Phone/Tablet', 'buckets.NUM_UNIQUE(users.signup_flow)', 'buckets.FR', 'IS_WEEKEND(timestamp_first_active)', 'LAST(sessions.action_detail)=update_user_profile', 'LAST(sessions.action_detail)=user_wishlists', 'MONTH(date_account_created)', 'buckets.MODE(users.language)=en', 'MODE(sessions.action_type)=view', 'LAST(sessions.action)=track_page_view', 'buckets.LAST(sessions.action)=show', 'MODE(sessions.action)=active', 'CUM_SUM(MEAN(sessions.secs_elapsed))', 'buckets.MODE(users.signup_method)=UNKNOWN', 'MEDIAN(sessions.CUM_SUM(secs_elapsed))', 'buckets.LAST(users.first_device_type)=UNKNOWN', 'MEDIAN(sessions.secs_elapsed)', 'buckets.LAST(users.signup_method)=basic', 'LAST(sessions.action)=ask_question', 'buckets.SKEW(sessions.secs_elapsed)', 'LAST(sessions.action)=facebook_auto_login', 'buckets.LAST(sessions.action)=update', 'LAST(sessions.action)=ajax_photo_widget_form_iframe', 'MAX(sessions.CUM_SUM(secs_elapsed))', 'buckets.MODE(users.first_affiliate_tracked)=linked', 'LAST(sessions.action)=UNKNOWN', 'age_gender_bucket=45-49female', 'LAST(sessions.action)=lookup', 'buckets.LAST(sessions.action_detail)=update_listing', 'LAST(sessions.action_detail)=create_listing', 'LAST(sessions.action_detail)=confirm_email_link', 'MODE(sessions.action)=account', 'buckets.LAST(sessions.action_detail)=user_wishlists', 'buckets.MODE(sessions.action)=UNKNOWN', 'language=it', 'LAST(sessions.action_detail)=user_social_connections', 'NUM_UNIQUE(sessions.action_detail)', 'MODE(sessions.action)=edit_verification', 'affiliate_channel=sem-non-brand', 'buckets.LAST(users.signup_method)=UNKNOWN', 'affiliate_provider=facebook-open-graph', 'LAST(sessions.action)=search_results', 'MODE(sessions.action)=ajax_refresh_subtotal', 'LAST(sessions.action)=payment_methods', 'buckets.MAX(users.age)', 'first_device_type=Other/Unknown', 'buckets.LAST(users.affiliate_channel)=sem-brand', 'MODE(sessions.action_detail)=create_user', 'buckets.NUM_UNIQUE(users.affiliate_provider)', 'buckets.MODE(users.language)=UNKNOWN', 'YEAR(date_account_created)', 'CUM_SUM(NUM_UNIQUE(sessions.action_detail))', 'MODE(sessions.action_type)=submit', 'language=pt', 'buckets.MODE(users.signup_app)=Web', 'affiliate_channel=content', 'MODE(sessions.action)=notifications', 'affiliate_provider=craigslist', 'affiliate_provider=other', 'buckets.LAST(users.affiliate_channel)=direct', 'LAST(sessions.action_detail)=profile_verifications', 'LAST(sessions.action_detail)=p1', 'LAST(sessions.action_detail)=user_listings', 'LAST(sessions.action)=open_graph_setting', 'buckets.LAST(users.affiliate_channel)=UNKNOWN', 'MODE(sessions.action)=campaigns', 'LAST(sessions.action)=cancellation_policies', 'LAST(sessions.action_detail)=account_notification_settings', 'buckets.CA', 'buckets.LAST(users.signup_flow)', 'buckets.LAST(sessions.device_type)=UNKNOWN', 'buckets.CUM_SUM(CA)', 'LAST(sessions.action_detail)=message_to_host_focus', 'MODE(sessions.action_detail)=change_trip_characteristics', 'buckets.SKEW(users.age)', 'MODE(sessions.action)=identity', 'buckets.LAST(sessions.secs_elapsed)', 'buckets.LAST(sessions.device_type)=iPad Tablet', 'buckets.MODE(sessions.action_detail)=UNKNOWN', 'age_gender_bucket=35-39female', 'first_device_type=Android Tablet', 'LAST(sessions.device_type)=Android Phone', 'LAST(sessions.action)=listings', 'buckets.MODE(users.signup_app)=UNKNOWN', 'buckets.MODE(users.first_device_type)=Mac Desktop']\n"
     ]
    }
   ],
   "source": [
    "# Get the union of all features\n",
    "all_selected_columns = list(set(chi_selected_columns + rfe_selected_columns + lr_selected_columns))\n",
    "print(\"TOTAL NUMBER OF FEATURES SELECTED: \", len(all_selected_columns))\n",
    "print(\"ALL SELECTED FEATURES: \", all_selected_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUMBER OF COMMON FEATURES SELECTED:  83\n",
      "COMMON SELECTED FEATURES:  ['MODE(sessions.action)=update', 'LAST(sessions.CUM_SUM(secs_elapsed))', 'MODE(sessions.action)=show', 'buckets.LAST(sessions.action_type)=view', 'buckets.LAST(users.affiliate_provider)=facebook', 'affiliate_provider=google', 'first_device_type=Windows Desktop', 'MAX(sessions.secs_elapsed)', 'MODE(sessions.action_type)=data', 'NUM_UNIQUE(sessions.action_type)', 'MODE(sessions.device_type)=iPhone', 'buckets.MODE(sessions.device_type)=Windows Desktop', 'first_device_type=iPhone', 'signup_method=basic', 'first_device_type=Mac Desktop', 'buckets.MIN(users.age)', 'buckets.MODE(users.first_affiliate_tracked)=omg', 'LAST(sessions.action_detail)=view_search_results', 'LAST(sessions.action_type)=data', 'buckets.MEDIAN(users.age)', 'MODE(sessions.device_type)=iPad Tablet', 'LAST(sessions.action_detail)=UNKNOWN', 'buckets.MODE(users.language)=en', 'age_gender_bucket=20-24female', 'first_browser=Firefox', 'affiliate_channel=sem-brand', 'buckets.LAST(users.signup_method)=basic', 'buckets.SKEW(sessions.secs_elapsed)', 'buckets.NUM_UNIQUE(sessions.action_type)', 'first_browser=IE', 'affiliate_provider=direct', 'first_affiliate_tracked=omg', 'MODE(sessions.action_detail)=view_search_results', 'MODE(sessions.action)=UNKNOWN', 'LAST(sessions.action)=UNKNOWN', 'buckets.LAST(sessions.action_type)=click', 'LAST(sessions.device_type)=iPhone', 'NUM_UNIQUE(sessions.device_type)', 'signup_method=facebook', 'signup_app=Web', 'MODE(sessions.action_detail)=p3', 'buckets.LAST(users.affiliate_channel)=seo', 'buckets.LAST(sessions.action_type)=UNKNOWN', 'signup_app=Android', 'buckets.LAST(users.first_affiliate_tracked)=untracked', 'LAST(sessions.action_type)=submit', 'MODE(sessions.action_detail)=update_listing', 'NUM_UNIQUE(sessions.action_detail)', 'LAST(sessions.action)=show', 'buckets.LAST(users.first_browser)=Chrome', 'affiliate_channel=sem-non-brand', 'first_browser=Mobile Safari', 'buckets.MODE(users.signup_method)=basic', 'buckets.MAX(users.age)', 'MODE(sessions.action_type)=UNKNOWN', 'first_device_type=Other/Unknown', 'LAST(sessions.action_type)=UNKNOWN', 'buckets.NUM_UNIQUE(users.language)', 'LAST(sessions.action)=update', 'YEAR(date_account_created)', 'MODE(sessions.action_type)=submit', 'buckets.MODE(users.signup_app)=Web', 'SKEW(sessions.secs_elapsed)', 'buckets.LAST(sessions.device_type)=Mac Desktop', 'affiliate_channel=content', 'first_device_type=iPad', 'affiliate_channel=direct', 'MODE(sessions.device_type)=UNKNOWN', 'MONTH(timestamp_first_active)', 'buckets.MODE(users.first_browser)=Chrome', 'buckets.LAST(sessions.device_type)=UNKNOWN', 'signup_app=iOS', 'buckets.MODE(sessions.action)=show', 'buckets.MODE(sessions.action_detail)=view_search_results', 'buckets.SKEW(users.age)', 'YEAR(timestamp_first_active)', 'buckets.MEAN(users.age)', 'first_browser=Chrome', 'buckets.LAST(users.signup_app)=Web', 'STD(sessions.CUM_SUM(secs_elapsed))', 'buckets.LAST(sessions.action_detail)=UNKNOWN', 'age', 'NUM_UNIQUE(sessions.action)']\n"
     ]
    }
   ],
   "source": [
    "# Get the intersection of all features\n",
    "common_selected_columns = list(set(chi_selected_columns).intersection(set(rfe_selected_columns), set(lr_selected_columns)))\n",
    "print(\"NUMBER OF COMMON FEATURES SELECTED: \", len(common_selected_columns))\n",
    "print(\"COMMON SELECTED FEATURES: \", common_selected_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The columns mentioned above are selected from the one-hot encoded representation of our training & test sets. We can convert them to base features corresponding to the original columns in our data when we read it in the beginning of this notebook. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUMBER OF BASE COLUMNS SELECTED:  53\n",
      "BASE COLUMNS SELECTED:  ['buckets.LAST(sessions.action_type)', 'age_gender_bucket', 'LAST(sessions.CUM_SUM(secs_elapsed))', 'buckets.MEDIAN(users.age)', 'buckets.MODE(sessions.action_detail)', 'signup_app', 'MODE(sessions.action_type)', 'buckets.LAST(sessions.action_detail)', 'buckets.LAST(users.signup_app)', 'buckets.LAST(users.affiliate_provider)', 'NUM_UNIQUE(sessions.action_detail)', 'buckets.MODE(users.signup_method)', 'MONTH(timestamp_first_active)', 'buckets.MODE(users.first_affiliate_tracked)', 'LAST(sessions.action_detail)', 'buckets.LAST(sessions.device_type)', 'first_device_type', 'MODE(sessions.action)', 'buckets.MODE(users.signup_app)', 'MAX(sessions.secs_elapsed)', 'buckets.SKEW(sessions.secs_elapsed)', 'buckets.NUM_UNIQUE(sessions.action_type)', 'NUM_UNIQUE(sessions.action_type)', 'signup_method', 'first_affiliate_tracked', 'affiliate_provider', 'buckets.MAX(users.age)', 'buckets.LAST(users.first_affiliate_tracked)', 'first_browser', 'buckets.SKEW(users.age)', 'MODE(sessions.device_type)', 'buckets.NUM_UNIQUE(users.language)', 'YEAR(timestamp_first_active)', 'buckets.LAST(users.signup_method)', 'LAST(sessions.device_type)', 'buckets.MODE(users.first_browser)', 'affiliate_channel', 'buckets.MODE(sessions.action)', 'MODE(sessions.action_detail)', 'buckets.MEAN(users.age)', 'NUM_UNIQUE(sessions.device_type)', 'buckets.MODE(sessions.device_type)', 'STD(sessions.CUM_SUM(secs_elapsed))', 'LAST(sessions.action)', 'age', 'YEAR(date_account_created)', 'NUM_UNIQUE(sessions.action)', 'buckets.LAST(users.first_browser)', 'LAST(sessions.action_type)', 'buckets.MODE(users.language)', 'buckets.MIN(users.age)', 'SKEW(sessions.secs_elapsed)', 'buckets.LAST(users.affiliate_channel)']\n"
     ]
    }
   ],
   "source": [
    "all_base_selected_columns = list(set([column.split('=')[0] for column in all_selected_columns]))\n",
    "common_base_selected_columns = list(set([column.split('=')[0] for column in common_selected_columns]))\n",
    "print(\"NUMBER OF BASE COLUMNS SELECTED: \", len(common_base_selected_columns))\n",
    "print(\"BASE COLUMNS SELECTED: \", common_base_selected_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's create the trimmed versions of our data frames."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buckets.LAST(sessions.action_type)</th>\n",
       "      <th>age_gender_bucket</th>\n",
       "      <th>LAST(sessions.CUM_SUM(secs_elapsed))</th>\n",
       "      <th>buckets.MEDIAN(users.age)</th>\n",
       "      <th>buckets.MODE(sessions.action_detail)</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>MODE(sessions.action_type)</th>\n",
       "      <th>buckets.LAST(sessions.action_detail)</th>\n",
       "      <th>buckets.LAST(users.signup_app)</th>\n",
       "      <th>buckets.LAST(users.affiliate_provider)</th>\n",
       "      <th>...</th>\n",
       "      <th>LAST(sessions.action)</th>\n",
       "      <th>age</th>\n",
       "      <th>YEAR(date_account_created)</th>\n",
       "      <th>NUM_UNIQUE(sessions.action)</th>\n",
       "      <th>buckets.LAST(users.first_browser)</th>\n",
       "      <th>LAST(sessions.action_type)</th>\n",
       "      <th>buckets.MODE(users.language)</th>\n",
       "      <th>buckets.MIN(users.age)</th>\n",
       "      <th>SKEW(sessions.secs_elapsed)</th>\n",
       "      <th>buckets.LAST(users.affiliate_channel)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>view</td>\n",
       "      <td>30-34female</td>\n",
       "      <td>0.577406</td>\n",
       "      <td>-0.556610</td>\n",
       "      <td>view_search_results</td>\n",
       "      <td>Web</td>\n",
       "      <td>view</td>\n",
       "      <td>p3</td>\n",
       "      <td>Web</td>\n",
       "      <td>google</td>\n",
       "      <td>...</td>\n",
       "      <td>personalize</td>\n",
       "      <td>-0.644729</td>\n",
       "      <td>1.039225</td>\n",
       "      <td>0.057369</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>data</td>\n",
       "      <td>en</td>\n",
       "      <td>-0.532341</td>\n",
       "      <td>0.501181</td>\n",
       "      <td>seo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>view</td>\n",
       "      <td>40-44female</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.362760</td>\n",
       "      <td>view_search_results</td>\n",
       "      <td>Web</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>user_profile</td>\n",
       "      <td>Web</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>0.504372</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>en</td>\n",
       "      <td>0.366522</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>direct</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>view</td>\n",
       "      <td>45-49female</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.699431</td>\n",
       "      <td>view_search_results</td>\n",
       "      <td>Web</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>message_thread</td>\n",
       "      <td>Web</td>\n",
       "      <td>direct</td>\n",
       "      <td>...</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>0.683121</td>\n",
       "      <td>-1.095002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Mobile Safari</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>en</td>\n",
       "      <td>0.815954</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>direct</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>40-44male</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.349811</td>\n",
       "      <td>view_search_results</td>\n",
       "      <td>Web</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>Web</td>\n",
       "      <td>google</td>\n",
       "      <td>...</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>0.159642</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Firefox</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>en</td>\n",
       "      <td>0.366522</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>sem-brand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>view</td>\n",
       "      <td>30-34female</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.556610</td>\n",
       "      <td>view_search_results</td>\n",
       "      <td>Moweb</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>p3</td>\n",
       "      <td>Web</td>\n",
       "      <td>google</td>\n",
       "      <td>...</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>-0.402141</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Chrome</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>en</td>\n",
       "      <td>-0.532341</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>seo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  buckets.LAST(sessions.action_type) age_gender_bucket  \\\n",
       "0                               view       30-34female   \n",
       "1                               view       40-44female   \n",
       "2                               view       45-49female   \n",
       "3                            UNKNOWN         40-44male   \n",
       "4                               view       30-34female   \n",
       "\n",
       "   LAST(sessions.CUM_SUM(secs_elapsed))  buckets.MEDIAN(users.age)  \\\n",
       "0                              0.577406                  -0.556610   \n",
       "1                              0.000000                   0.362760   \n",
       "2                              0.000000                   0.699431   \n",
       "3                              0.000000                   0.349811   \n",
       "4                              0.000000                  -0.556610   \n",
       "\n",
       "  buckets.MODE(sessions.action_detail) signup_app MODE(sessions.action_type)  \\\n",
       "0                  view_search_results        Web                       view   \n",
       "1                  view_search_results        Web                    UNKNOWN   \n",
       "2                  view_search_results        Web                    UNKNOWN   \n",
       "3                  view_search_results        Web                    UNKNOWN   \n",
       "4                  view_search_results      Moweb                    UNKNOWN   \n",
       "\n",
       "  buckets.LAST(sessions.action_detail) buckets.LAST(users.signup_app)  \\\n",
       "0                                   p3                            Web   \n",
       "1                         user_profile                            Web   \n",
       "2                       message_thread                            Web   \n",
       "3                              UNKNOWN                            Web   \n",
       "4                                   p3                            Web   \n",
       "\n",
       "  buckets.LAST(users.affiliate_provider)  \\\n",
       "0                                 google   \n",
       "1                                 direct   \n",
       "2                                 direct   \n",
       "3                                 google   \n",
       "4                                 google   \n",
       "\n",
       "                   ...                    LAST(sessions.action)       age  \\\n",
       "0                  ...                              personalize -0.644729   \n",
       "1                  ...                                  UNKNOWN  0.504372   \n",
       "2                  ...                                  UNKNOWN  0.683121   \n",
       "3                  ...                                  UNKNOWN  0.159642   \n",
       "4                  ...                                  UNKNOWN -0.402141   \n",
       "\n",
       "   YEAR(date_account_created) NUM_UNIQUE(sessions.action)  \\\n",
       "0                    1.039225                    0.057369   \n",
       "1                   -0.027888                    0.000000   \n",
       "2                   -1.095002                    0.000000   \n",
       "3                   -0.027888                    0.000000   \n",
       "4                   -0.027888                    0.000000   \n",
       "\n",
       "  buckets.LAST(users.first_browser) LAST(sessions.action_type)  \\\n",
       "0                            Chrome                       data   \n",
       "1                            Chrome                    UNKNOWN   \n",
       "2                     Mobile Safari                    UNKNOWN   \n",
       "3                           Firefox                    UNKNOWN   \n",
       "4                            Chrome                    UNKNOWN   \n",
       "\n",
       "  buckets.MODE(users.language) buckets.MIN(users.age)  \\\n",
       "0                           en              -0.532341   \n",
       "1                           en               0.366522   \n",
       "2                           en               0.815954   \n",
       "3                           en               0.366522   \n",
       "4                           en              -0.532341   \n",
       "\n",
       "  SKEW(sessions.secs_elapsed)  buckets.LAST(users.affiliate_channel)  \n",
       "0                    0.501181                                    seo  \n",
       "1                    0.000000                                 direct  \n",
       "2                    0.000000                                 direct  \n",
       "3                    0.000000                              sem-brand  \n",
       "4                    0.000000                                    seo  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_trimmed = X_train[common_base_selected_columns]\n",
    "X_test_trimmed = X_test[common_base_selected_columns]\n",
    "assert X_train_trimmed.shape[1] == X_test_trimmed.shape[1]\n",
    "X_train_trimmed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Progress with Raw & Trimmed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add back the response variable to sets & save progress\n",
    "X_train_trimmed.loc[:, 'country_destination'] = Y_train.loc[:]\n",
    "X_train_trimmed.to_csv('(3)data_trimmed/raw/train_users.csv', index=None)\n",
    "X_test_trimmed.to_csv('(3)data_trimmed/raw/test_users.csv', index=None)\n",
    "# Drop response variable again, as we will continue processing features\n",
    "X_train_trimmed.drop('country_destination', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoding Categorical Variables\n",
    "Although we had chosen **one-hot encoding** as our strategy previously, those encodings were solely used for the purposes of feature selection and abondoned afterwards. Besides, there exists other and more efficient strategies in the literature that can be explored:\n",
    "\n",
    "* **Labeled Encoding**: Interprets categories as ordered integers. The extracted ordinality is almost always wrong, hence this is not really a preferred method.\n",
    "* **Frequency Encoding**: Encodes categorical levels of each feature to values between 0.0 and 1.0 based on their relative frequincy. This method especially works when there is a high number of categorical levels that are somewhat imbalanced in distribution.\n",
    "* **Target Mean Encoding**: Encodes categorical levels of each feature to the mean of the response. This method works best with binary classification, but it often yields *data leakage*.\n",
    "\n",
    "It should be noted that one-hot encoding variables increases the feature space vastly, and this may decrease the potential performance of the *tree-based models*. This is why we don't want to use this type of encoding to get our final data form. Instead, we have chosen to go with labeled encoding as an experiment. As previously discussed, `h2o`'s automated modelling method applies different kind of encodings, but labeled encoding was one strategy that wasn't automatically applied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Currently encoding column:  buckets.LAST(sessions.action_type)\n",
      "Currently encoding column:  age_gender_bucket\n",
      "Currently encoding column:  buckets.MODE(sessions.action_detail)\n",
      "Currently encoding column:  signup_app\n",
      "Currently encoding column:  MODE(sessions.action_type)\n",
      "Currently encoding column:  buckets.LAST(sessions.action_detail)\n",
      "Currently encoding column:  buckets.LAST(users.signup_app)\n",
      "Currently encoding column:  buckets.LAST(users.affiliate_provider)\n",
      "Currently encoding column:  buckets.MODE(users.signup_method)\n",
      "Currently encoding column:  buckets.MODE(users.first_affiliate_tracked)\n",
      "Currently encoding column:  LAST(sessions.action_detail)\n",
      "Currently encoding column:  buckets.LAST(sessions.device_type)\n",
      "Currently encoding column:  first_device_type\n",
      "Currently encoding column:  MODE(sessions.action)\n",
      "Currently encoding column:  buckets.MODE(users.signup_app)\n",
      "Currently encoding column:  signup_method\n",
      "Currently encoding column:  first_affiliate_tracked\n",
      "Currently encoding column:  affiliate_provider\n",
      "Currently encoding column:  buckets.LAST(users.first_affiliate_tracked)\n",
      "Currently encoding column:  first_browser\n",
      "Currently encoding column:  MODE(sessions.device_type)\n",
      "Currently encoding column:  buckets.LAST(users.signup_method)\n",
      "Currently encoding column:  LAST(sessions.device_type)\n",
      "Currently encoding column:  buckets.MODE(users.first_browser)\n",
      "Currently encoding column:  affiliate_channel\n",
      "Currently encoding column:  buckets.MODE(sessions.action)\n",
      "Currently encoding column:  MODE(sessions.action_detail)\n",
      "Currently encoding column:  buckets.MODE(sessions.device_type)\n",
      "Currently encoding column:  LAST(sessions.action)\n",
      "Currently encoding column:  buckets.LAST(users.first_browser)\n",
      "Currently encoding column:  LAST(sessions.action_type)\n",
      "Currently encoding column:  buckets.MODE(users.language)\n",
      "Currently encoding column:  buckets.LAST(users.affiliate_channel)\n"
     ]
    }
   ],
   "source": [
    "remaining_categorical_vars = []\n",
    "for column in X_train_trimmed.columns.values.tolist():\n",
    "    if not is_numeric_dtype(X_train_trimmed[column]):\n",
    "        print(\"Currently encoding column: \", column)\n",
    "        remaining_categorical_vars.append(column)\n",
    "        \n",
    "        encoder = LabelEncoder()\n",
    "        encoder.fit(X_train_trimmed[column])\n",
    "        available_levels = list(encoder.classes_)\n",
    "        for test_level in set(X_test_trimmed[column].values.tolist()):\n",
    "            if test_level not in available_levels:\n",
    "                X_test_trimmed.loc[X_test_trimmed[column] == test_level, column] = X_train_trimmed[column].mode()[0]\n",
    "        \n",
    "        X_train_trimmed.loc[:, column] = encoder.transform(X_train_trimmed[column])\n",
    "        X_test_trimmed.loc[:, column] = encoder.transform(X_test_trimmed[column])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buckets.LAST(sessions.action_type)</th>\n",
       "      <th>age_gender_bucket</th>\n",
       "      <th>LAST(sessions.CUM_SUM(secs_elapsed))</th>\n",
       "      <th>buckets.MEDIAN(users.age)</th>\n",
       "      <th>buckets.MODE(sessions.action_detail)</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>MODE(sessions.action_type)</th>\n",
       "      <th>buckets.LAST(sessions.action_detail)</th>\n",
       "      <th>buckets.LAST(users.signup_app)</th>\n",
       "      <th>buckets.LAST(users.affiliate_provider)</th>\n",
       "      <th>...</th>\n",
       "      <th>LAST(sessions.action)</th>\n",
       "      <th>age</th>\n",
       "      <th>YEAR(date_account_created)</th>\n",
       "      <th>NUM_UNIQUE(sessions.action)</th>\n",
       "      <th>buckets.LAST(users.first_browser)</th>\n",
       "      <th>LAST(sessions.action_type)</th>\n",
       "      <th>buckets.MODE(users.language)</th>\n",
       "      <th>buckets.MIN(users.age)</th>\n",
       "      <th>SKEW(sessions.secs_elapsed)</th>\n",
       "      <th>buckets.LAST(users.affiliate_channel)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>0.577406</td>\n",
       "      <td>-0.556610</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>143</td>\n",
       "      <td>-0.644729</td>\n",
       "      <td>1.039225</td>\n",
       "      <td>0.057369</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.532341</td>\n",
       "      <td>0.501181</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.362760</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.504372</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.366522</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.699431</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.683121</td>\n",
       "      <td>-1.095002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.815954</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.349811</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.159642</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.366522</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.556610</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.402141</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.532341</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   buckets.LAST(sessions.action_type)  age_gender_bucket  \\\n",
       "0                                   6                 11   \n",
       "1                                   6                 17   \n",
       "2                                   6                 20   \n",
       "3                                   0                 18   \n",
       "4                                   6                 11   \n",
       "\n",
       "   LAST(sessions.CUM_SUM(secs_elapsed))  buckets.MEDIAN(users.age)  \\\n",
       "0                              0.577406                  -0.556610   \n",
       "1                              0.000000                   0.362760   \n",
       "2                              0.000000                   0.699431   \n",
       "3                              0.000000                   0.349811   \n",
       "4                              0.000000                  -0.556610   \n",
       "\n",
       "   buckets.MODE(sessions.action_detail)  signup_app  \\\n",
       "0                                     3           2   \n",
       "1                                     3           2   \n",
       "2                                     3           2   \n",
       "3                                     3           2   \n",
       "4                                     3           1   \n",
       "\n",
       "   MODE(sessions.action_type)  buckets.LAST(sessions.action_detail)  \\\n",
       "0                           7                                     7   \n",
       "1                           0                                    14   \n",
       "2                           0                                     6   \n",
       "3                           0                                     0   \n",
       "4                           0                                     7   \n",
       "\n",
       "   buckets.LAST(users.signup_app)  buckets.LAST(users.affiliate_provider)  \\\n",
       "0                               2                                       4   \n",
       "1                               2                                       2   \n",
       "2                               2                                       2   \n",
       "3                               2                                       4   \n",
       "4                               2                                       4   \n",
       "\n",
       "                   ...                    LAST(sessions.action)       age  \\\n",
       "0                  ...                                      143 -0.644729   \n",
       "1                  ...                                        4  0.504372   \n",
       "2                  ...                                        4  0.683121   \n",
       "3                  ...                                        4  0.159642   \n",
       "4                  ...                                        4 -0.402141   \n",
       "\n",
       "   YEAR(date_account_created)  NUM_UNIQUE(sessions.action)  \\\n",
       "0                    1.039225                     0.057369   \n",
       "1                   -0.027888                     0.000000   \n",
       "2                   -1.095002                     0.000000   \n",
       "3                   -0.027888                     0.000000   \n",
       "4                   -0.027888                     0.000000   \n",
       "\n",
       "   buckets.LAST(users.first_browser)  LAST(sessions.action_type)  \\\n",
       "0                                  1                           3   \n",
       "1                                  1                           0   \n",
       "2                                  4                           0   \n",
       "3                                  2                           0   \n",
       "4                                  1                           0   \n",
       "\n",
       "   buckets.MODE(users.language)  buckets.MIN(users.age)  \\\n",
       "0                             1               -0.532341   \n",
       "1                             1                0.366522   \n",
       "2                             1                0.815954   \n",
       "3                             1                0.366522   \n",
       "4                             1               -0.532341   \n",
       "\n",
       "   SKEW(sessions.secs_elapsed)  buckets.LAST(users.affiliate_channel)  \n",
       "0                     0.501181                                      6  \n",
       "1                     0.000000                                      2  \n",
       "2                     0.000000                                      2  \n",
       "3                     0.000000                                      4  \n",
       "4                     0.000000                                      6  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_trimmed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Scaling for Remaining Categorical Variables\n",
    "\n",
    "After this step, both our training & test sets are fully scaled and ready for being passed on to predictive models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buckets.LAST(sessions.action_type)</th>\n",
       "      <th>age_gender_bucket</th>\n",
       "      <th>LAST(sessions.CUM_SUM(secs_elapsed))</th>\n",
       "      <th>buckets.MEDIAN(users.age)</th>\n",
       "      <th>buckets.MODE(sessions.action_detail)</th>\n",
       "      <th>signup_app</th>\n",
       "      <th>MODE(sessions.action_type)</th>\n",
       "      <th>buckets.LAST(sessions.action_detail)</th>\n",
       "      <th>buckets.LAST(users.signup_app)</th>\n",
       "      <th>buckets.LAST(users.affiliate_provider)</th>\n",
       "      <th>...</th>\n",
       "      <th>LAST(sessions.action)</th>\n",
       "      <th>age</th>\n",
       "      <th>YEAR(date_account_created)</th>\n",
       "      <th>NUM_UNIQUE(sessions.action)</th>\n",
       "      <th>buckets.LAST(users.first_browser)</th>\n",
       "      <th>LAST(sessions.action_type)</th>\n",
       "      <th>buckets.MODE(users.language)</th>\n",
       "      <th>buckets.MIN(users.age)</th>\n",
       "      <th>SKEW(sessions.secs_elapsed)</th>\n",
       "      <th>buckets.LAST(users.affiliate_channel)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.673844</td>\n",
       "      <td>-0.611161</td>\n",
       "      <td>0.577406</td>\n",
       "      <td>-0.556610</td>\n",
       "      <td>0.066229</td>\n",
       "      <td>-0.018717</td>\n",
       "      <td>2.199263</td>\n",
       "      <td>-0.569216</td>\n",
       "      <td>-0.380246</td>\n",
       "      <td>1.331396</td>\n",
       "      <td>...</td>\n",
       "      <td>1.404108</td>\n",
       "      <td>-0.644729</td>\n",
       "      <td>1.039225</td>\n",
       "      <td>0.057369</td>\n",
       "      <td>-0.803753</td>\n",
       "      <td>0.743575</td>\n",
       "      <td>0.03625</td>\n",
       "      <td>-0.532341</td>\n",
       "      <td>0.501181</td>\n",
       "      <td>2.129106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.673844</td>\n",
       "      <td>0.406107</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.362760</td>\n",
       "      <td>0.066229</td>\n",
       "      <td>-0.018717</td>\n",
       "      <td>-0.575552</td>\n",
       "      <td>0.773677</td>\n",
       "      <td>-0.380246</td>\n",
       "      <td>-0.697524</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.599231</td>\n",
       "      <td>0.504372</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.803753</td>\n",
       "      <td>-0.540773</td>\n",
       "      <td>0.03625</td>\n",
       "      <td>0.366522</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.646366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.673844</td>\n",
       "      <td>0.914741</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.699431</td>\n",
       "      <td>0.066229</td>\n",
       "      <td>-0.018717</td>\n",
       "      <td>-0.575552</td>\n",
       "      <td>-0.761058</td>\n",
       "      <td>-0.380246</td>\n",
       "      <td>-0.697524</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.599231</td>\n",
       "      <td>0.683121</td>\n",
       "      <td>-1.095002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.100618</td>\n",
       "      <td>-0.540773</td>\n",
       "      <td>0.03625</td>\n",
       "      <td>0.815954</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.646366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2.192308</td>\n",
       "      <td>0.575652</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.349811</td>\n",
       "      <td>0.066229</td>\n",
       "      <td>-0.018717</td>\n",
       "      <td>-0.575552</td>\n",
       "      <td>-1.912109</td>\n",
       "      <td>-0.380246</td>\n",
       "      <td>1.331396</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.599231</td>\n",
       "      <td>0.159642</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.168962</td>\n",
       "      <td>-0.540773</td>\n",
       "      <td>0.03625</td>\n",
       "      <td>0.366522</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.741370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.673844</td>\n",
       "      <td>-0.611161</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.556610</td>\n",
       "      <td>0.066229</td>\n",
       "      <td>-2.145454</td>\n",
       "      <td>-0.575552</td>\n",
       "      <td>-0.569216</td>\n",
       "      <td>-0.380246</td>\n",
       "      <td>1.331396</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.599231</td>\n",
       "      <td>-0.402141</td>\n",
       "      <td>-0.027888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.803753</td>\n",
       "      <td>-0.540773</td>\n",
       "      <td>0.03625</td>\n",
       "      <td>-0.532341</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.129106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   buckets.LAST(sessions.action_type)  age_gender_bucket  \\\n",
       "0                            0.673844          -0.611161   \n",
       "1                            0.673844           0.406107   \n",
       "2                            0.673844           0.914741   \n",
       "3                           -2.192308           0.575652   \n",
       "4                            0.673844          -0.611161   \n",
       "\n",
       "   LAST(sessions.CUM_SUM(secs_elapsed))  buckets.MEDIAN(users.age)  \\\n",
       "0                              0.577406                  -0.556610   \n",
       "1                              0.000000                   0.362760   \n",
       "2                              0.000000                   0.699431   \n",
       "3                              0.000000                   0.349811   \n",
       "4                              0.000000                  -0.556610   \n",
       "\n",
       "   buckets.MODE(sessions.action_detail)  signup_app  \\\n",
       "0                              0.066229   -0.018717   \n",
       "1                              0.066229   -0.018717   \n",
       "2                              0.066229   -0.018717   \n",
       "3                              0.066229   -0.018717   \n",
       "4                              0.066229   -2.145454   \n",
       "\n",
       "   MODE(sessions.action_type)  buckets.LAST(sessions.action_detail)  \\\n",
       "0                    2.199263                             -0.569216   \n",
       "1                   -0.575552                              0.773677   \n",
       "2                   -0.575552                             -0.761058   \n",
       "3                   -0.575552                             -1.912109   \n",
       "4                   -0.575552                             -0.569216   \n",
       "\n",
       "   buckets.LAST(users.signup_app)  buckets.LAST(users.affiliate_provider)  \\\n",
       "0                       -0.380246                                1.331396   \n",
       "1                       -0.380246                               -0.697524   \n",
       "2                       -0.380246                               -0.697524   \n",
       "3                       -0.380246                                1.331396   \n",
       "4                       -0.380246                                1.331396   \n",
       "\n",
       "                   ...                    LAST(sessions.action)       age  \\\n",
       "0                  ...                                 1.404108 -0.644729   \n",
       "1                  ...                                -0.599231  0.504372   \n",
       "2                  ...                                -0.599231  0.683121   \n",
       "3                  ...                                -0.599231  0.159642   \n",
       "4                  ...                                -0.599231 -0.402141   \n",
       "\n",
       "   YEAR(date_account_created)  NUM_UNIQUE(sessions.action)  \\\n",
       "0                    1.039225                     0.057369   \n",
       "1                   -0.027888                     0.000000   \n",
       "2                   -1.095002                     0.000000   \n",
       "3                   -0.027888                     0.000000   \n",
       "4                   -0.027888                     0.000000   \n",
       "\n",
       "   buckets.LAST(users.first_browser)  LAST(sessions.action_type)  \\\n",
       "0                          -0.803753                    0.743575   \n",
       "1                          -0.803753                   -0.540773   \n",
       "2                           1.100618                   -0.540773   \n",
       "3                          -0.168962                   -0.540773   \n",
       "4                          -0.803753                   -0.540773   \n",
       "\n",
       "   buckets.MODE(users.language)  buckets.MIN(users.age)  \\\n",
       "0                       0.03625               -0.532341   \n",
       "1                       0.03625                0.366522   \n",
       "2                       0.03625                0.815954   \n",
       "3                       0.03625                0.366522   \n",
       "4                       0.03625               -0.532341   \n",
       "\n",
       "   SKEW(sessions.secs_elapsed)  buckets.LAST(users.affiliate_channel)  \n",
       "0                     0.501181                               2.129106  \n",
       "1                     0.000000                              -0.646366  \n",
       "2                     0.000000                              -0.646366  \n",
       "3                     0.000000                               0.741370  \n",
       "4                     0.000000                               2.129106  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = StandardScaler()\n",
    "scaler = scaler.fit(X_train_trimmed[remaining_categorical_vars])\n",
    "X_train_trimmed.loc[:, remaining_categorical_vars] = scaler.transform(X_train_trimmed.loc[:, remaining_categorical_vars])\n",
    "X_test_trimmed.loc[:, remaining_categorical_vars] = scaler.transform(X_test_trimmed.loc[:, remaining_categorical_vars])\n",
    "\n",
    "X_train_trimmed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Progress with Label Encoded & Trimmed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add back the response variable to sets & save progress\n",
    "X_train_trimmed.loc[:, 'country_destination'] = Y_train.loc[:]\n",
    "X_train_trimmed.to_csv('(3)data_trimmed/label_encoded/train_users.csv', index=None)\n",
    "X_test_trimmed.to_csv('(3)data_trimmed/label_encoded/test_users.csv', index=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}