Another Approach.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In this approach, first we are going to build a classification model to predict the gender. Then we are going to split the data in two groups, Males and Females according to the predictions. Then we are going to build seperate neural networks to predict the probabilites of Males group and Females Group"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from scipy.sparse import csr_matrix, hstack\n",
    "from sklearn.preprocessing import StandardScaler,MinMaxScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.metrics.classification import log_loss\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.externals import joblib as jobl\n",
    "from tqdm import tqdm\n",
    "from joblib import dump\n",
    "from scipy import sparse\n",
    "import numpy as np\n",
    "from scipy.integrate import simps\n",
    "from numpy import trapz\n",
    "from tqdm import tqdm\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading the data\n",
    "train_data = pd.read_csv('Data/gender_age_train.csv',index_col='device_id')\n",
    "test_data = pd.read_csv('Data/gender_age_test.csv',index_col='device_id')\n",
    "phone_data = pd.read_csv('Data/phone_brand_device_model.csv',encoding='utf-8')\n",
    "# Get rid of duplicate device ids in phone\n",
    "phone_data = phone_data.drop_duplicates('device_id',keep='first').set_index('device_id') \n",
    "label_categories = pd.read_csv('Data/label_categories.csv')\n",
    "app_labels = pd.read_csv('Data/app_labels.csv')\n",
    "events = pd.read_csv('Data/events.csv',parse_dates=['timestamp'], index_col='event_id')\n",
    "app_events = pd.read_csv('Data/app_events.csv',usecols=['event_id','app_id','is_active'],dtype={'is_active':bool})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['trainrow'] = np.arange(train_data.shape[0])\n",
    "test_data['testrow'] = np.arange(test_data.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One Hot Encoding of Phone Brands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding the brands using LabelEncoder\n",
    "# We could have used OneHotEncoder Directly too but OneHotEncoder Transfrom function returns a sparse matrix \n",
    "# and adding that to the dataframe (as we had to add this column to the train data) would had un-necessarily increased the space\n",
    "enc_brand = LabelEncoder().fit(phone_data['phone_brand'])\n",
    "phone_data['enc_brand'] = enc_brand.transform(phone_data['phone_brand'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As both of these dataframes have the same indices, the encoded brands will get assigned to the specific device_ids\n",
    "train_data['brand'] = phone_data['enc_brand']\n",
    "test_data['brand'] = phone_data['enc_brand']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of Xtr_brand_no_events is (74645, 131) the shape of Xte_brand_no_events is (112071, 131)\n"
     ]
    }
   ],
   "source": [
    "# Getting the csr matrix from the encoded labels\n",
    "Xtr_brand = csr_matrix((np.ones(train_data.shape[0]), \n",
    "                       (train_data.trainrow, train_data.brand)))\n",
    "Xte_brand = csr_matrix((np.ones(test_data.shape[0]), \n",
    "                       (test_data.testrow, test_data.brand)))\n",
    "print('The shape of Xtr_brand_no_events is', Xtr_brand.shape, 'the shape of Xte_brand_no_events is', Xte_brand.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One Hot Encoding of Device Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding the device models using LabelEncoder same way as done for Brands\n",
    "enc_model = LabelEncoder().fit(phone_data['device_model'])\n",
    "phone_data['enc_model'] = enc_model.transform(phone_data['device_model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As both of these dataframes have the same indices, the encoded brands will get assigned to the specific device_ids\n",
    "train_data['model'] = phone_data['enc_model']\n",
    "test_data['model'] = phone_data['enc_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of Xtr_model_data is (74645, 1599) the shape of Xte_model_data is (112071, 1599)\n"
     ]
    }
   ],
   "source": [
    "# Getting the csr matrix from the encoded labels\n",
    "Xtr_model = csr_matrix((np.ones(train_data.shape[0]), \n",
    "                       (train_data.trainrow, train_data.model)))\n",
    "Xte_model = csr_matrix((np.ones(test_data.shape[0]), \n",
    "                       (test_data.testrow, test_data.model)))\n",
    "print('The shape of Xtr_model_data is', Xtr_model.shape, 'the shape of Xte_model_data is', Xte_model.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One Hot Encoding for app_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>device_id</th>\n",
       "      <th>enc_app</th>\n",
       "      <th>size</th>\n",
       "      <th>trainrow</th>\n",
       "      <th>testrow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>548</td>\n",
       "      <td>18</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>1096</td>\n",
       "      <td>18</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>1248</td>\n",
       "      <td>26</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>1545</td>\n",
       "      <td>12</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>1664</td>\n",
       "      <td>18</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2369020</td>\n",
       "      <td>9222539910510672930</td>\n",
       "      <td>17358</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>82667.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2369021</td>\n",
       "      <td>9222539910510672930</td>\n",
       "      <td>17587</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>82667.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2369022</td>\n",
       "      <td>9222539910510672930</td>\n",
       "      <td>18039</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>82667.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2369023</td>\n",
       "      <td>9222539910510672930</td>\n",
       "      <td>18686</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>82667.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2369024</td>\n",
       "      <td>9222539910510672930</td>\n",
       "      <td>18837</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>82667.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2369025 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   device_id  enc_app  size  trainrow  testrow\n",
       "0       -9222956879900151005      548    18   21594.0      NaN\n",
       "1       -9222956879900151005     1096    18   21594.0      NaN\n",
       "2       -9222956879900151005     1248    26   21594.0      NaN\n",
       "3       -9222956879900151005     1545    12   21594.0      NaN\n",
       "4       -9222956879900151005     1664    18   21594.0      NaN\n",
       "...                      ...      ...   ...       ...      ...\n",
       "2369020  9222539910510672930    17358     1       NaN  82667.0\n",
       "2369021  9222539910510672930    17587     1       NaN  82667.0\n",
       "2369022  9222539910510672930    18039     1       NaN  82667.0\n",
       "2369023  9222539910510672930    18686     1       NaN  82667.0\n",
       "2369024  9222539910510672930    18837     1       NaN  82667.0\n",
       "\n",
       "[2369025 rows x 5 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For each device, we have to mark the apps that are installed in that device.  \n",
    "# First, we will encode all the app_ids to integers using LabelEncoder. \n",
    "enc_apps = LabelEncoder().fit(app_events['app_id'])\n",
    "app_events['enc_app'] = enc_apps.transform(app_events['app_id'])\n",
    "\n",
    "# Then we merge the app_events with device_id column from the events dataframe grouping by device_id and encoded app_ids.\n",
    "# Finally we merge it with trainrow and testrow columns to understand at which row we should put each device in features matrix.\n",
    "deviceapps = (app_events.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)\n",
    "                       .groupby(['device_id','enc_app'])['enc_app'].agg(['size'])\n",
    "                       .merge(train_data[['trainrow']], how='left', left_index=True, right_index=True)\n",
    "                       .merge(test_data[['testrow']], how='left', left_index=True, right_index=True)\n",
    "                       .reset_index())\n",
    "deviceapps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Apps data: train shape (74645, 19237), test shape (112071, 19237)\n"
     ]
    }
   ],
   "source": [
    "# Getting the number of classes\n",
    "napps = len(enc_apps.classes_)\n",
    "\n",
    "# Getting the csr matrix for Apps\n",
    "d = deviceapps.dropna(subset=['trainrow'])\n",
    "Xtr_app = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.enc_app)), \n",
    "                      shape=(train_data.shape[0],napps))\n",
    "d = deviceapps.dropna(subset=['testrow'])\n",
    "Xte_app = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.enc_app)), \n",
    "                      shape=(test_data.shape[0],napps))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One Hot Encoding for app_labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_labels = app_labels.loc[app_labels.app_id.isin(app_events.app_id.unique())]\n",
    "app_labels['enc_app'] = enc_apps.transform(app_labels.app_id)\n",
    "\n",
    "# Encoding the labels\n",
    "enc_labels = LabelEncoder().fit(app_labels['label_id'])\n",
    "app_labels['enc_label'] = enc_labels.transform(app_labels['label_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>device_id</th>\n",
       "      <th>enc_label</th>\n",
       "      <th>size</th>\n",
       "      <th>trainrow</th>\n",
       "      <th>testrow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>117</td>\n",
       "      <td>1</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>120</td>\n",
       "      <td>1</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>126</td>\n",
       "      <td>1</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>138</td>\n",
       "      <td>2</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>147</td>\n",
       "      <td>2</td>\n",
       "      <td>21594.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             device_id  enc_label  size  trainrow  testrow\n",
       "0 -9222956879900151005        117     1   21594.0      NaN\n",
       "1 -9222956879900151005        120     1   21594.0      NaN\n",
       "2 -9222956879900151005        126     1   21594.0      NaN\n",
       "3 -9222956879900151005        138     2   21594.0      NaN\n",
       "4 -9222956879900151005        147     2   21594.0      NaN"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlabels = len(enc_labels.classes_)\n",
    "\n",
    "devicelabels = (deviceapps[['device_id','enc_app']]\n",
    "                .merge(app_labels[['enc_app','enc_label']])\n",
    "                .groupby(['device_id','enc_label'])['enc_app'].agg(['size'])\n",
    "                .merge(train_data[['trainrow']], how='left', left_index=True, right_index=True)\n",
    "                .merge(test_data[['testrow']], how='left', left_index=True, right_index=True)\n",
    "                .reset_index())\n",
    "\n",
    "devicelabels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels data: train shape (74645, 492), test shape (112071, 492)\n"
     ]
    }
   ],
   "source": [
    "d = devicelabels.dropna(subset=['trainrow'])\n",
    "Xtr_label = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.enc_label)), \n",
    "                      shape=(train_data.shape[0],nlabels))\n",
    "d = devicelabels.dropna(subset=['testrow'])\n",
    "Xte_label = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.enc_label)), \n",
    "                      shape=(test_data.shape[0],nlabels))\n",
    "print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Events"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HourBins Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binhour(x):\n",
    "    '''\n",
    "    This function returns the phase of the day at which the event occurred.\n",
    "    So it will be easier to count the number of events that occurred during a specific phase of the day.\n",
    "    '''\n",
    "    if x < 5:\n",
    "        return \"midnight\"\n",
    "    elif x < 8:\n",
    "        return \"early_morning\"\n",
    "    elif x < 20:\n",
    "        return \"daytime\"\n",
    "    elif x< 23:\n",
    "        return \"night\"\n",
    "    else:\n",
    "        return \"midnight\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Timestamp.html\n",
    "# Extracting the hour at which a specific event occurred to put it in phase\n",
    "events['hour'] = events['timestamp'].apply(lambda x : x.hour)\n",
    "# Appling the above function to get the phase\n",
    "events['hour_bin'] = events['hour'].apply(binhour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Grouping the data by device_id and getting the information of all the phases at which the event of that device occurred\n",
    "l_hours_cnt = events.groupby('device_id')['hour_bin'].apply(lambda x: \" \".join(s for s in x)).reset_index()\n",
    "\n",
    "midnight_counts = []\n",
    "daytime_counts = []\n",
    "early_morning_counts = []\n",
    "night_counts = []\n",
    "\n",
    "# Counting the events which occurred at the specific phases from l_hours_count\n",
    "for i in range(len(l_hours_cnt)):\n",
    "    lis = l_hours_cnt['hour_bin'][i].split(' ')\n",
    "    midnight_counts.append(lis.count('midnight'))\n",
    "    daytime_counts.append(lis.count('daytime'))\n",
    "    early_morning_counts.append(lis.count('early_morning'))\n",
    "    night_counts.append(lis.count('night'))\n",
    "\n",
    "# Adding the counts column in the data\n",
    "l_hours_cnt['midnight_counts'] = midnight_counts\n",
    "l_hours_cnt['daytime_counts'] = daytime_counts\n",
    "l_hours_cnt['early_morning_counts'] = early_morning_counts\n",
    "l_hours_cnt['night_counts'] = night_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>device_id</th>\n",
       "      <th>hour_bin</th>\n",
       "      <th>midnight_counts</th>\n",
       "      <th>daytime_counts</th>\n",
       "      <th>early_morning_counts</th>\n",
       "      <th>night_counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>daytime daytime daytime daytime daytime night ...</td>\n",
       "      <td>3</td>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-9222661944218806987</td>\n",
       "      <td>night daytime night daytime daytime daytime mi...</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-9222399302879214035</td>\n",
       "      <td>daytime daytime midnight night daytime midnigh...</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-9221825537663503111</td>\n",
       "      <td>early_morning early_morning early_morning dayt...</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-9221767098072603291</td>\n",
       "      <td>early_morning daytime daytime daytime daytime ...</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             device_id                                           hour_bin  \\\n",
       "0 -9222956879900151005  daytime daytime daytime daytime daytime night ...   \n",
       "1 -9222661944218806987  night daytime night daytime daytime daytime mi...   \n",
       "2 -9222399302879214035  daytime daytime midnight night daytime midnigh...   \n",
       "3 -9221825537663503111  early_morning early_morning early_morning dayt...   \n",
       "4 -9221767098072603291  early_morning daytime daytime daytime daytime ...   \n",
       "\n",
       "   midnight_counts  daytime_counts  early_morning_counts  night_counts  \n",
       "0                3              50                     2            10  \n",
       "1                1               5                     0             2  \n",
       "2                3               6                     0             1  \n",
       "3                0              73                    22             4  \n",
       "4                0               5                     3             0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l_hours_cnt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "hourevents = events.groupby(\"device_id\")[\"hour\"].apply(lambda x: \" \".join('0'+str(s) for s in x))\n",
    "hourevents = hourevents.reset_index().set_index('device_id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DaysOfTheWeek Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def day_of_week(x):\n",
    "    '''\n",
    "    This function returns the day of the week at which the event occurred.\n",
    "    So it will be easier to count the number of events that occurred during a specific day of the day.\n",
    "    '''  \n",
    "    if(x==0):\n",
    "        return \"Monday\"\n",
    "    elif(x==1):\n",
    "        return \"Tuesday\"\n",
    "    elif(x==2):\n",
    "        return \"Wednesday\"\n",
    "    elif(x==3):\n",
    "        return \"Thursday\"\n",
    "    elif(x==4):\n",
    "        return \"Friday\"\n",
    "    elif(x==5):\n",
    "        return \"Saturday\"\n",
    "    elif(x==6):\n",
    "        return \"Sunday\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the day at which a specific event occurred from the timestamp object\n",
    "events['dayofweek'] = events['timestamp'].apply(lambda x : x.dayofweek)\n",
    "# Applying the function to get the day in words\n",
    "events['day'] = events['dayofweek'].apply(day_of_week)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Grouping events data according to device_ids and getting all the days at which that particular device performed that events\n",
    "l_days_cnt = events.groupby('device_id')['day'].apply(lambda x: \" \".join(s for s in x)).reset_index()\n",
    "\n",
    "monday_counts = []\n",
    "tuesday_counts = []\n",
    "wednesday_counts = []\n",
    "thursday_counts = []\n",
    "friday_counts = []\n",
    "saturday_counts = []\n",
    "sunday_counts = []\n",
    "\n",
    "# Getting the counts of events that occurred on a specific data\n",
    "for i in range(len(l_hours_cnt)):\n",
    "    lis = l_days_cnt['day'][i].split(' ')\n",
    "    monday_counts.append(lis.count('Monday'))\n",
    "    tuesday_counts.append(lis.count('Tuesday'))\n",
    "    wednesday_counts.append(lis.count('Wednesday'))\n",
    "    thursday_counts.append(lis.count('Thursday'))\n",
    "    friday_counts.append(lis.count('Friday'))\n",
    "    saturday_counts.append(lis.count('Saturday'))\n",
    "    sunday_counts.append(lis.count('Sunday'))\n",
    "\n",
    "# Adding the counts column in the dataframe\n",
    "l_days_cnt['monday_counts'] = monday_counts\n",
    "l_days_cnt['tuesday_counts'] = tuesday_counts\n",
    "l_days_cnt['wednesday_counts'] = wednesday_counts\n",
    "l_days_cnt['thursday_counts'] = thursday_counts\n",
    "l_days_cnt['friday_counts'] = friday_counts\n",
    "l_days_cnt['saturday_counts'] = saturday_counts\n",
    "l_days_cnt['sunday_counts'] = sunday_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting the index as device is so it will be easier to merge the data\n",
    "l_days_cnt = l_days_cnt.set_index('device_id')\n",
    "l_hours_cnt = l_hours_cnt.set_index('device_id')\n",
    "events = events.set_index('device_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merging all the data to get the counts that were extracted above\n",
    "events_count = events.merge(l_days_cnt, on='device_id').\\\n",
    "                merge(l_hours_cnt, on='device_id').\\\n",
    "                drop(['timestamp','longitude','latitude','hour','dayofweek','day_x','day_y','hour_bin_x','hour_bin_y'],axis=1).\\\n",
    "                drop_duplicates().\\\n",
    "                merge(train_data[['trainrow']], how='left', left_index=True, right_index=True).\\\n",
    "                merge(test_data[['testrow']], how='left', left_index=True, right_index=True).\\\n",
    "                reset_index().set_index('device_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>monday_counts</th>\n",
       "      <th>tuesday_counts</th>\n",
       "      <th>wednesday_counts</th>\n",
       "      <th>thursday_counts</th>\n",
       "      <th>friday_counts</th>\n",
       "      <th>saturday_counts</th>\n",
       "      <th>sunday_counts</th>\n",
       "      <th>midnight_counts</th>\n",
       "      <th>daytime_counts</th>\n",
       "      <th>early_morning_counts</th>\n",
       "      <th>night_counts</th>\n",
       "      <th>trainrow</th>\n",
       "      <th>testrow</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>device_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>29182687948017175</td>\n",
       "      <td>53</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>55</td>\n",
       "      <td>25</td>\n",
       "      <td>62</td>\n",
       "      <td>76</td>\n",
       "      <td>86</td>\n",
       "      <td>68</td>\n",
       "      <td>26</td>\n",
       "      <td>58469.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-6401643145415154744</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>68691.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-4833982096941402721</td>\n",
       "      <td>42</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>35</td>\n",
       "      <td>28</td>\n",
       "      <td>16</td>\n",
       "      <td>42</td>\n",
       "      <td>21</td>\n",
       "      <td>185</td>\n",
       "      <td>10</td>\n",
       "      <td>32</td>\n",
       "      <td>7337.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-6815121365017318426</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>30</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>9287.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-5373797595892518570</td>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "      <td>203</td>\n",
       "      <td>118</td>\n",
       "      <td>3</td>\n",
       "      <td>65</td>\n",
       "      <td>102</td>\n",
       "      <td>96</td>\n",
       "      <td>341</td>\n",
       "      <td>47</td>\n",
       "      <td>41</td>\n",
       "      <td>41396.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-6214565785039593168</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56726.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1530011109762458400</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21397.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2659296957641269347</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21441.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-7567402485735900406</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>66060.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-1779579474805197357</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21793.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44951 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      monday_counts  tuesday_counts  wednesday_counts  \\\n",
       "device_id                                                               \n",
       " 29182687948017175               53              20                20   \n",
       "-6401643145415154744             12              11                 6   \n",
       "-4833982096941402721             42              39                46   \n",
       "-6815121365017318426              0               0                 0   \n",
       "-5373797595892518570             26               8               203   \n",
       "...                             ...             ...               ...   \n",
       "-6214565785039593168              1               1                 1   \n",
       " 1530011109762458400              0               0                 0   \n",
       " 2659296957641269347              0               0                 1   \n",
       "-7567402485735900406              1               0                 1   \n",
       "-1779579474805197357              0               0                 0   \n",
       "\n",
       "                      thursday_counts  friday_counts  saturday_counts  \\\n",
       "device_id                                                               \n",
       " 29182687948017175                 21             55               25   \n",
       "-6401643145415154744                5              0                0   \n",
       "-4833982096941402721               35             28               16   \n",
       "-6815121365017318426                0              0                1   \n",
       "-5373797595892518570              118              3               65   \n",
       "...                               ...            ...              ...   \n",
       "-6214565785039593168                0              0                1   \n",
       " 1530011109762458400                0              1                2   \n",
       " 2659296957641269347                0              1                0   \n",
       "-7567402485735900406                0              0                0   \n",
       "-1779579474805197357                1              2                0   \n",
       "\n",
       "                      sunday_counts  midnight_counts  daytime_counts  \\\n",
       "device_id                                                              \n",
       " 29182687948017175               62               76              86   \n",
       "-6401643145415154744             39               17              49   \n",
       "-4833982096941402721             42               21             185   \n",
       "-6815121365017318426             46               30               9   \n",
       "-5373797595892518570            102               96             341   \n",
       "...                             ...              ...             ...   \n",
       "-6214565785039593168              1                0               4   \n",
       " 1530011109762458400              0                1               0   \n",
       " 2659296957641269347              0                1               0   \n",
       "-7567402485735900406              0                1               0   \n",
       "-1779579474805197357              0                0               0   \n",
       "\n",
       "                      early_morning_counts  night_counts  trainrow  testrow  \n",
       "device_id                                                                    \n",
       " 29182687948017175                      68            26   58469.0      NaN  \n",
       "-6401643145415154744                     1             6       NaN  68691.0  \n",
       "-4833982096941402721                    10            32    7337.0      NaN  \n",
       "-6815121365017318426                     8             0    9287.0      NaN  \n",
       "-5373797595892518570                    47            41   41396.0      NaN  \n",
       "...                                    ...           ...       ...      ...  \n",
       "-6214565785039593168                     0             1       NaN  56726.0  \n",
       " 1530011109762458400                     0             2       NaN  21397.0  \n",
       " 2659296957641269347                     1             0   21441.0      NaN  \n",
       "-7567402485735900406                     1             0   66060.0      NaN  \n",
       "-1779579474805197357                     2             1       NaN  21793.0  \n",
       "\n",
       "[44951 rows x 13 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As both of these eventsframes have the same indices, the encoded brands will get assigned to the specific device_ids\n",
    "# Adding the counts columns to train and test data\n",
    "train_data['monday_counts'] = events_count['monday_counts']\n",
    "test_data['monday_counts'] = events_count['monday_counts']\n",
    "\n",
    "train_data['tuesday_counts'] = events_count['tuesday_counts']\n",
    "test_data['tuesday_counts'] = events_count['tuesday_counts']\n",
    "\n",
    "train_data['wednesday_counts'] = events_count['wednesday_counts']\n",
    "test_data['wednesday_counts'] = events_count['wednesday_counts']\n",
    "\n",
    "train_data['thursday_counts'] = events_count['thursday_counts']\n",
    "test_data['thursday_counts'] = events_count['thursday_counts']\n",
    "\n",
    "train_data['friday_counts'] = events_count['friday_counts']\n",
    "test_data['friday_counts'] = events_count['friday_counts']\n",
    "\n",
    "train_data['saturday_counts'] = events_count['saturday_counts']\n",
    "test_data['saturday_counts'] = events_count['saturday_counts']\n",
    "\n",
    "train_data['sunday_counts'] = events_count['sunday_counts']\n",
    "test_data['sunday_counts'] = events_count['sunday_counts']\n",
    "\n",
    "train_data['midnight_counts'] = events_count['midnight_counts']\n",
    "test_data['midnight_counts'] = events_count['midnight_counts']\n",
    "\n",
    "train_data['daytime_counts'] = events_count['daytime_counts']\n",
    "test_data['daytime_counts'] = events_count['daytime_counts']\n",
    "\n",
    "train_data['early_morning_counts'] = events_count['early_morning_counts']\n",
    "test_data['early_morning_counts'] = events_count['early_morning_counts']\n",
    "\n",
    "train_data['night_counts'] = events_count['night_counts']\n",
    "test_data['night_counts'] = events_count['night_counts']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.fillna(0)\n",
    "test_data = test_data.fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MinMaxScaler for Days Counts and HourBins Counts "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scaling the days count as the values vary on a very wide range\n",
    "\n",
    "std_monday = MinMaxScaler()\n",
    "std_monday.fit(train_data['monday_counts'].values.reshape(-1,1))\n",
    "train_data['std_monday_counts'] = std_monday.transform(train_data['monday_counts'].values.reshape(-1,1))\n",
    "test_data['std_monday_counts'] = std_monday.transform(test_data['monday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_tuesday = MinMaxScaler()\n",
    "std_tuesday.fit(train_data['tuesday_counts'].values.reshape(-1,1))\n",
    "train_data['std_tuesday_counts'] = std_tuesday.transform(train_data['tuesday_counts'].values.reshape(-1,1))\n",
    "test_data['std_tuesday_counts'] = std_tuesday.transform(test_data['tuesday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_wednesday = MinMaxScaler()\n",
    "std_wednesday.fit(train_data['wednesday_counts'].values.reshape(-1,1))\n",
    "train_data['std_wednesday_counts'] = std_wednesday.transform(train_data['wednesday_counts'].values.reshape(-1,1))\n",
    "test_data['std_wednesday_counts'] = std_wednesday.transform(test_data['wednesday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_thursday = MinMaxScaler()\n",
    "std_thursday.fit(train_data['thursday_counts'].values.reshape(-1,1))\n",
    "train_data['std_thursday_counts'] = std_thursday.transform(train_data['thursday_counts'].values.reshape(-1,1))\n",
    "test_data['std_thursday_counts'] = std_thursday.transform(test_data['thursday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_friday = MinMaxScaler()\n",
    "std_friday.fit(train_data['friday_counts'].values.reshape(-1,1))\n",
    "train_data['std_friday_counts'] = std_friday.transform(train_data['friday_counts'].values.reshape(-1,1))\n",
    "test_data['std_friday_counts'] = std_friday.transform(test_data['friday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_saturday = MinMaxScaler()\n",
    "std_saturday.fit(train_data['saturday_counts'].values.reshape(-1,1))\n",
    "train_data['std_saturday_counts'] = std_saturday.transform(train_data['saturday_counts'].values.reshape(-1,1))\n",
    "test_data['std_saturday_counts'] = std_saturday.transform(test_data['saturday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_sunday = MinMaxScaler()\n",
    "std_sunday.fit(train_data['sunday_counts'].values.reshape(-1,1))\n",
    "train_data['std_sunday_counts'] = std_sunday.transform(train_data['sunday_counts'].values.reshape(-1,1))\n",
    "test_data['std_sunday_counts'] = std_sunday.transform(test_data['sunday_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_midnight = MinMaxScaler()\n",
    "std_midnight.fit(train_data['midnight_counts'].values.reshape(-1,1))\n",
    "train_data['std_midnight_counts'] = std_midnight.transform(train_data['midnight_counts'].values.reshape(-1,1))\n",
    "test_data['std_midnight_counts'] = std_midnight.transform(test_data['midnight_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_daytime = MinMaxScaler()\n",
    "std_daytime.fit(train_data['daytime_counts'].values.reshape(-1,1))\n",
    "train_data['std_daytime_counts'] = std_daytime.transform(train_data['daytime_counts'].values.reshape(-1,1))\n",
    "test_data['std_daytime_counts'] = std_daytime.transform(test_data['daytime_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_early_morning = MinMaxScaler()\n",
    "std_early_morning.fit(train_data['early_morning_counts'].values.reshape(-1,1))\n",
    "train_data['std_early_morning_counts'] = std_early_morning.transform(train_data['early_morning_counts'].values.reshape(-1,1))\n",
    "test_data['std_early_morning_counts'] = std_early_morning.transform(test_data['early_morning_counts'].values.reshape(-1,1))\n",
    "\n",
    "std_night = MinMaxScaler()\n",
    "std_night.fit(train_data['night_counts'].values.reshape(-1,1))\n",
    "train_data['std_night_counts'] = std_night.transform(train_data['night_counts'].values.reshape(-1,1))\n",
    "test_data['std_night_counts'] = std_night.transform(test_data['night_counts'].values.reshape(-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr_monday_counts = train_data['std_monday_counts'].values.reshape(-1,1)\n",
    "Xte_monday_counts = test_data['std_monday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_tuesday_counts = train_data['std_tuesday_counts'].values.reshape(-1,1)\n",
    "Xte_tuesday_counts = test_data['std_tuesday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_wednesday_counts = train_data['std_wednesday_counts'].values.reshape(-1,1)\n",
    "Xte_wednesday_counts = test_data['std_wednesday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_thursday_counts = train_data['std_thursday_counts'].values.reshape(-1,1)\n",
    "Xte_thursday_counts = test_data['std_thursday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_friday_counts = train_data['std_friday_counts'].values.reshape(-1,1)\n",
    "Xte_friday_counts = test_data['std_friday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_saturday_counts = train_data['std_saturday_counts'].values.reshape(-1,1)\n",
    "Xte_saturday_counts = test_data['std_saturday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_sunday_counts = train_data['std_sunday_counts'].values.reshape(-1,1)\n",
    "Xte_sunday_counts = test_data['std_sunday_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_midnight_counts = train_data['std_midnight_counts'].values.reshape(-1,1)\n",
    "Xte_midnight_counts = test_data['std_midnight_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_daytime_counts = train_data['std_daytime_counts'].values.reshape(-1,1)\n",
    "Xte_daytime_counts = test_data['std_daytime_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_early_morning_counts = train_data['std_early_morning_counts'].values.reshape(-1,1)\n",
    "Xte_early_morning_counts = test_data['std_early_morning_counts'].values.reshape(-1,1)\n",
    "\n",
    "Xtr_night_counts = train_data['std_night_counts'].values.reshape(-1,1)\n",
    "Xte_night_counts = test_data['std_night_counts'].values.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TFIDF Vectorizer for each hour of the day "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['event_hours'] = hourevents['hour']\n",
    "test_data['event_hours'] = hourevents['hour']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.fillna('0')\n",
    "test_data = test_data.fillna('0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape :  (74645, 24)\n",
      "Test shape :  (112071, 24)\n"
     ]
    }
   ],
   "source": [
    "vectorizer_hours=TfidfVectorizer()\n",
    "vectorizer_hours.fit(train_data['event_hours'].values)\n",
    "\n",
    "Xtr_hours = vectorizer_hours.transform(train_data['event_hours'].values)\n",
    "Xte_hours = vectorizer_hours.transform(test_data['event_hours'].values)\n",
    "\n",
    "print(\"Train shape : \",Xtr_hours.shape)\n",
    "print(\"Test shape : \",Xte_hours.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TFIDF Vectorizer for Apps Active "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>apps_active</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>device_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>-9222956879900151005</td>\n",
       "      <td>False False False False False False False True...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-9222661944218806987</td>\n",
       "      <td>True False True True True True True False Fals...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-9222399302879214035</td>\n",
       "      <td>False False False False False False False Fals...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-9221825537663503111</td>\n",
       "      <td>False False True False False True True False F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>-9221767098072603291</td>\n",
       "      <td>True False False False False True False True F...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                            apps_active\n",
       "device_id                                                              \n",
       "-9222956879900151005  False False False False False False False True...\n",
       "-9222661944218806987  True False True True True True True False Fals...\n",
       "-9222399302879214035  False False False False False False False Fals...\n",
       "-9221825537663503111  False False True False False True True False F...\n",
       "-9221767098072603291  True False False False False True False True F..."
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events = pd.read_csv('Data/events.csv',parse_dates=['timestamp'], index_col='event_id')\n",
    "\n",
    "apps_active = app_events.groupby(['event_id'])['is_active'].apply(lambda x: \" \".join(str(s) for s in x))\n",
    "events['apps_active'] = events.index.map(apps_active)\n",
    "events_apps_active = events.groupby(\"device_id\")[\"apps_active\"].apply(lambda x: \" \".join(str(s) for s in x if str(s)!='nan'))\n",
    "events_apps_active = events_apps_active.reset_index().set_index('device_id')\n",
    "events_apps_active.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['apps_active'] = events_apps_active['apps_active']\n",
    "test_data['apps_active'] = events_apps_active['apps_active']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.fillna('0')\n",
    "test_data = test_data.fillna('0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Shape  (74645, 2)  Test Shape  (112071, 2)\n"
     ]
    }
   ],
   "source": [
    "vectorizer_apps_active=TfidfVectorizer()\n",
    "vectorizer_apps_active.fit(train_data['apps_active'].values)\n",
    "\n",
    "Xtr_apps_active = vectorizer_apps_active.transform(train_data['apps_active'].values)\n",
    "Xte_apps_active = vectorizer_apps_active.transform(test_data['apps_active'].values)\n",
    "print(\"Train Shape \",Xtr_apps_active.shape,\" Test Shape \",Xte_apps_active.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Stacking  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape  (74645, 21496)\n",
      "X_test shape  (112071, 21496)\n"
     ]
    }
   ],
   "source": [
    "X_train = hstack((Xtr_brand,\n",
    "                  Xtr_model,\n",
    "                  Xtr_app,\n",
    "                  Xtr_label,\n",
    "                  Xtr_hours,\n",
    "                  Xtr_monday_counts,\n",
    "                  Xtr_tuesday_counts,\n",
    "                  Xtr_wednesday_counts,\n",
    "                  Xtr_thursday_counts,\n",
    "                  Xtr_friday_counts,\n",
    "                  Xtr_saturday_counts,\n",
    "                  Xtr_sunday_counts,\n",
    "                  Xtr_midnight_counts,\n",
    "                  Xtr_early_morning_counts,\n",
    "                  Xtr_daytime_counts,\n",
    "                  Xtr_night_counts,\n",
    "                  Xtr_apps_active)).tocsr()\n",
    "\n",
    "X_test = hstack((Xte_brand,\n",
    "                  Xte_model,\n",
    "                  Xte_app,\n",
    "                  Xte_label,\n",
    "                  Xte_hours,\n",
    "                  Xte_monday_counts,\n",
    "                  Xte_tuesday_counts,\n",
    "                  Xte_wednesday_counts,\n",
    "                  Xte_thursday_counts,\n",
    "                  Xte_friday_counts,\n",
    "                  Xte_saturday_counts,\n",
    "                  Xte_sunday_counts,\n",
    "                  Xte_midnight_counts,\n",
    "                  Xte_early_morning_counts,\n",
    "                  Xte_daytime_counts,\n",
    "                  Xte_night_counts,\n",
    "                  Xte_apps_active)).tocsr()\n",
    "\n",
    "print(\"X_train shape \",X_train.shape)\n",
    "print(\"X_test shape \",X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y shape  (74645,)\n"
     ]
    }
   ],
   "source": [
    "targetencoder = LabelEncoder().fit(train_data.gender)\n",
    "y = targetencoder.transform(train_data.gender)\n",
    "nclasses = len(targetencoder.classes_)\n",
    "print(\"y shape \", y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import save_npz\n",
    "\n",
    "save_npz('X_train.npz',X_train)\n",
    "save_npz('X_test.npz',X_test)\n",
    "np.save('y_gender',y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(test_y, predict_y):\n",
    "    '''\n",
    "    This function buils a confusion matrix along with precision and recall matrices\n",
    "    '''\n",
    "    C = confusion_matrix(test_y, predict_y)\n",
    "    A =(((C.T)/(C.sum(axis=1))).T)\n",
    "    B =(C/C.sum(axis=0))\n",
    "   \n",
    "    labels = ['F','M']\n",
    "    print(\"-\"*20, \"Confusion matrix\", \"-\"*20)\n",
    "    plt.figure()\n",
    "    sns.heatmap(C, annot=True, cmap=\"YlGnBu\", fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n",
    "    plt.xlabel('Predicted Class')\n",
    "    plt.ylabel('Original Class')\n",
    "    plt.show()\n",
    "\n",
    "    print(\"-\"*20, \"Precision matrix (Columm Sum=1)\", \"-\"*20)\n",
    "    plt.figure()\n",
    "    sns.heatmap(B, annot=True, cmap=\"YlGnBu\", fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n",
    "    plt.xlabel('Predicted Class')\n",
    "    plt.ylabel('Original Class')\n",
    "    plt.show()\n",
    "    \n",
    "    print(\"-\"*20, \"Recall matrix (Row sum=1)\", \"-\"*20)\n",
    "    plt.figure()\n",
    "    sns.heatmap(A, annot=True, cmap=\"YlGnBu\", fmt=\".3f\", xticklabels=labels, yticklabels=labels)\n",
    "    plt.xlabel('Predicted Class')\n",
    "    plt.ylabel('Original Class')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the csr matrix\n",
    "from scipy.sparse import load_npz\n",
    "\n",
    "X = load_npz('X_train.npz')\n",
    "X_test = load_npz('X_test.npz')\n",
    "y = np.load('y_gender.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All features Events: train shape (59716, 21496), test shape (14929, 21496)\n"
     ]
    }
   ],
   "source": [
    "# Splitting the data\n",
    "X_train, X_val, y_train, y_val = \\\n",
    "            train_test_split(X, y, random_state=1026, test_size=0.2, stratify = y)\n",
    "print('All features Events: train shape {}, test shape {}'.format(X_train.shape, X_val.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Building "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Activation, BatchNormalization,Input,PReLU\n",
    "from keras.optimizers import Adam, Adagrad\n",
    "from keras.utils import np_utils\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_model(history):\n",
    "    # Plot training & validation accuracy values\n",
    "    plt.plot(history.history['accuracy'])\n",
    "    plt.plot(history.history['val_accuracy'])\n",
    "    plt.title('Model accuracy')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['Train', 'Test'], loc='upper left')\n",
    "    plt.show()\n",
    "\n",
    "    # Plot training & validation loss values\n",
    "    plt.plot(history.history['loss'])\n",
    "    plt.plot(history.history['val_loss'])\n",
    "    plt.title('Model loss')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['Train', 'Test'], loc='upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np_utils.to_categorical(y_train)\n",
    "y_val = np_utils.to_categorical(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.59772, saving model to best_models/NN2/best_model_gender.hdf5\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.59772 to 0.59058, saving model to best_models/NN2/best_model_gender.hdf5\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.59058 to 0.58924, saving model to best_models/NN2/best_model_gender.hdf5\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.58924\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.58924\n"
     ]
    }
   ],
   "source": [
    "# Reference -> https://www.kaggle.com/c/talkingdata-mobile-user-demographics/discussion/23424\n",
    "model_list = []\n",
    "avg_val_loss = 0\n",
    "filepath=\"best_models/NN2/best_model_gender.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\n",
    "input_dim = X.shape[1]\n",
    "output_dim = 2\n",
    "model = Sequential()\n",
    "model.add(Dropout(0.3, input_shape=(input_dim,)))\n",
    "model.add(Dense(80))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(50, init='normal', activation='relu'))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(output_dim, init='normal', activation='softmax'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, batch_size=350, epochs=5, verbose=0, validation_data=(X_val,y_val), callbacks = [checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dim = X.shape[1]\n",
    "output_dim = 2\n",
    "model = Sequential()\n",
    "model.add(Dropout(0.3, input_shape=(input_dim,)))\n",
    "model.add(Dense(80))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(50, init='normal', activation='relu'))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(output_dim, init='normal', activation='softmax'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "model.load_weights(\"best_models/NN2/best_model_gender.hdf5\")\n",
    "X_train_gender = model.predict_proba(X_train)\n",
    "X_val_gender = model.predict_proba(X_val)\n",
    "X_test_gender = model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_gender_class = np.argmax(X_train_gender,axis=1)\n",
    "X_val_gender_class = np.argmax(X_val_gender,axis=1)\n",
    "X_test_gender_class = np.argmax(X_test_gender,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y shape  (74645,)\n"
     ]
    }
   ],
   "source": [
    "targetencoder = LabelEncoder().fit(train_data.group)\n",
    "y = targetencoder.transform(train_data.group)\n",
    "nclasses = len(targetencoder.classes_)\n",
    "print(\"y shape \", y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, _, y_train, y_val = train_test_split(X, y, random_state=1026, test_size=0.2, stratify = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_male = []\n",
    "X_train_female = []\n",
    "y_train_male = []\n",
    "y_train_female = []\n",
    "\n",
    "for idx,i in enumerate(X_train_gender_class):\n",
    "    if(i==1):\n",
    "        X_train_male.append(X_train[idx])\n",
    "        y_train_male.append(y_train[idx])\n",
    "    elif(i==0):\n",
    "        X_train_female.append(X_train[idx])\n",
    "        y_train_female.append(y_train[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val_male = []\n",
    "X_val_female = []\n",
    "y_val_male = []\n",
    "y_val_female = []\n",
    "\n",
    "for idx,i in enumerate(X_val_gender_class):\n",
    "    if(i==1):\n",
    "        X_val_male.append(X_val[idx])\n",
    "        y_val_male.append(y_val[idx])\n",
    "    elif(i==0):\n",
    "        X_val_female.append(X_val[idx])\n",
    "        y_val_female.append(y_val[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_male = np_utils.to_categorical(y_train_male)\n",
    "y_train_female = np_utils.to_categorical(y_train_female)\n",
    "y_val_male = np_utils.to_categorical(y_val_male)\n",
    "y_val_female = np_utils.to_categorical(y_val_female)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_male = sparse.vstack((i for i in X_train_male), format='csr')\n",
    "X_train_female = sparse.vstack((i for i in X_train_female), format='csr')\n",
    "\n",
    "X_val_male = sparse.vstack((i for i in X_val_male), format='csr')\n",
    "X_val_female = sparse.vstack((i for i in X_val_female), format='csr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Females Neural Network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 2.42576, saving model to best_models/NN2/best_model_female.hdf5\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 2.42576\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 2.42576\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 2.42576\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 2.42576\n"
     ]
    }
   ],
   "source": [
    "# Reference -> https://www.kaggle.com/c/talkingdata-mobile-user-demographics/discussion/23424\n",
    "model_list = []\n",
    "avg_val_loss = 0\n",
    "filepath=\"best_models/NN2/best_model_female.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\n",
    "input_dim = X_train_female.shape[1]\n",
    "output_dim = 12\n",
    "model = Sequential()\n",
    "model.add(Dropout(0.3, input_shape=(input_dim,)))\n",
    "model.add(Dense(80))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(50, init='normal', activation='relu'))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(output_dim, init='normal', activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train_female, y_train_female, batch_size=350, epochs=5, verbose=0, validation_data=(X_val_female,y_val_female), callbacks = [checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(\"best_models/NN2/best_model_female.hdf5\")\n",
    "X_test_pred = model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Males Neural Network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 2.42916, saving model to best_models/NN2/best_model_male.hdf5\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 2.42916\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 2.42916\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 2.42916\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 2.42916\n"
     ]
    }
   ],
   "source": [
    "# Reference -> https://www.kaggle.com/c/talkingdata-mobile-user-demographics/discussion/23424\n",
    "model_list = []\n",
    "avg_val_loss = 0\n",
    "filepath=\"best_models/NN2/best_model_male.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')\n",
    "input_dim = X_train_male.shape[1]\n",
    "output_dim = 12\n",
    "model = Sequential()\n",
    "model.add(Dropout(0.3, input_shape=(input_dim,)))\n",
    "model.add(Dense(80))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(50, init='normal', activation='relu'))\n",
    "model.add(PReLU())\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(output_dim, init='normal', activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train_male, y_train_male, batch_size=350, epochs=5, verbose=0, validation_data=(X_val_male,y_val_male), callbacks = [checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(\"best_models/NN2/best_model_male.hdf5\")\n",
    "X_test_pred_2 = model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictions = []\n",
    "\n",
    "for idx,i in enumerate(X_test_gender_class):\n",
    "    if(i==1):\n",
    "        test_predictions.append(X_test_pred_2[idx])\n",
    "    elif(i==0):\n",
    "        test_predictions.append(X_test_pred[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_data = pd.DataFrame(test_predictions).set_index(test_data.index)\n",
    "predict_data.columns = np.unique(train_data.group)\n",
    "predict_data.to_csv('submissions_final.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}