xgboost_skf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost with Stratified K-Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time, re, string\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from scipy.sparse import hstack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Train and Test Set with Derived Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>is_open</th>\n",
       "      <th>is_click</th>\n",
       "      <th>date</th>\n",
       "      <th>month</th>\n",
       "      <th>hour</th>\n",
       "      <th>mins</th>\n",
       "      <th>communication_type</th>\n",
       "      <th>total_links</th>\n",
       "      <th>no_of_internal_links</th>\n",
       "      <th>...</th>\n",
       "      <th>is_open_confidence</th>\n",
       "      <th>email_3_similar</th>\n",
       "      <th>sub_3_similar</th>\n",
       "      <th>campaign_id</th>\n",
       "      <th>sub_period</th>\n",
       "      <th>comm_type_click_percent</th>\n",
       "      <th>body_polarity</th>\n",
       "      <th>title_polarity</th>\n",
       "      <th>body_subjectivity</th>\n",
       "      <th>title_subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14051</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>55</td>\n",
       "      <td>3</td>\n",
       "      <td>88.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>42</td>\n",
       "      <td>247</td>\n",
       "      <td>12.70333</td>\n",
       "      <td>0.278346</td>\n",
       "      <td>0.462500</td>\n",
       "      <td>0.644388</td>\n",
       "      <td>0.650000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>134438</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>53</td>\n",
       "      <td>3</td>\n",
       "      <td>67.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>52</td>\n",
       "      <td>150</td>\n",
       "      <td>12.70333</td>\n",
       "      <td>0.071053</td>\n",
       "      <td>0.170455</td>\n",
       "      <td>0.376754</td>\n",
       "      <td>0.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>181789</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33</td>\n",
       "      <td>249</td>\n",
       "      <td>14.27746</td>\n",
       "      <td>0.408333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.579365</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231448</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>36</td>\n",
       "      <td>5</td>\n",
       "      <td>60.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>44</td>\n",
       "      <td>216</td>\n",
       "      <td>13.02976</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>185580</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>67.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>29</td>\n",
       "      <td>220</td>\n",
       "      <td>12.70333</td>\n",
       "      <td>0.390083</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.588671</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  is_open  is_click  date  month  hour  mins  communication_type  \\\n",
       "0    14051        0         0     1      9    19    55                   3   \n",
       "1   134438        0         0     2     11    12    53                   3   \n",
       "2   181789        0         0    24      7    15    15                   4   \n",
       "3   231448        0         0     5      9    11    36                   5   \n",
       "4   185580        0         0     1      7    18     1                   3   \n",
       "\n",
       "   total_links  no_of_internal_links         ...          is_open_confidence  \\\n",
       "0         88.0                  79.0         ...                         0.0   \n",
       "1         67.0                  62.0         ...                         0.0   \n",
       "2          7.0                   3.0         ...                         0.0   \n",
       "3         60.0                  56.0         ...                         0.0   \n",
       "4         67.0                  61.0         ...                         0.0   \n",
       "\n",
       "   email_3_similar  sub_3_similar  campaign_id  sub_period  \\\n",
       "0              0.0            2.0           42         247   \n",
       "1              1.0            2.0           52         150   \n",
       "2              1.0            1.0           33         249   \n",
       "3              1.0            1.0           44         216   \n",
       "4              0.0            1.0           29         220   \n",
       "\n",
       "   comm_type_click_percent  body_polarity  title_polarity  body_subjectivity  \\\n",
       "0                 12.70333       0.278346        0.462500           0.644388   \n",
       "1                 12.70333       0.071053        0.170455           0.376754   \n",
       "2                 14.27746       0.408333        0.000000           0.579365   \n",
       "3                 13.02976       0.166667        0.000000           0.766667   \n",
       "4                 12.70333       0.390083        0.000000           0.588671   \n",
       "\n",
       "   title_subjectivity  \n",
       "0            0.650000  \n",
       "1            0.454545  \n",
       "2            0.000000  \n",
       "3            0.000000  \n",
       "4            0.000000  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('input/train_featv3.csv')\n",
    "test = pd.read_csv('input/test_featv3.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'is_open', 'is_click', 'date', 'month', 'hour', 'mins',\n",
       "       'communication_type', 'total_links', 'no_of_internal_links',\n",
       "       'no_of_images', 'no_of_sections', 'count_sent', 'count_word',\n",
       "       'count_unique_word', 'count_letters', 'count_punctuations',\n",
       "       'count_stopwords', 'mean_word_len', 'word_unique_percent',\n",
       "       'punct_percent', 'email_count_word', 'email_count_unique_word',\n",
       "       'email_count_letters', 'email_count_punctuations', 'email_cap_count',\n",
       "       'day_of_week', 'count_click', 'count_user', 'click_confidence',\n",
       "       'count_is_open', 'is_open_confidence', 'email_3_similar',\n",
       "       'sub_3_similar', 'campaign_id', 'sub_period', 'comm_type_click_percent',\n",
       "       'body_polarity', 'title_polarity', 'body_subjectivity',\n",
       "       'title_subjectivity'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Date \n",
    "- Time (in minutes)\n",
    "- Day of Week\n",
    "- Communication Type\n",
    "- Total Links\n",
    "- No of Internal Links, No of Images\n",
    "- Subject - Count of Sentences, Letters, Punctuations and Stopwords\n",
    "- Subject - Unique Word Percentage\n",
    "- Subject - Punctuation Percentage\n",
    "- Email - Count of Word, Punctuation and Capital Letters\n",
    "- Count Click\n",
    "- Count User\n",
    "- Click Confidence\n",
    "- Count of People Opening the Mail\n",
    "- Open Confidence\n",
    "- Email Similarity, Subject Similarity\n",
    "- Subscription Period\n",
    "- Communication Type Click Percentage\n",
    "- Count User Frequency\n",
    "- Sentiment of Mail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['time'] = train['hour']*60 + train['mins']\n",
    "test['time'] = test['hour']*60 + test['mins']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = train['is_click'].values\n",
    "\n",
    "cols = ['user_id', 'date', 'time', 'communication_type', 'total_links',  \n",
    "        'no_of_internal_links', 'no_of_images', 'count_sent', 'count_letters', 'count_punctuations', \n",
    "        'count_stopwords', 'word_unique_percent', 'punct_percent', 'email_count_word', \n",
    "        'email_count_unique_word', 'email_count_punctuations', 'email_cap_count', 'day_of_week', \n",
    "        'count_click', 'count_user', 'click_confidence','count_is_open','is_open_confidence', \n",
    "        'body_polarity', 'title_polarity','body_subjectivity', 'title_subjectivity', \n",
    "        'email_3_similar', 'sub_3_similar', 'sub_period', 'comm_type_click_percent']\n",
    "\n",
    "X_train = train[cols]\n",
    "X_test = test[cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fill NaN values with mean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:537: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "## filled new user with click confidence of mean of first users ###\n",
    "X_test['click_confidence'] = X_test['click_confidence'].fillna(0.0072169867589168555)\n",
    "X_test['is_open_confidence'] = X_test['is_open_confidence'].fillna(0.10831444590242156)\n",
    "X_test.loc[X_test['count_user'] == 0,'count_click'] = 0.0072169867589168555\n",
    "X_test.loc[X_test['count_user'] == 0,'count_is_open'] = 0.10831444590242156\n",
    "X_test.loc[X_test['count_user'] == 0,'count_user'] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Sentiment Score from Polarity and Subjectivity Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:357: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[key] = _infer_fill_value(value)\n",
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:537: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "X_train.loc[:,'count_user_freq'] = (1/X_train.loc[:,'count_user'])\n",
    "X_test.loc[:,'count_user_freq'] = (1/X_test.loc[:,'count_user'])\n",
    "\n",
    "dropcols = ['body_polarity', 'title_polarity','body_subjectivity', 'title_subjectivity']\n",
    "X_train.loc[:,'sentiment'] = X_train.loc[:,'body_polarity'] + X_train.loc[:,'title_polarity'] + X_train.loc[:,'body_subjectivity'] + X_train.loc[:,'title_subjectivity']\n",
    "X_test.loc[:,'sentiment'] = X_test.loc[:,'body_polarity'] + X_test.loc[:,'title_polarity'] + X_test.loc[:,'body_subjectivity'] + X_test.loc[:,'title_subjectivity']\n",
    "X_train.drop(dropcols,axis=1,inplace=True)\n",
    "X_test.drop(dropcols,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>communication_type</th>\n",
       "      <th>total_links</th>\n",
       "      <th>no_of_internal_links</th>\n",
       "      <th>no_of_images</th>\n",
       "      <th>count_sent</th>\n",
       "      <th>count_letters</th>\n",
       "      <th>count_punctuations</th>\n",
       "      <th>...</th>\n",
       "      <th>count_user</th>\n",
       "      <th>click_confidence</th>\n",
       "      <th>count_is_open</th>\n",
       "      <th>is_open_confidence</th>\n",
       "      <th>email_3_similar</th>\n",
       "      <th>sub_3_similar</th>\n",
       "      <th>sub_period</th>\n",
       "      <th>comm_type_click_percent</th>\n",
       "      <th>count_user_freq</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14051</td>\n",
       "      <td>1</td>\n",
       "      <td>1195</td>\n",
       "      <td>3</td>\n",
       "      <td>88.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>247</td>\n",
       "      <td>12.70333</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>2.035234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>134438</td>\n",
       "      <td>2</td>\n",
       "      <td>773</td>\n",
       "      <td>3</td>\n",
       "      <td>67.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150</td>\n",
       "      <td>12.70333</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.072807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>181789</td>\n",
       "      <td>24</td>\n",
       "      <td>915</td>\n",
       "      <td>4</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>249</td>\n",
       "      <td>14.27746</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.987698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231448</td>\n",
       "      <td>5</td>\n",
       "      <td>696</td>\n",
       "      <td>5</td>\n",
       "      <td>60.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>216</td>\n",
       "      <td>13.02976</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.933333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>185580</td>\n",
       "      <td>1</td>\n",
       "      <td>1081</td>\n",
       "      <td>3</td>\n",
       "      <td>67.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>220</td>\n",
       "      <td>12.70333</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.978753</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  date  time  communication_type  total_links  no_of_internal_links  \\\n",
       "0    14051     1  1195                   3         88.0                  79.0   \n",
       "1   134438     2   773                   3         67.0                  62.0   \n",
       "2   181789    24   915                   4          7.0                   3.0   \n",
       "3   231448     5   696                   5         60.0                  56.0   \n",
       "4   185580     1  1081                   3         67.0                  61.0   \n",
       "\n",
       "   no_of_images  count_sent  count_letters  count_punctuations    ...      \\\n",
       "0          13.0         1.0           88.0                 5.0    ...       \n",
       "1          10.0         1.0          111.0                 5.0    ...       \n",
       "2           1.0         1.0           71.0                 3.0    ...       \n",
       "3          19.0         1.0           73.0                 3.0    ...       \n",
       "4          12.0         1.0           55.0                 2.0    ...       \n",
       "\n",
       "   count_user  click_confidence  count_is_open  is_open_confidence  \\\n",
       "0         9.0               0.0            0.0                 0.0   \n",
       "1         4.0               0.0            0.0                 0.0   \n",
       "2         7.0               0.0            0.0                 0.0   \n",
       "3         6.0               0.0            0.0                 0.0   \n",
       "4         5.0               0.0            0.0                 0.0   \n",
       "\n",
       "   email_3_similar  sub_3_similar  sub_period  comm_type_click_percent  \\\n",
       "0              0.0            2.0         247                 12.70333   \n",
       "1              1.0            2.0         150                 12.70333   \n",
       "2              1.0            1.0         249                 14.27746   \n",
       "3              1.0            1.0         216                 13.02976   \n",
       "4              0.0            1.0         220                 12.70333   \n",
       "\n",
       "   count_user_freq  sentiment  \n",
       "0         0.111111   2.035234  \n",
       "1         0.250000   1.072807  \n",
       "2         0.142857   0.987698  \n",
       "3         0.166667   0.933333  \n",
       "4         0.200000   0.978753  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Under Sampling using Repeated Edited Nearest Neighbour Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RENN\n",
      "Reduced 5.09%\n"
     ]
    }
   ],
   "source": [
    "from imblearn.under_sampling import (AllKNN, EditedNearestNeighbours, RepeatedEditedNearestNeighbours)\n",
    "\n",
    "print('RENN')\n",
    "renn = RepeatedEditedNearestNeighbours(return_indices=True)\n",
    "X_res, Y_res, idx_res = renn.fit_sample(X_train, Y_train)\n",
    "reduction_str = ('Reduced {:.2f}%'.format(100 * (1 - float(len(X_res))/len(X_train))))\n",
    "print(reduction_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(971083, 29) (971083,)\n",
      "12782 12782\n"
     ]
    }
   ],
   "source": [
    "print (X_res.shape, Y_res.shape)\n",
    "print (Y_res.sum(), Y_train.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Model with 10 Folds "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "import xgboost as xgb\n",
    "\n",
    "def fit_cv(X, y, label, n_splits=10):\n",
    "    estimators, scores = [],[]\n",
    "    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1234)\n",
    "    skf.get_n_splits(label)\n",
    "    for train_idx, valid_idx in tqdm(skf.split(X, y)):\n",
    "        X_train_ = X[train_idx]\n",
    "        y_train_ = y[train_idx]\n",
    "        X_valid_ = X[valid_idx]\n",
    "        y_valid_ =  y[valid_idx]\n",
    "        \n",
    "        params = {}\n",
    "        params['booster'] = 'gbtree'\n",
    "        params['objective'] = 'binary:logistic'\n",
    "        params['eta'] = 0.01\n",
    "        params['eval_metric'] = 'auc'\n",
    "        params['max_depth'] = 3\n",
    "        params['colsample_bytree'] = 0.8\n",
    "        params['subsample'] = 0.8\n",
    "        # params['min_child_weight'] = 5\n",
    "        params['silent'] = 1\n",
    "\n",
    "        d_train = xgb.DMatrix(X_train_, label=y_train_)\n",
    "        d_valid = xgb.DMatrix(X_valid_, label=y_valid_)\n",
    "        watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "        estimator_ = xgb.train(params, d_train, 7200, watchlist, early_stopping_rounds=50, verbose_eval=100)\n",
    "\n",
    "        estimators.append(estimator_)\n",
    "    return estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3f428236a614c9da377d500bf95134d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.976037\tvalid-auc:0.976025\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "\n",
      "[100]\ttrain-auc:0.978533\tvalid-auc:0.977895\n",
      "[200]\ttrain-auc:0.978936\tvalid-auc:0.978352\n",
      "[300]\ttrain-auc:0.97925\tvalid-auc:0.978628\n",
      "[400]\ttrain-auc:0.97946\tvalid-auc:0.978839\n",
      "[500]\ttrain-auc:0.979616\tvalid-auc:0.978985\n",
      "[600]\ttrain-auc:0.979724\tvalid-auc:0.979093\n",
      "[700]\ttrain-auc:0.979798\tvalid-auc:0.979164\n",
      "[800]\ttrain-auc:0.97986\tvalid-auc:0.979213\n",
      "[900]\ttrain-auc:0.979914\tvalid-auc:0.979265\n",
      "[1000]\ttrain-auc:0.979967\tvalid-auc:0.979311\n",
      "[1100]\ttrain-auc:0.980015\tvalid-auc:0.979344\n",
      "[1200]\ttrain-auc:0.980097\tvalid-auc:0.979392\n",
      "[1300]\ttrain-auc:0.980206\tvalid-auc:0.979469\n",
      "[1400]\ttrain-auc:0.980292\tvalid-auc:0.979528\n",
      "[1500]\ttrain-auc:0.980371\tvalid-auc:0.979558\n",
      "[1600]\ttrain-auc:0.98045\tvalid-auc:0.9796\n",
      "[1700]\ttrain-auc:0.980524\tvalid-auc:0.979632\n",
      "[1800]\ttrain-auc:0.980596\tvalid-auc:0.979654\n",
      "[1900]\ttrain-auc:0.980659\tvalid-auc:0.979665\n",
      "[2000]\ttrain-auc:0.980725\tvalid-auc:0.979679\n",
      "[2100]\ttrain-auc:0.980784\tvalid-auc:0.979695\n",
      "[2200]\ttrain-auc:0.980842\tvalid-auc:0.979705\n",
      "Stopping. Best iteration:\n",
      "[2183]\ttrain-auc:0.980832\tvalid-auc:0.979709\n",
      "\n",
      "[0]\ttrain-auc:0.976295\tvalid-auc:0.975718\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978442\tvalid-auc:0.977959\n",
      "[200]\ttrain-auc:0.978932\tvalid-auc:0.978443\n",
      "[300]\ttrain-auc:0.979253\tvalid-auc:0.978746\n",
      "[400]\ttrain-auc:0.979454\tvalid-auc:0.978988\n",
      "[500]\ttrain-auc:0.97961\tvalid-auc:0.979137\n",
      "[600]\ttrain-auc:0.97972\tvalid-auc:0.979217\n",
      "[700]\ttrain-auc:0.979807\tvalid-auc:0.979279\n",
      "[800]\ttrain-auc:0.97987\tvalid-auc:0.979321\n",
      "[900]\ttrain-auc:0.979926\tvalid-auc:0.979349\n",
      "[1000]\ttrain-auc:0.979979\tvalid-auc:0.979379\n",
      "[1100]\ttrain-auc:0.980033\tvalid-auc:0.979411\n",
      "[1200]\ttrain-auc:0.980112\tvalid-auc:0.979448\n",
      "[1300]\ttrain-auc:0.980207\tvalid-auc:0.979492\n",
      "[1400]\ttrain-auc:0.980297\tvalid-auc:0.979536\n",
      "[1500]\ttrain-auc:0.980389\tvalid-auc:0.979574\n",
      "[1600]\ttrain-auc:0.98047\tvalid-auc:0.979597\n",
      "Stopping. Best iteration:\n",
      "[1587]\ttrain-auc:0.98046\tvalid-auc:0.9796\n",
      "\n",
      "[0]\ttrain-auc:0.974057\tvalid-auc:0.974317\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978256\tvalid-auc:0.978805\n",
      "[200]\ttrain-auc:0.978803\tvalid-auc:0.979382\n",
      "[300]\ttrain-auc:0.979125\tvalid-auc:0.97976\n",
      "[400]\ttrain-auc:0.979365\tvalid-auc:0.980003\n",
      "[500]\ttrain-auc:0.979524\tvalid-auc:0.980103\n",
      "[600]\ttrain-auc:0.979631\tvalid-auc:0.980165\n",
      "Stopping. Best iteration:\n",
      "[598]\ttrain-auc:0.979632\tvalid-auc:0.980166\n",
      "\n",
      "[0]\ttrain-auc:0.974084\tvalid-auc:0.974074\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978332\tvalid-auc:0.977553\n",
      "[200]\ttrain-auc:0.978918\tvalid-auc:0.977869\n",
      "[300]\ttrain-auc:0.979252\tvalid-auc:0.978167\n",
      "[400]\ttrain-auc:0.979495\tvalid-auc:0.978428\n",
      "[500]\ttrain-auc:0.979656\tvalid-auc:0.978576\n",
      "[600]\ttrain-auc:0.979768\tvalid-auc:0.978668\n",
      "[700]\ttrain-auc:0.979854\tvalid-auc:0.978723\n",
      "[800]\ttrain-auc:0.979933\tvalid-auc:0.97875\n",
      "[900]\ttrain-auc:0.979992\tvalid-auc:0.978784\n",
      "[1000]\ttrain-auc:0.980042\tvalid-auc:0.978812\n",
      "[1100]\ttrain-auc:0.980092\tvalid-auc:0.978843\n",
      "[1200]\ttrain-auc:0.980171\tvalid-auc:0.978879\n",
      "[1300]\ttrain-auc:0.980261\tvalid-auc:0.978929\n",
      "[1400]\ttrain-auc:0.980352\tvalid-auc:0.978971\n",
      "[1500]\ttrain-auc:0.980432\tvalid-auc:0.979019\n",
      "[1600]\ttrain-auc:0.980521\tvalid-auc:0.979057\n",
      "[1700]\ttrain-auc:0.980599\tvalid-auc:0.979095\n",
      "[1800]\ttrain-auc:0.980669\tvalid-auc:0.979113\n",
      "[1900]\ttrain-auc:0.980731\tvalid-auc:0.979126\n",
      "Stopping. Best iteration:\n",
      "[1876]\ttrain-auc:0.980717\tvalid-auc:0.979128\n",
      "\n",
      "[0]\ttrain-auc:0.973998\tvalid-auc:0.974902\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978185\tvalid-auc:0.979544\n",
      "[200]\ttrain-auc:0.978672\tvalid-auc:0.980168\n",
      "[300]\ttrain-auc:0.979023\tvalid-auc:0.98059\n",
      "[400]\ttrain-auc:0.979259\tvalid-auc:0.980808\n",
      "[500]\ttrain-auc:0.979399\tvalid-auc:0.98091\n",
      "[600]\ttrain-auc:0.979494\tvalid-auc:0.980992\n",
      "[700]\ttrain-auc:0.979584\tvalid-auc:0.981055\n",
      "[800]\ttrain-auc:0.979643\tvalid-auc:0.981104\n",
      "[900]\ttrain-auc:0.9797\tvalid-auc:0.98115\n",
      "[1000]\ttrain-auc:0.979739\tvalid-auc:0.981171\n",
      "[1100]\ttrain-auc:0.97979\tvalid-auc:0.981207\n",
      "[1200]\ttrain-auc:0.979863\tvalid-auc:0.981253\n",
      "[1300]\ttrain-auc:0.979956\tvalid-auc:0.981314\n",
      "[1400]\ttrain-auc:0.980042\tvalid-auc:0.981372\n",
      "[1500]\ttrain-auc:0.980123\tvalid-auc:0.981424\n",
      "[1600]\ttrain-auc:0.980205\tvalid-auc:0.981474\n",
      "[1700]\ttrain-auc:0.980292\tvalid-auc:0.981521\n",
      "[1800]\ttrain-auc:0.980358\tvalid-auc:0.981545\n",
      "[1900]\ttrain-auc:0.980427\tvalid-auc:0.981576\n",
      "[2000]\ttrain-auc:0.980489\tvalid-auc:0.981606\n",
      "[2100]\ttrain-auc:0.980549\tvalid-auc:0.981626\n",
      "[2200]\ttrain-auc:0.980604\tvalid-auc:0.981638\n",
      "[2300]\ttrain-auc:0.980665\tvalid-auc:0.981651\n",
      "[2400]\ttrain-auc:0.980719\tvalid-auc:0.981672\n",
      "[2500]\ttrain-auc:0.980769\tvalid-auc:0.981682\n",
      "[2600]\ttrain-auc:0.980821\tvalid-auc:0.981699\n",
      "[2700]\ttrain-auc:0.98087\tvalid-auc:0.981705\n",
      "Stopping. Best iteration:\n",
      "[2682]\ttrain-auc:0.980861\tvalid-auc:0.981708\n",
      "\n",
      "[0]\ttrain-auc:0.97416\tvalid-auc:0.973395\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978442\tvalid-auc:0.977587\n",
      "[200]\ttrain-auc:0.978947\tvalid-auc:0.977867\n",
      "[300]\ttrain-auc:0.979291\tvalid-auc:0.978097\n",
      "[400]\ttrain-auc:0.979517\tvalid-auc:0.978305\n",
      "[500]\ttrain-auc:0.979666\tvalid-auc:0.978451\n",
      "[600]\ttrain-auc:0.979773\tvalid-auc:0.978555\n",
      "[700]\ttrain-auc:0.979867\tvalid-auc:0.978646\n",
      "[800]\ttrain-auc:0.97994\tvalid-auc:0.978706\n",
      "[900]\ttrain-auc:0.980001\tvalid-auc:0.978735\n",
      "[1000]\ttrain-auc:0.980059\tvalid-auc:0.978755\n",
      "[1100]\ttrain-auc:0.980099\tvalid-auc:0.97877\n",
      "Stopping. Best iteration:\n",
      "[1066]\ttrain-auc:0.980088\tvalid-auc:0.978774\n",
      "\n",
      "[0]\ttrain-auc:0.974132\tvalid-auc:0.973646\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978431\tvalid-auc:0.977646\n",
      "[200]\ttrain-auc:0.978896\tvalid-auc:0.97818\n",
      "[300]\ttrain-auc:0.979246\tvalid-auc:0.978551\n",
      "[400]\ttrain-auc:0.979469\tvalid-auc:0.978761\n",
      "[500]\ttrain-auc:0.97961\tvalid-auc:0.978921\n",
      "[600]\ttrain-auc:0.979728\tvalid-auc:0.979055\n",
      "[700]\ttrain-auc:0.979821\tvalid-auc:0.979145\n",
      "[800]\ttrain-auc:0.979879\tvalid-auc:0.979201\n",
      "[900]\ttrain-auc:0.979933\tvalid-auc:0.979229\n",
      "[1000]\ttrain-auc:0.979984\tvalid-auc:0.979258\n",
      "[1100]\ttrain-auc:0.980034\tvalid-auc:0.979281\n",
      "[1200]\ttrain-auc:0.980126\tvalid-auc:0.97932\n",
      "[1300]\ttrain-auc:0.980205\tvalid-auc:0.979333\n",
      "[1400]\ttrain-auc:0.980316\tvalid-auc:0.979356\n",
      "[1500]\ttrain-auc:0.980406\tvalid-auc:0.979377\n",
      "[1600]\ttrain-auc:0.980489\tvalid-auc:0.9794\n",
      "[1700]\ttrain-auc:0.98057\tvalid-auc:0.979411\n",
      "[1800]\ttrain-auc:0.980636\tvalid-auc:0.979438\n",
      "Stopping. Best iteration:\n",
      "[1810]\ttrain-auc:0.980641\tvalid-auc:0.979443\n",
      "\n",
      "[0]\ttrain-auc:0.974\tvalid-auc:0.974325\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978312\tvalid-auc:0.978754\n",
      "[200]\ttrain-auc:0.978753\tvalid-auc:0.979264\n",
      "[300]\ttrain-auc:0.979115\tvalid-auc:0.979611\n",
      "[400]\ttrain-auc:0.979343\tvalid-auc:0.979826\n",
      "[500]\ttrain-auc:0.979481\tvalid-auc:0.979949\n",
      "[600]\ttrain-auc:0.979602\tvalid-auc:0.980089\n",
      "[700]\ttrain-auc:0.979685\tvalid-auc:0.980196\n",
      "[800]\ttrain-auc:0.979747\tvalid-auc:0.98026\n",
      "[900]\ttrain-auc:0.979799\tvalid-auc:0.98032\n",
      "[1000]\ttrain-auc:0.979846\tvalid-auc:0.980376\n",
      "[1100]\ttrain-auc:0.979892\tvalid-auc:0.980425\n",
      "[1200]\ttrain-auc:0.979989\tvalid-auc:0.980489\n",
      "[1300]\ttrain-auc:0.980078\tvalid-auc:0.980534\n",
      "[1400]\ttrain-auc:0.980169\tvalid-auc:0.980586\n",
      "[1500]\ttrain-auc:0.98025\tvalid-auc:0.98062\n",
      "[1600]\ttrain-auc:0.980323\tvalid-auc:0.980653\n",
      "[1700]\ttrain-auc:0.980399\tvalid-auc:0.980675\n",
      "[1800]\ttrain-auc:0.980473\tvalid-auc:0.9807\n",
      "[1900]\ttrain-auc:0.98054\tvalid-auc:0.980723\n",
      "[2000]\ttrain-auc:0.980604\tvalid-auc:0.980733\n",
      "Stopping. Best iteration:\n",
      "[2013]\ttrain-auc:0.980611\tvalid-auc:0.980737\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.974096\tvalid-auc:0.973971\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978339\tvalid-auc:0.978281\n",
      "[200]\ttrain-auc:0.97885\tvalid-auc:0.978818\n",
      "[300]\ttrain-auc:0.979181\tvalid-auc:0.979174\n",
      "[400]\ttrain-auc:0.979387\tvalid-auc:0.979395\n",
      "[500]\ttrain-auc:0.979528\tvalid-auc:0.979581\n",
      "[600]\ttrain-auc:0.979651\tvalid-auc:0.979713\n",
      "[700]\ttrain-auc:0.979745\tvalid-auc:0.979789\n",
      "[800]\ttrain-auc:0.97982\tvalid-auc:0.97986\n",
      "[900]\ttrain-auc:0.979863\tvalid-auc:0.979903\n",
      "[1000]\ttrain-auc:0.97991\tvalid-auc:0.979949\n",
      "[1100]\ttrain-auc:0.979959\tvalid-auc:0.980004\n",
      "[1200]\ttrain-auc:0.980045\tvalid-auc:0.980061\n",
      "[1300]\ttrain-auc:0.98012\tvalid-auc:0.980099\n",
      "[1400]\ttrain-auc:0.980206\tvalid-auc:0.980137\n",
      "[1500]\ttrain-auc:0.980288\tvalid-auc:0.980177\n",
      "[1600]\ttrain-auc:0.980374\tvalid-auc:0.980216\n",
      "[1700]\ttrain-auc:0.980459\tvalid-auc:0.980258\n",
      "[1800]\ttrain-auc:0.980533\tvalid-auc:0.980275\n",
      "Stopping. Best iteration:\n",
      "[1763]\ttrain-auc:0.980509\tvalid-auc:0.980282\n",
      "\n",
      "[0]\ttrain-auc:0.974063\tvalid-auc:0.974273\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[100]\ttrain-auc:0.978353\tvalid-auc:0.978289\n",
      "[200]\ttrain-auc:0.978793\tvalid-auc:0.978722\n",
      "[300]\ttrain-auc:0.979183\tvalid-auc:0.979123\n",
      "[400]\ttrain-auc:0.979415\tvalid-auc:0.97935\n",
      "[500]\ttrain-auc:0.979557\tvalid-auc:0.979452\n",
      "[600]\ttrain-auc:0.979667\tvalid-auc:0.979547\n",
      "[700]\ttrain-auc:0.979741\tvalid-auc:0.979616\n",
      "[800]\ttrain-auc:0.979803\tvalid-auc:0.979694\n",
      "[900]\ttrain-auc:0.97985\tvalid-auc:0.979746\n",
      "[1000]\ttrain-auc:0.979894\tvalid-auc:0.979798\n",
      "[1100]\ttrain-auc:0.979943\tvalid-auc:0.979845\n",
      "[1200]\ttrain-auc:0.980031\tvalid-auc:0.979942\n",
      "[1300]\ttrain-auc:0.980113\tvalid-auc:0.980005\n",
      "[1400]\ttrain-auc:0.980191\tvalid-auc:0.980068\n",
      "[1500]\ttrain-auc:0.980277\tvalid-auc:0.980128\n",
      "[1600]\ttrain-auc:0.980362\tvalid-auc:0.980187\n",
      "[1700]\ttrain-auc:0.980433\tvalid-auc:0.980227\n",
      "[1800]\ttrain-auc:0.980503\tvalid-auc:0.980252\n",
      "[1900]\ttrain-auc:0.980571\tvalid-auc:0.980274\n",
      "[2000]\ttrain-auc:0.980636\tvalid-auc:0.980296\n",
      "[2100]\ttrain-auc:0.980696\tvalid-auc:0.980311\n",
      "[2200]\ttrain-auc:0.980758\tvalid-auc:0.98032\n",
      "Stopping. Best iteration:\n",
      "[2245]\ttrain-auc:0.980788\tvalid-auc:0.980334\n",
      "\n",
      "\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'scores' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-6f2e26dd0ca5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# lb 66.82 --> train-auc:0.980625 valid-auc:0.98055  best all feat renn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mestimators\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_cv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_res\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_res\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'is_click'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Mean ROC-AUC score for is_click : {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'scores' is not defined"
     ]
    }
   ],
   "source": [
    "estimators = fit_cv(X_res, Y_res, 'is_click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting on Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_train = xgb.DMatrix(X_res, label=Y_res)\n",
    "d_test = xgb.DMatrix(X_test)\n",
    "d_test.feature_names = d_train.feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_click = []\n",
    "for estimator in estimators:\n",
    "    p_test = estimator.predict(d_test) \n",
    "    pred_click.append(p_test)\n",
    "pred_click = np.mean(np.stack(pred_click, axis=-1), axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63_122715</td>\n",
       "      <td>0.000078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>56_76206</td>\n",
       "      <td>0.000078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>57_96189</td>\n",
       "      <td>0.000092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56_166917</td>\n",
       "      <td>0.000078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>56_172838</td>\n",
       "      <td>0.000091</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  is_click\n",
       "0  63_122715  0.000078\n",
       "1   56_76206  0.000078\n",
       "2   57_96189  0.000092\n",
       "3  56_166917  0.000078\n",
       "4  56_172838  0.000091"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = pd.read_csv('input/sample_submission.csv')\n",
    "sub['is_click'] = pred_click\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('sub_xgb_10fold.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>773858.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.013560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.062454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.987405</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            is_click\n",
       "count  773858.000000\n",
       "mean        0.013560\n",
       "std         0.062454\n",
       "min         0.000078\n",
       "25%         0.000078\n",
       "50%         0.000079\n",
       "75%         0.000089\n",
       "max         0.987405"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}