Exploration 3.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('data/train.csv')\n",
    "df_test = pd.read_csv('data/test.csv')\n",
    "df_hist_trans = pd.read_csv('data/hist_txn_tmp.csv')\n",
    "df_new_merchant_trans = pd.read_csv('data/new_merchant_txn_tmp.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_new_columns(name,aggs):\n",
    "    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]\n",
    "\n",
    "    '''\n",
    "    for k in aggs.keys():\n",
    "        for agg in aggs[k]:\n",
    "            name+_+k+agg\n",
    "    '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "aggs = {}\n",
    "for col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n",
    "    aggs[col] = ['nunique']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "aggs['purchase_amount'] = ['sum','max','min','mean','var']\n",
    "aggs['installments'] = ['sum','max','min','mean','var']\n",
    "aggs['purchase_date'] = ['max','min']\n",
    "aggs['month_lag'] = ['max','min','mean','var']\n",
    "aggs['month_diff'] = ['mean']\n",
    "aggs['authorized_flag'] = ['sum', 'mean']\n",
    "aggs['weekend'] = ['sum', 'mean']\n",
    "aggs['category_1'] = ['sum', 'mean']\n",
    "aggs['card_id'] = ['size']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'authorized_flag': ['sum', 'mean'],\n",
       " 'card_id': ['size'],\n",
       " 'category_1': ['sum', 'mean'],\n",
       " 'dayofweek': ['nunique'],\n",
       " 'hour': ['nunique'],\n",
       " 'installments': ['sum', 'max', 'min', 'mean', 'var'],\n",
       " 'merchant_category_id': ['nunique'],\n",
       " 'merchant_id': ['nunique'],\n",
       " 'month': ['nunique'],\n",
       " 'month_diff': ['mean'],\n",
       " 'month_lag': ['max', 'min', 'mean', 'var'],\n",
       " 'purchase_amount': ['sum', 'max', 'min', 'mean', 'var'],\n",
       " 'purchase_date': ['max', 'min'],\n",
       " 'subsector_id': ['nunique'],\n",
       " 'weekend': ['sum', 'mean'],\n",
       " 'weekofyear': ['nunique'],\n",
       " 'year': ['nunique']}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in ['category_2','category_3']:\n",
    "    df_hist_trans[col+'_mean'] = df_hist_trans.groupby([col])['purchase_amount'].transform('mean')\n",
    "    aggs[col+'_mean'] = ['mean']  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>category_1</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>...</th>\n",
       "      <th>subsector_id</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>hour</th>\n",
       "      <th>weekend</th>\n",
       "      <th>month_diff</th>\n",
       "      <th>category_2_mean</th>\n",
       "      <th>category_3_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>80</td>\n",
       "      <td>M_ID_e020e9b302</td>\n",
       "      <td>-8</td>\n",
       "      <td>-0.703331</td>\n",
       "      <td>...</td>\n",
       "      <td>37</td>\n",
       "      <td>2017</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0.074568</td>\n",
       "      <td>0.358458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>367</td>\n",
       "      <td>M_ID_86ec983688</td>\n",
       "      <td>-7</td>\n",
       "      <td>-0.733128</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>2017</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0.074568</td>\n",
       "      <td>0.358458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>80</td>\n",
       "      <td>M_ID_979ed661fc</td>\n",
       "      <td>-6</td>\n",
       "      <td>-0.720386</td>\n",
       "      <td>...</td>\n",
       "      <td>37</td>\n",
       "      <td>2017</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.074568</td>\n",
       "      <td>0.358458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>560</td>\n",
       "      <td>M_ID_e6d5ae8ea6</td>\n",
       "      <td>-5</td>\n",
       "      <td>-0.735352</td>\n",
       "      <td>...</td>\n",
       "      <td>34</td>\n",
       "      <td>2017</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0.074568</td>\n",
       "      <td>0.358458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_4e6213e9bc</td>\n",
       "      <td>88</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>80</td>\n",
       "      <td>M_ID_e020e9b302</td>\n",
       "      <td>-11</td>\n",
       "      <td>-0.722865</td>\n",
       "      <td>...</td>\n",
       "      <td>37</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.074568</td>\n",
       "      <td>0.358458</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   authorized_flag          card_id  city_id  category_1  installments  \\\n",
       "0                1  C_ID_4e6213e9bc       88           0             0   \n",
       "1                1  C_ID_4e6213e9bc       88           0             0   \n",
       "2                1  C_ID_4e6213e9bc       88           0             0   \n",
       "3                1  C_ID_4e6213e9bc       88           0             0   \n",
       "4                1  C_ID_4e6213e9bc       88           0             0   \n",
       "\n",
       "  category_3  merchant_category_id      merchant_id  month_lag  \\\n",
       "0          A                    80  M_ID_e020e9b302         -8   \n",
       "1          A                   367  M_ID_86ec983688         -7   \n",
       "2          A                    80  M_ID_979ed661fc         -6   \n",
       "3          A                   560  M_ID_e6d5ae8ea6         -5   \n",
       "4          A                    80  M_ID_e020e9b302        -11   \n",
       "\n",
       "   purchase_amount  ... subsector_id  year  month  dayofweek  weekofyear  \\\n",
       "0        -0.703331  ...           37  2017      6          6          25   \n",
       "1        -0.733128  ...           16  2017      7          5          28   \n",
       "2        -0.720386  ...           37  2017      8          2          32   \n",
       "3        -0.735352  ...           34  2017      9          5          35   \n",
       "4        -0.722865  ...           37  2017      3          4          10   \n",
       "\n",
       "   hour  weekend  month_diff  category_2_mean  category_3_mean  \n",
       "0    15        1          11         0.074568         0.358458  \n",
       "1    12        1          12         0.074568         0.358458  \n",
       "2    22        0          12         0.074568         0.358458  \n",
       "3    10        1          12         0.074568         0.358458  \n",
       "4     1        0          12         0.074568         0.358458  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 3., 5., 2., 4.])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans.category_2.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['A', 'B', 'C'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans.category_3.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans.category_1.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "category_1\n",
       "0    0.071540\n",
       "1   -0.419327\n",
       "Name: purchase_amount, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans.groupby('category_1')['purchase_amount'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_columns = get_new_columns('hist',aggs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hist_month_nunique',\n",
       " 'hist_hour_nunique',\n",
       " 'hist_weekofyear_nunique',\n",
       " 'hist_dayofweek_nunique',\n",
       " 'hist_year_nunique',\n",
       " 'hist_subsector_id_nunique',\n",
       " 'hist_merchant_id_nunique',\n",
       " 'hist_merchant_category_id_nunique',\n",
       " 'hist_purchase_amount_sum',\n",
       " 'hist_purchase_amount_max',\n",
       " 'hist_purchase_amount_min',\n",
       " 'hist_purchase_amount_mean',\n",
       " 'hist_purchase_amount_var',\n",
       " 'hist_installments_sum',\n",
       " 'hist_installments_max',\n",
       " 'hist_installments_min',\n",
       " 'hist_installments_mean',\n",
       " 'hist_installments_var',\n",
       " 'hist_purchase_date_max',\n",
       " 'hist_purchase_date_min',\n",
       " 'hist_month_lag_max',\n",
       " 'hist_month_lag_min',\n",
       " 'hist_month_lag_mean',\n",
       " 'hist_month_lag_var',\n",
       " 'hist_month_diff_mean',\n",
       " 'hist_authorized_flag_sum',\n",
       " 'hist_authorized_flag_mean',\n",
       " 'hist_weekend_sum',\n",
       " 'hist_weekend_mean',\n",
       " 'hist_category_1_sum',\n",
       " 'hist_category_1_mean',\n",
       " 'hist_card_id_size',\n",
       " 'hist_category_2_mean_mean',\n",
       " 'hist_category_3_mean_mean']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hist_trans_group = df_hist_trans.groupby('card_id').agg(aggs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "card_id\n",
       "C_ID_00007093c1    149\n",
       "C_ID_0001238066    123\n",
       "C_ID_0001506ef0     66\n",
       "C_ID_0001793786    216\n",
       "C_ID_000183fdda    144\n",
       "                  ... \n",
       "C_ID_ffff1d9928     12\n",
       "C_ID_ffff579d3a    114\n",
       "C_ID_ffff756266     24\n",
       "C_ID_ffff828181    190\n",
       "C_ID_fffffd5772     84\n",
       "Length: 325540, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#card_id': ['size'],\n",
    "df_hist_trans.groupby('card_id').agg('size')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>category_1</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>...</th>\n",
       "      <th>subsector_id</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>hour</th>\n",
       "      <th>weekend</th>\n",
       "      <th>month_diff</th>\n",
       "      <th>category_2_mean</th>\n",
       "      <th>category_3_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19095774</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_9400cf2342</td>\n",
       "      <td>-4</td>\n",
       "      <td>-0.530526</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>2017</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095775</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>705</td>\n",
       "      <td>M_ID_a33355a1b7</td>\n",
       "      <td>-12</td>\n",
       "      <td>-0.467415</td>\n",
       "      <td>...</td>\n",
       "      <td>33</td>\n",
       "      <td>2017</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095776</th>\n",
       "      <td>0</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>108</td>\n",
       "      <td>M_ID_15446e939c</td>\n",
       "      <td>-9</td>\n",
       "      <td>0.307953</td>\n",
       "      <td>...</td>\n",
       "      <td>34</td>\n",
       "      <td>2017</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095777</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>C</td>\n",
       "      <td>45</td>\n",
       "      <td>M_ID_84e60960a3</td>\n",
       "      <td>-6</td>\n",
       "      <td>-0.559076</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>2017</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>0.106023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095778</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>755</td>\n",
       "      <td>M_ID_57df19bf28</td>\n",
       "      <td>-7</td>\n",
       "      <td>-0.709342</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>2017</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.074568</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095918</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_9400cf2342</td>\n",
       "      <td>-2</td>\n",
       "      <td>-0.701753</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>2017</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095919</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_5357547c3e</td>\n",
       "      <td>-4</td>\n",
       "      <td>-0.515500</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>2017</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095920</th>\n",
       "      <td>0</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_9400cf2342</td>\n",
       "      <td>-1</td>\n",
       "      <td>-0.521510</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>2018</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095921</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>705</td>\n",
       "      <td>M_ID_8261f7a734</td>\n",
       "      <td>-4</td>\n",
       "      <td>-0.586124</td>\n",
       "      <td>...</td>\n",
       "      <td>33</td>\n",
       "      <td>2017</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19095922</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>244</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>705</td>\n",
       "      <td>M_ID_69423b34e4</td>\n",
       "      <td>-11</td>\n",
       "      <td>-0.677936</td>\n",
       "      <td>...</td>\n",
       "      <td>33</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0.180375</td>\n",
       "      <td>-0.404556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>149 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          authorized_flag          card_id  city_id  category_1  installments  \\\n",
       "19095774                1  C_ID_00007093c1      244           0             1   \n",
       "19095775                1  C_ID_00007093c1      244           0             1   \n",
       "19095776                0  C_ID_00007093c1      244           0             1   \n",
       "19095777                1  C_ID_00007093c1       76           0             3   \n",
       "19095778                1  C_ID_00007093c1       -1           1             1   \n",
       "...                   ...              ...      ...         ...           ...   \n",
       "19095918                1  C_ID_00007093c1      244           0             1   \n",
       "19095919                1  C_ID_00007093c1      244           0             1   \n",
       "19095920                0  C_ID_00007093c1      244           0             1   \n",
       "19095921                1  C_ID_00007093c1      244           0             1   \n",
       "19095922                1  C_ID_00007093c1      244           0             1   \n",
       "\n",
       "         category_3  merchant_category_id      merchant_id  month_lag  \\\n",
       "19095774          B                   307  M_ID_9400cf2342         -4   \n",
       "19095775          B                   705  M_ID_a33355a1b7        -12   \n",
       "19095776          B                   108  M_ID_15446e939c         -9   \n",
       "19095777          C                    45  M_ID_84e60960a3         -6   \n",
       "19095778          B                   755  M_ID_57df19bf28         -7   \n",
       "...             ...                   ...              ...        ...   \n",
       "19095918          B                   307  M_ID_9400cf2342         -2   \n",
       "19095919          B                   307  M_ID_5357547c3e         -4   \n",
       "19095920          B                   307  M_ID_9400cf2342         -1   \n",
       "19095921          B                   705  M_ID_8261f7a734         -4   \n",
       "19095922          B                   705  M_ID_69423b34e4        -11   \n",
       "\n",
       "          purchase_amount  ... subsector_id  year  month  dayofweek  \\\n",
       "19095774        -0.530526  ...           19  2017     10          0   \n",
       "19095775        -0.467415  ...           33  2017      2          1   \n",
       "19095776         0.307953  ...           34  2017      5          3   \n",
       "19095777        -0.559076  ...           18  2017      8          5   \n",
       "19095778        -0.709342  ...            8  2017      7          0   \n",
       "...                   ...  ...          ...   ...    ...        ...   \n",
       "19095918        -0.701753  ...           19  2017     12          2   \n",
       "19095919        -0.515500  ...           19  2017     10          4   \n",
       "19095920        -0.521510  ...           19  2018      1          0   \n",
       "19095921        -0.586124  ...           33  2017     10          0   \n",
       "19095922        -0.677936  ...           33  2017      3          5   \n",
       "\n",
       "          weekofyear  hour  weekend  month_diff  category_2_mean  \\\n",
       "19095774          40    19        0          12         0.180375   \n",
       "19095775           7    15        0          12         0.180375   \n",
       "19095776          20    16        0          12         0.180375   \n",
       "19095777          31    11        1          12         0.180375   \n",
       "19095778          27    15        0          12         0.074568   \n",
       "...              ...   ...      ...         ...              ...   \n",
       "19095918          50    20        0          12         0.180375   \n",
       "19095919          40    21        0          12         0.180375   \n",
       "19095920           2    18        0          12         0.180375   \n",
       "19095921          40    17        0          12         0.180375   \n",
       "19095922          10    15        1          12         0.180375   \n",
       "\n",
       "          category_3_mean  \n",
       "19095774        -0.404556  \n",
       "19095775        -0.404556  \n",
       "19095776        -0.404556  \n",
       "19095777         0.106023  \n",
       "19095778        -0.404556  \n",
       "...                   ...  \n",
       "19095918        -0.404556  \n",
       "19095919        -0.404556  \n",
       "19095920        -0.404556  \n",
       "19095921        -0.404556  \n",
       "19095922        -0.404556  \n",
       "\n",
       "[149 rows x 23 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans[df_hist_trans['card_id']=='C_ID_00007093c1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hist_trans_group.columns = new_columns\n",
    "df_hist_trans_group.reset_index(drop=False,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- calculate the difference between min and max purchase date\n",
    "- calculate the ratio of purchase duration over number of transactions\n",
    "- calculate the duration of now and last purchase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 325540 entries, 0 to 325539\n",
      "Data columns (total 35 columns):\n",
      "card_id                              325540 non-null object\n",
      "hist_month_nunique                   325540 non-null int64\n",
      "hist_hour_nunique                    325540 non-null int64\n",
      "hist_weekofyear_nunique              325540 non-null int64\n",
      "hist_dayofweek_nunique               325540 non-null int64\n",
      "hist_year_nunique                    325540 non-null int64\n",
      "hist_subsector_id_nunique            325540 non-null int64\n",
      "hist_merchant_id_nunique             325540 non-null int64\n",
      "hist_merchant_category_id_nunique    325540 non-null int64\n",
      "hist_purchase_amount_sum             325540 non-null float64\n",
      "hist_purchase_amount_max             325540 non-null float64\n",
      "hist_purchase_amount_min             325540 non-null float64\n",
      "hist_purchase_amount_mean            325540 non-null float64\n",
      "hist_purchase_amount_var             325540 non-null float64\n",
      "hist_installments_sum                325540 non-null int64\n",
      "hist_installments_max                325540 non-null int64\n",
      "hist_installments_min                325540 non-null int64\n",
      "hist_installments_mean               325540 non-null float64\n",
      "hist_installments_var                325540 non-null float64\n",
      "hist_purchase_date_max               325540 non-null object\n",
      "hist_purchase_date_min               325540 non-null object\n",
      "hist_month_lag_max                   325540 non-null int64\n",
      "hist_month_lag_min                   325540 non-null int64\n",
      "hist_month_lag_mean                  325540 non-null float64\n",
      "hist_month_lag_var                   325540 non-null float64\n",
      "hist_month_diff_mean                 325540 non-null float64\n",
      "hist_authorized_flag_sum             325540 non-null int64\n",
      "hist_authorized_flag_mean            325540 non-null float64\n",
      "hist_weekend_sum                     325540 non-null int64\n",
      "hist_weekend_mean                    325540 non-null float64\n",
      "hist_category_1_sum                  325540 non-null int64\n",
      "hist_category_1_mean                 325540 non-null float64\n",
      "hist_card_id_size                    325540 non-null int64\n",
      "hist_category_2_mean_mean            325540 non-null float64\n",
      "hist_category_3_mean_mean            325540 non-null float64\n",
      "dtypes: float64(15), int64(17), object(3)\n",
      "memory usage: 86.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df_hist_trans_group.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         377\n",
       "1         151\n",
       "2         398\n",
       "3         283\n",
       "4         202\n",
       "         ... \n",
       "325535     89\n",
       "325536    199\n",
       "325537    260\n",
       "325538    410\n",
       "325539    188\n",
       "Length: 325540, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(pd.to_datetime(df_hist_trans_group['hist_purchase_date_max'])- pd.to_datetime(df_hist_trans_group['hist_purchase_date_min'])).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hist_trans_group['hist_purchase_date_diff'] = (pd.to_datetime(df_hist_trans_group['hist_purchase_date_max'])- pd.to_datetime(df_hist_trans_group['hist_purchase_date_min'])).dt.days\n",
    "df_hist_trans_group['hist_purchase_date_average'] = df_hist_trans_group['hist_purchase_date_diff']/df_hist_trans_group['hist_card_id_size']\n",
    "df_hist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - pd.to_datetime(df_hist_trans_group['hist_purchase_date_max'])).dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "252"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\n",
    "df_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\n",
    "del df_hist_trans_group;gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do the same for new transactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in ['category_2','category_3']:\n",
    "    df_new_merchant_trans[col+'_mean'] = df_new_merchant_trans.groupby([col])['purchase_amount'].transform('mean')\n",
    "    aggs[col+'_mean'] = ['mean']\n",
    "    \n",
    "new_columns = get_new_columns('new_hist',aggs)\n",
    "df_hist_trans_group = df_new_merchant_trans.groupby('card_id').agg(aggs)\n",
    "df_hist_trans_group.columns = new_columns\n",
    "df_hist_trans_group.reset_index(drop=False,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hist_trans_group['new_hist_purchase_date_diff'] = (pd.to_datetime(df_hist_trans_group['new_hist_purchase_date_max']) - pd.to_datetime(df_hist_trans_group['new_hist_purchase_date_min'])).dt.days\n",
    "df_hist_trans_group['new_hist_purchase_date_average'] = df_hist_trans_group['new_hist_purchase_date_diff']/df_hist_trans_group['new_hist_card_id_size']\n",
    "df_hist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - pd.to_datetime(df_hist_trans_group['new_hist_purchase_date_max'])).dt.days\n",
    "df_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\n",
    "df_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\n",
    "del df_hist_trans_group;gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_active_month</th>\n",
       "      <th>card_id</th>\n",
       "      <th>feature_1</th>\n",
       "      <th>feature_2</th>\n",
       "      <th>feature_3</th>\n",
       "      <th>target</th>\n",
       "      <th>hist_month_nunique</th>\n",
       "      <th>hist_hour_nunique</th>\n",
       "      <th>hist_weekofyear_nunique</th>\n",
       "      <th>hist_dayofweek_nunique</th>\n",
       "      <th>...</th>\n",
       "      <th>new_hist_weekend_sum</th>\n",
       "      <th>new_hist_weekend_mean</th>\n",
       "      <th>new_hist_category_1_sum</th>\n",
       "      <th>new_hist_category_1_mean</th>\n",
       "      <th>new_hist_card_id_size</th>\n",
       "      <th>new_hist_category_2_mean_mean</th>\n",
       "      <th>new_hist_category_3_mean_mean</th>\n",
       "      <th>new_hist_purchase_date_diff</th>\n",
       "      <th>new_hist_purchase_date_average</th>\n",
       "      <th>new_hist_purchase_date_uptonow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-06</td>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.820283</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.0</td>\n",
       "      <td>-0.550160</td>\n",
       "      <td>-0.592993</td>\n",
       "      <td>54.0</td>\n",
       "      <td>2.347826</td>\n",
       "      <td>938.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-01</td>\n",
       "      <td>C_ID_3d0044924f</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.392913</td>\n",
       "      <td>12</td>\n",
       "      <td>24</td>\n",
       "      <td>50</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.0</td>\n",
       "      <td>-0.550160</td>\n",
       "      <td>-0.606486</td>\n",
       "      <td>56.0</td>\n",
       "      <td>9.333333</td>\n",
       "      <td>968.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-08</td>\n",
       "      <td>C_ID_d639edf6cd</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.688056</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.549015</td>\n",
       "      <td>-0.592993</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>938.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-09</td>\n",
       "      <td>C_ID_186d6a6901</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142495</td>\n",
       "      <td>6</td>\n",
       "      <td>16</td>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-0.556518</td>\n",
       "      <td>-0.604559</td>\n",
       "      <td>41.0</td>\n",
       "      <td>5.857143</td>\n",
       "      <td>949.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-11</td>\n",
       "      <td>C_ID_cdbd2c0db2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.159749</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.055556</td>\n",
       "      <td>36.0</td>\n",
       "      <td>-0.555446</td>\n",
       "      <td>-0.588217</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.583333</td>\n",
       "      <td>938.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_active_month          card_id  feature_1  feature_2  feature_3  \\\n",
       "0            2017-06  C_ID_92a2005557          5          2          1   \n",
       "1            2017-01  C_ID_3d0044924f          4          1          0   \n",
       "2            2016-08  C_ID_d639edf6cd          2          2          0   \n",
       "3            2017-09  C_ID_186d6a6901          4          3          0   \n",
       "4            2017-11  C_ID_cdbd2c0db2          1          3          0   \n",
       "\n",
       "     target  hist_month_nunique  hist_hour_nunique  hist_weekofyear_nunique  \\\n",
       "0 -0.820283                   9                 23                       35   \n",
       "1  0.392913                  12                 24                       50   \n",
       "2  0.688056                  10                 14                       22   \n",
       "3  0.142495                   6                 16                       20   \n",
       "4 -0.159749                   4                 22                       17   \n",
       "\n",
       "   hist_dayofweek_nunique  ...  new_hist_weekend_sum  new_hist_weekend_mean  \\\n",
       "0                       7  ...                   6.0               0.260870   \n",
       "1                       7  ...                   0.0               0.000000   \n",
       "2                       7  ...                   1.0               1.000000   \n",
       "3                       7  ...                   3.0               0.428571   \n",
       "4                       7  ...                  12.0               0.333333   \n",
       "\n",
       "   new_hist_category_1_sum  new_hist_category_1_mean  new_hist_card_id_size  \\\n",
       "0                      0.0                  0.000000                   23.0   \n",
       "1                      0.0                  0.000000                    6.0   \n",
       "2                      0.0                  0.000000                    1.0   \n",
       "3                      1.0                  0.142857                    7.0   \n",
       "4                      2.0                  0.055556                   36.0   \n",
       "\n",
       "   new_hist_category_2_mean_mean  new_hist_category_3_mean_mean  \\\n",
       "0                      -0.550160                      -0.592993   \n",
       "1                      -0.550160                      -0.606486   \n",
       "2                      -0.549015                      -0.592993   \n",
       "3                      -0.556518                      -0.604559   \n",
       "4                      -0.555446                      -0.588217   \n",
       "\n",
       "   new_hist_purchase_date_diff  new_hist_purchase_date_average  \\\n",
       "0                         54.0                        2.347826   \n",
       "1                         56.0                        9.333333   \n",
       "2                          0.0                        0.000000   \n",
       "3                         41.0                        5.857143   \n",
       "4                         57.0                        1.583333   \n",
       "\n",
       "   new_hist_purchase_date_uptonow  \n",
       "0                           938.0  \n",
       "1                           968.0  \n",
       "2                           938.0  \n",
       "3                           949.0  \n",
       "4                           938.0  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_hist_trans;gc.collect()\n",
    "del df_new_merchant_trans;gc.collect()\n",
    "df_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [df_train,df_test]:\n",
    "    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n",
    "    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n",
    "    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n",
    "    df['month'] = df['first_active_month'].dt.month\n",
    "    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n",
    "    df['hist_purchase_date_max'] = pd.to_datetime(df['hist_purchase_date_max'])\n",
    "    df['hist_purchase_date_min'] = pd.to_datetime(df['hist_purchase_date_min'])\n",
    "    df['new_hist_purchase_date_max'] = pd.to_datetime(df['new_hist_purchase_date_max'])\n",
    "    df['new_hist_purchase_date_min'] = pd.to_datetime(df['new_hist_purchase_date_min'])\n",
    "    \n",
    "    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n",
    "    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n",
    "    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n",
    "                     'new_hist_purchase_date_min']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9 #a way to change it from datetime to a numeric value that the model could use\n",
    "    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']\n",
    "    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    199710\n",
       "1      2207\n",
       "Name: outliers, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['outliers'] = 0\n",
    "df_train.loc[df_train['target'] < -30, 'outliers'] = 1\n",
    "df_train['outliers'].value_counts()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "replace feature 1 with the mean of each value of feature 1: the smaller the mean, the less outliers it has"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in ['feature_1','feature_2','feature_3']:\n",
    "    order_label = df_train.groupby([f])['outliers'].mean()\n",
    "    df_train[f] = df_train[f].map(order_label)\n",
    "    df_test[f] = df_test[f].map(order_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\n",
    "target = df_train['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'target'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2896\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2897\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'target'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-57-a8d000bb52e6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'target'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2993\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2994\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2995\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2996\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2997\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2897\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'target'"
     ]
    }
   ],
   "source": [
    "df_train['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['feature_1',\n",
       " 'feature_2',\n",
       " 'feature_3',\n",
       " 'hist_month_nunique',\n",
       " 'hist_hour_nunique',\n",
       " 'hist_weekofyear_nunique',\n",
       " 'hist_dayofweek_nunique',\n",
       " 'hist_year_nunique',\n",
       " 'hist_subsector_id_nunique',\n",
       " 'hist_merchant_id_nunique',\n",
       " 'hist_merchant_category_id_nunique',\n",
       " 'hist_purchase_amount_sum',\n",
       " 'hist_purchase_amount_max',\n",
       " 'hist_purchase_amount_min',\n",
       " 'hist_purchase_amount_mean',\n",
       " 'hist_purchase_amount_var',\n",
       " 'hist_installments_sum',\n",
       " 'hist_installments_max',\n",
       " 'hist_installments_min',\n",
       " 'hist_installments_mean',\n",
       " 'hist_installments_var',\n",
       " 'hist_purchase_date_max',\n",
       " 'hist_purchase_date_min',\n",
       " 'hist_month_lag_max',\n",
       " 'hist_month_lag_min',\n",
       " 'hist_month_lag_mean',\n",
       " 'hist_month_lag_var',\n",
       " 'hist_month_diff_mean',\n",
       " 'hist_authorized_flag_sum',\n",
       " 'hist_authorized_flag_mean',\n",
       " 'hist_weekend_sum',\n",
       " 'hist_weekend_mean',\n",
       " 'hist_category_1_sum',\n",
       " 'hist_category_1_mean',\n",
       " 'hist_card_id_size',\n",
       " 'hist_category_2_mean_mean',\n",
       " 'hist_category_3_mean_mean',\n",
       " 'hist_purchase_date_diff',\n",
       " 'hist_purchase_date_average',\n",
       " 'hist_purchase_date_uptonow',\n",
       " 'new_hist_month_nunique',\n",
       " 'new_hist_hour_nunique',\n",
       " 'new_hist_weekofyear_nunique',\n",
       " 'new_hist_dayofweek_nunique',\n",
       " 'new_hist_year_nunique',\n",
       " 'new_hist_subsector_id_nunique',\n",
       " 'new_hist_merchant_id_nunique',\n",
       " 'new_hist_merchant_category_id_nunique',\n",
       " 'new_hist_purchase_amount_sum',\n",
       " 'new_hist_purchase_amount_max',\n",
       " 'new_hist_purchase_amount_min',\n",
       " 'new_hist_purchase_amount_mean',\n",
       " 'new_hist_purchase_amount_var',\n",
       " 'new_hist_installments_sum',\n",
       " 'new_hist_installments_max',\n",
       " 'new_hist_installments_min',\n",
       " 'new_hist_installments_mean',\n",
       " 'new_hist_installments_var',\n",
       " 'new_hist_purchase_date_max',\n",
       " 'new_hist_purchase_date_min',\n",
       " 'new_hist_month_lag_max',\n",
       " 'new_hist_month_lag_min',\n",
       " 'new_hist_month_lag_mean',\n",
       " 'new_hist_month_lag_var',\n",
       " 'new_hist_month_diff_mean',\n",
       " 'new_hist_authorized_flag_sum',\n",
       " 'new_hist_authorized_flag_mean',\n",
       " 'new_hist_weekend_sum',\n",
       " 'new_hist_weekend_mean',\n",
       " 'new_hist_category_1_sum',\n",
       " 'new_hist_category_1_mean',\n",
       " 'new_hist_card_id_size',\n",
       " 'new_hist_category_2_mean_mean',\n",
       " 'new_hist_category_3_mean_mean',\n",
       " 'new_hist_purchase_date_diff',\n",
       " 'new_hist_purchase_date_average',\n",
       " 'new_hist_purchase_date_uptonow',\n",
       " 'dayofweek',\n",
       " 'weekofyear',\n",
       " 'month',\n",
       " 'elapsed_time',\n",
       " 'hist_first_buy',\n",
       " 'new_hist_first_buy',\n",
       " 'card_id_total',\n",
       " 'purchase_amount_total']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df_train['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train2 = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['target'] = df_train2['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_active_month</th>\n",
       "      <th>card_id</th>\n",
       "      <th>feature_1</th>\n",
       "      <th>feature_2</th>\n",
       "      <th>feature_3</th>\n",
       "      <th>hist_month_nunique</th>\n",
       "      <th>hist_hour_nunique</th>\n",
       "      <th>hist_weekofyear_nunique</th>\n",
       "      <th>hist_dayofweek_nunique</th>\n",
       "      <th>hist_year_nunique</th>\n",
       "      <th>...</th>\n",
       "      <th>dayofweek</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>month</th>\n",
       "      <th>elapsed_time</th>\n",
       "      <th>hist_first_buy</th>\n",
       "      <th>new_hist_first_buy</th>\n",
       "      <th>card_id_total</th>\n",
       "      <th>purchase_amount_total</th>\n",
       "      <th>outliers</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-06-01</td>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "      <td>0.013145</td>\n",
       "      <td>0.008752</td>\n",
       "      <td>0.011428</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>6</td>\n",
       "      <td>1270</td>\n",
       "      <td>26</td>\n",
       "      <td>277.0</td>\n",
       "      <td>283.0</td>\n",
       "      <td>-179.212942</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.820283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>C_ID_3d0044924f</td>\n",
       "      <td>0.010712</td>\n",
       "      <td>0.011385</td>\n",
       "      <td>0.010283</td>\n",
       "      <td>12</td>\n",
       "      <td>24</td>\n",
       "      <td>50</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>1421</td>\n",
       "      <td>5</td>\n",
       "      <td>396.0</td>\n",
       "      <td>356.0</td>\n",
       "      <td>-214.362071</td>\n",
       "      <td>0</td>\n",
       "      <td>0.392913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-08-01</td>\n",
       "      <td>C_ID_d639edf6cd</td>\n",
       "      <td>0.010610</td>\n",
       "      <td>0.008752</td>\n",
       "      <td>0.010283</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>8</td>\n",
       "      <td>1574</td>\n",
       "      <td>163</td>\n",
       "      <td>635.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>-29.867717</td>\n",
       "      <td>0</td>\n",
       "      <td>0.688056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-09-01</td>\n",
       "      <td>C_ID_186d6a6901</td>\n",
       "      <td>0.010712</td>\n",
       "      <td>0.014166</td>\n",
       "      <td>0.010283</td>\n",
       "      <td>6</td>\n",
       "      <td>16</td>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>35</td>\n",
       "      <td>9</td>\n",
       "      <td>1178</td>\n",
       "      <td>25</td>\n",
       "      <td>187.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>-54.145736</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-11-01</td>\n",
       "      <td>C_ID_cdbd2c0db2</td>\n",
       "      <td>0.008058</td>\n",
       "      <td>0.014166</td>\n",
       "      <td>0.010283</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>44</td>\n",
       "      <td>11</td>\n",
       "      <td>1117</td>\n",
       "      <td>11</td>\n",
       "      <td>121.0</td>\n",
       "      <td>169.0</td>\n",
       "      <td>-68.613893</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.159749</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_active_month          card_id  feature_1  feature_2  feature_3  \\\n",
       "0         2017-06-01  C_ID_92a2005557   0.013145   0.008752   0.011428   \n",
       "1         2017-01-01  C_ID_3d0044924f   0.010712   0.011385   0.010283   \n",
       "2         2016-08-01  C_ID_d639edf6cd   0.010610   0.008752   0.010283   \n",
       "3         2017-09-01  C_ID_186d6a6901   0.010712   0.014166   0.010283   \n",
       "4         2017-11-01  C_ID_cdbd2c0db2   0.008058   0.014166   0.010283   \n",
       "\n",
       "   hist_month_nunique  hist_hour_nunique  hist_weekofyear_nunique  \\\n",
       "0                   9                 23                       35   \n",
       "1                  12                 24                       50   \n",
       "2                  10                 14                       22   \n",
       "3                   6                 16                       20   \n",
       "4                   4                 22                       17   \n",
       "\n",
       "   hist_dayofweek_nunique  hist_year_nunique  ...  dayofweek  weekofyear  \\\n",
       "0                       7                  2  ...          3          22   \n",
       "1                       7                  2  ...          6          52   \n",
       "2                       7                  2  ...          0          31   \n",
       "3                       7                  2  ...          4          35   \n",
       "4                       7                  2  ...          2          44   \n",
       "\n",
       "   month  elapsed_time  hist_first_buy  new_hist_first_buy  card_id_total  \\\n",
       "0      6          1270              26               277.0          283.0   \n",
       "1      1          1421               5               396.0          356.0   \n",
       "2      8          1574             163               635.0           44.0   \n",
       "3      9          1178              25               187.0           84.0   \n",
       "4     11          1117              11               121.0          169.0   \n",
       "\n",
       "   purchase_amount_total  outliers    target  \n",
       "0            -179.212942         0 -0.820283  \n",
       "1            -214.362071         0  0.392913  \n",
       "2             -29.867717         0  0.688056  \n",
       "3             -54.145736         0  0.142495  \n",
       "4             -68.613893         0 -0.159749  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv('data/processed_train_'+ pd.datetime.now().strftime(\"%d%m%Y_%H%M\") + '.csv',index=False)\n",
    "df_test.to_csv('data/processed_test_'+ pd.datetime.now().strftime(\"%d%m%Y_%H%M\") + '.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917, 89)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.outliers.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}