darius-model-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "pk5WiopT9oJw",
    "outputId": "f81ff94a-3399-4a56-9365-6c5e54d80782"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: You are using pip version 20.1; however, version 20.1.1 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "# installing catboost\n",
    "# Catboost == 0.22 was the version of catboost at the start of this competition\n",
    "!pip install catboost==0.22 --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "id": "jVXG4QlTuVYr"
   },
   "outputs": [],
   "source": [
    "# Importing libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lightgbm import LGBMRegressor\n",
    "from xgboost import XGBRegressor, XGBRFRegressor\n",
    "from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.utils import shuffle\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "from functools import reduce\n",
    "from catboost import CatBoostRegressor, CatBoostClassifier\n",
    "import joblib\n",
    "\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "id": "agGrUcYwQ7on",
    "outputId": "5d4d760c-135f-4e6e-ae14-bc275b174734"
   },
   "outputs": [],
   "source": [
    "# Loading data\n",
    "train = pd.read_csv('./input/Train.csv')\n",
    "test = pd.read_csv('./input/Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "id": "jNB_PDTNpxu4"
   },
   "outputs": [],
   "source": [
    "# Feature interaction functions\n",
    "# There are 4 types of interactions: product interactions, division interactions, sum interactions and divide interactions\n",
    "\n",
    "def add_prod_interacts(df, inter_cols):   \n",
    "    def apply_interacts(x, inter_cols):\n",
    "        cols = [x + '_prod_' + c for c in inter_cols[inter_cols.index(x)+1:]]\n",
    "        interacts_df[cols] = pd.concat([df[x] * df[c] for c in inter_cols[inter_cols.index(x)+1:]], axis=1)\n",
    "    \n",
    "    interacts_df = pd.DataFrame()\n",
    "    _ = df[inter_cols[:-1]].apply(lambda x: apply_interacts(x.name, inter_cols))\n",
    "    df = pd.concat([df, interacts_df], axis=1)\n",
    "    return df\n",
    "\n",
    "\n",
    "def add_div_interacts(df, inter_cols):   \n",
    "    def apply_interacts(x, inter_cols):\n",
    "        cols = [x + '_div_' + c for c in inter_cols[inter_cols.index(x)+1:]]\n",
    "        interacts_df[cols] = pd.concat([df[x] / df[c] for c in inter_cols[inter_cols.index(x)+1:]], axis=1)\n",
    "    \n",
    "    interacts_df = pd.DataFrame()\n",
    "    _ = df[inter_cols[:-1]].apply(lambda x: apply_interacts(x.name, inter_cols))\n",
    "    df = pd.concat([df, interacts_df], axis=1)\n",
    "\n",
    "    return df\n",
    "\n",
    "def add_sum_interacts(df, inter_cols):   \n",
    "    def apply_interacts(x, inter_cols):\n",
    "        cols = [x + '_sum_' + c for c in inter_cols[inter_cols.index(x)+1:]]\n",
    "        interacts_df[cols] = pd.concat([df[x] + df[c] for c in inter_cols[inter_cols.index(x)+1:]], axis=1)\n",
    "    \n",
    "    interacts_df = pd.DataFrame()\n",
    "    _ = df[inter_cols[:-1]].apply(lambda x: apply_interacts(x.name, inter_cols))\n",
    "    df = pd.concat([df, interacts_df], axis=1)\n",
    "\n",
    "    return df\n",
    "\n",
    "def add_diff_interacts(df, inter_cols):   \n",
    "    def apply_interacts(x, inter_cols):\n",
    "        cols = [x + '_diff_' + c for c in inter_cols[inter_cols.index(x)+1:]]\n",
    "        interacts_df[cols] = pd.concat([df[x] - df[c] for c in inter_cols[inter_cols.index(x)+1:]], axis=1)\n",
    "    \n",
    "    interacts_df = pd.DataFrame()\n",
    "    _ = df[inter_cols[:-1]].apply(lambda x: apply_interacts(x.name, inter_cols))\n",
    "    df = pd.concat([df, interacts_df], axis=1)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "id": "VJjtuM3kvCAT"
   },
   "outputs": [],
   "source": [
    "# Loading data\n",
    "train = pd.read_csv('./input/Train.csv')\n",
    "test = pd.read_csv('./input/Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "id": "4MhDKOpYF9dm"
   },
   "outputs": [],
   "source": [
    "# Separating the target variable from the training dataframe\n",
    "#\n",
    "target = train.target\n",
    "\n",
    "# Aligning the train and test dataframes\n",
    "#\n",
    "train, test = train.align(test, join = 'inner',axis = 1)\n",
    "\n",
    "# Creating a separator column to both train and test,\n",
    "# This is to be used in separation\n",
    "#\n",
    "train['separator'] = 0\n",
    "test['separator'] = 1\n",
    "# Combing the train and test dataframes together\n",
    "#\n",
    "comb = pd.concat([train, test])\n",
    "\n",
    "# Creating a function to replace all spaces in the dataframe with np.nan\n",
    "#\n",
    "def replace_nan(x):\n",
    "    if x==\" \":\n",
    "        return np.nan\n",
    "    else :\n",
    "        return float(x)\n",
    "\n",
    "# Creating a list of the main columns\n",
    "#\n",
    "main_cols = [\"temp\",\"precip\",\"rel_humidity\",\"wind_dir\",\"wind_spd\",\"atmos_press\"]\n",
    "\n",
    "# Replacing spaces with np.nan\n",
    "#\n",
    "for col in main_cols: \n",
    "    comb[col]=comb[col].apply(lambda x: [ replace_nan(X) for X in x.replace(\"nan\",\" \").split(\",\")])\n",
    "\n",
    "def make_columns(feature):\n",
    "    return [f\"{feature}_{i}\" for i in range(1, 122)]\n",
    "    \n",
    "# Generating dataframes of hours for each main column\n",
    "#\n",
    "comb_temp         = pd.DataFrame([x for x in comb.temp],         columns=make_columns('temp'))\n",
    "comb_precip       = pd.DataFrame([x for x in comb.precip],       columns=make_columns('precip'))\n",
    "comb_rel_humidity = pd.DataFrame([x for x in comb.rel_humidity], columns=make_columns('rel_humidity'))\n",
    "comb_wind_dir     = pd.DataFrame([x for x in comb.wind_dir],     columns=make_columns('wind_dir'))\n",
    "comb_wind_spd     = pd.DataFrame([x for x in comb.wind_spd],     columns=make_columns('wind_spd'))\n",
    "comb_atmos_press  = pd.DataFrame([x for x in comb.atmos_press],  columns=make_columns('atmos_press'))\n",
    "\n",
    "comb_temp['ID'], comb_precip['ID'], comb_rel_humidity['ID'], comb_wind_dir['ID'], comb_wind_spd['ID'], comb_atmos_press['ID'] = [list(comb.ID)] * 6\n",
    "\n",
    "# Combining the generated dataframes together\n",
    "#\n",
    "comb_dfs = [comb, comb_temp, comb_precip, comb_rel_humidity, comb_wind_dir, comb_wind_spd, comb_atmos_press]\n",
    "comb = reduce(lambda  left,right: pd.merge(left,right,on=['ID'], how='outer'), comb_dfs)\n",
    "comb.drop(main_cols, axis = 1, inplace = True)\n",
    "df = comb.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "id": "pEyFVf1lqInY"
   },
   "outputs": [],
   "source": [
    "comb = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "xzn_w6FYftB8",
    "outputId": "9c6b5360-766f-4c1b-c223-c00ceaa834f9"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "999f9adfcfd444af978f061a15a74ea8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Adding aggregation features for each variable\n",
    "#\n",
    "aggs = ['mean', 'std', 'var', 'kurt', 'skew', 'max', 'median', 'sum', 'mode', 'sem', 'min']\n",
    "for col in tqdm_notebook(main_cols):\n",
    "  for ag in tqdm_notebook(aggs, leave = False):\n",
    "    if ag == 'mode':\n",
    "      comb[col[0] + col[-1] + '_'+ag] = comb[[x for x in comb.columns if x.startswith(col)]].agg(ag, axis = 1)[0]\n",
    "    else:\n",
    "      comb[col[0] + col[-1] + '_'+ag] = comb[[x for x in comb.columns if x.startswith(col)]].agg(ag, axis = 1)\n",
    "\n",
    "# Creating separate dataframes for each variable\n",
    "# Creating a list of columns for each separate dataframe\n",
    "#\n",
    "temp, temp_cols  = comb[[x for x in comb.columns if x.startswith('temp')]], [x for x in comb.columns if x.startswith('temp')]\n",
    "precip, precip_cols = comb[[x for x in comb.columns if x.startswith('precip')]], [x for x in comb.columns if x.startswith('precip')]\n",
    "humid, humid_cols = comb[[x for x in comb.columns if x.startswith('rel_humidity')]], [x for x in comb.columns if x.startswith('rel_humidity')]\n",
    "wind_dir, wind_dir_cols = comb[[x for x in comb.columns if x.startswith('wind_dir')]], [x for x in comb.columns if x.startswith('wind_dir')]\n",
    "wind_spd, wind_spd_cols = comb[[x for x in comb.columns if x.startswith('wind_spd')]], [x for x in comb.columns if x.startswith('wind_spd')]\n",
    "atmp, atmp_cols = comb[[x for x in comb.columns if x.startswith('atmos_press')]], [x for x in comb.columns if x.startswith('atmos_press')]\n",
    "fill_cols = comb.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "XHCE-yW1r3q8",
    "outputId": "dd3d6870-4a06-4794-c94c-eba320fa9335"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>location</th>\n",
       "      <th>separator</th>\n",
       "      <th>temp_1</th>\n",
       "      <th>temp_2</th>\n",
       "      <th>temp_3</th>\n",
       "      <th>temp_4</th>\n",
       "      <th>temp_5</th>\n",
       "      <th>temp_6</th>\n",
       "      <th>temp_7</th>\n",
       "      <th>...</th>\n",
       "      <th>as_std</th>\n",
       "      <th>as_var</th>\n",
       "      <th>as_kurt</th>\n",
       "      <th>as_skew</th>\n",
       "      <th>as_max</th>\n",
       "      <th>as_median</th>\n",
       "      <th>as_sum</th>\n",
       "      <th>as_mode</th>\n",
       "      <th>as_sem</th>\n",
       "      <th>as_min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_train_0</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.072682</td>\n",
       "      <td>0.005283</td>\n",
       "      <td>-0.158696</td>\n",
       "      <td>-0.383144</td>\n",
       "      <td>87.871667</td>\n",
       "      <td>87.762083</td>\n",
       "      <td>1404.038939</td>\n",
       "      <td>87.614167</td>\n",
       "      <td>0.018170</td>\n",
       "      <td>87.614167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_train_1</td>\n",
       "      <td>D</td>\n",
       "      <td>0</td>\n",
       "      <td>22.533333</td>\n",
       "      <td>21.716667</td>\n",
       "      <td>20.833333</td>\n",
       "      <td>20.983333</td>\n",
       "      <td>20.875000</td>\n",
       "      <td>20.141667</td>\n",
       "      <td>19.375000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.156648</td>\n",
       "      <td>0.024539</td>\n",
       "      <td>-0.446340</td>\n",
       "      <td>-0.173356</td>\n",
       "      <td>90.725000</td>\n",
       "      <td>90.429167</td>\n",
       "      <td>10942.020833</td>\n",
       "      <td>90.219167</td>\n",
       "      <td>0.014241</td>\n",
       "      <td>90.056667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_train_10</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>28.975000</td>\n",
       "      <td>27.950000</td>\n",
       "      <td>29.600000</td>\n",
       "      <td>26.425000</td>\n",
       "      <td>22.091667</td>\n",
       "      <td>21.775000</td>\n",
       "      <td>22.333333</td>\n",
       "      <td>...</td>\n",
       "      <td>0.180233</td>\n",
       "      <td>0.032484</td>\n",
       "      <td>-0.227481</td>\n",
       "      <td>-0.243561</td>\n",
       "      <td>88.813333</td>\n",
       "      <td>88.425000</td>\n",
       "      <td>10610.511667</td>\n",
       "      <td>88.287500</td>\n",
       "      <td>0.016453</td>\n",
       "      <td>87.982500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_train_100</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>22.966667</td>\n",
       "      <td>24.266667</td>\n",
       "      <td>25.275000</td>\n",
       "      <td>25.625000</td>\n",
       "      <td>25.866667</td>\n",
       "      <td>25.091667</td>\n",
       "      <td>24.025000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.162430</td>\n",
       "      <td>0.026384</td>\n",
       "      <td>-0.462889</td>\n",
       "      <td>-0.347610</td>\n",
       "      <td>88.685000</td>\n",
       "      <td>88.400000</td>\n",
       "      <td>10693.606667</td>\n",
       "      <td>88.271667</td>\n",
       "      <td>0.014766</td>\n",
       "      <td>87.965000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_train_1000</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>21.875000</td>\n",
       "      <td>21.575000</td>\n",
       "      <td>21.525000</td>\n",
       "      <td>21.433333</td>\n",
       "      <td>20.508333</td>\n",
       "      <td>19.916667</td>\n",
       "      <td>18.991667</td>\n",
       "      <td>...</td>\n",
       "      <td>0.120393</td>\n",
       "      <td>0.014494</td>\n",
       "      <td>-0.062557</td>\n",
       "      <td>-0.705667</td>\n",
       "      <td>88.719167</td>\n",
       "      <td>88.552500</td>\n",
       "      <td>2656.143106</td>\n",
       "      <td>88.268333</td>\n",
       "      <td>0.021981</td>\n",
       "      <td>88.268333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 795 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              ID location  separator     temp_1     temp_2     temp_3  \\\n",
       "0     ID_train_0        C          0        NaN        NaN        NaN   \n",
       "1     ID_train_1        D          0  22.533333  21.716667  20.833333   \n",
       "2    ID_train_10        A          0  28.975000  27.950000  29.600000   \n",
       "3   ID_train_100        A          0  22.966667  24.266667  25.275000   \n",
       "4  ID_train_1000        A          0  21.875000  21.575000  21.525000   \n",
       "\n",
       "      temp_4     temp_5     temp_6     temp_7  ...    as_std    as_var  \\\n",
       "0        NaN        NaN        NaN        NaN  ...  0.072682  0.005283   \n",
       "1  20.983333  20.875000  20.141667  19.375000  ...  0.156648  0.024539   \n",
       "2  26.425000  22.091667  21.775000  22.333333  ...  0.180233  0.032484   \n",
       "3  25.625000  25.866667  25.091667  24.025000  ...  0.162430  0.026384   \n",
       "4  21.433333  20.508333  19.916667  18.991667  ...  0.120393  0.014494   \n",
       "\n",
       "    as_kurt   as_skew     as_max  as_median        as_sum    as_mode  \\\n",
       "0 -0.158696 -0.383144  87.871667  87.762083   1404.038939  87.614167   \n",
       "1 -0.446340 -0.173356  90.725000  90.429167  10942.020833  90.219167   \n",
       "2 -0.227481 -0.243561  88.813333  88.425000  10610.511667  88.287500   \n",
       "3 -0.462889 -0.347610  88.685000  88.400000  10693.606667  88.271667   \n",
       "4 -0.062557 -0.705667  88.719167  88.552500   2656.143106  88.268333   \n",
       "\n",
       "     as_sem     as_min  \n",
       "0  0.018170  87.614167  \n",
       "1  0.014241  90.056667  \n",
       "2  0.016453  87.982500  \n",
       "3  0.014766  87.965000  \n",
       "4  0.021981  88.268333  \n",
       "\n",
       "[5 rows x 795 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Previewing the head of the generated dataframe\n",
    "#\n",
    "comb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "v4DlbAzKp_c8",
    "outputId": "26c4e9d8-68ae-4ed1-8e9a-e9a009f7343c"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27712189b04b46b68969a0482c5243a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=795.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=3.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Creating a list of columns containing aggregates only\n",
    "cols_mean = []\n",
    "cols_max = []\n",
    "cols_min = []\n",
    "for x in tqdm_notebook(comb.columns):\n",
    "  if 'mean' in x:\n",
    "    cols_mean.append(x)\n",
    "  elif 'max' in x:\n",
    "    cols_max.append(x)\n",
    "  elif 'min' in x:\n",
    "    cols_min.append(x)\n",
    "  else:\n",
    "    pass\n",
    "\n",
    "# Generating feature interactions between aggregates only\n",
    "for num_cols in tqdm_notebook([cols_mean, cols_max, cols_min], leave = False):\n",
    "    comb = add_prod_interacts(comb, num_cols)\n",
    "    comb = add_div_interacts(comb, num_cols)\n",
    "    comb = add_diff_interacts(comb, num_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "id": "qdVrf3ZmcFZn"
   },
   "outputs": [],
   "source": [
    "# Generating new features, by adding each variable per hour\n",
    "for x, y, z, a, b in zip(temp.columns, precip.columns, humid.columns, wind_spd.columns, atmp.columns):\n",
    "  comb['add_tp' +y[-4:]] = temp[x] + precip[y] + humid[z] + wind_spd[a] + atmp[b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true,
    "id": "IU2Z7uDigrgW"
   },
   "outputs": [],
   "source": [
    "# Filling missing values using forward fill\n",
    "comb = comb.ffill(axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true,
    "id": "YnsfoPv_R5yN"
   },
   "outputs": [],
   "source": [
    "# Calculating the difference between features per each dataframe\n",
    "\n",
    "dfs = [temp, precip, humid, wind_spd, atmp]\n",
    "\n",
    "diff_dfs = []\n",
    "for i in range(5):\n",
    "  i = dfs[i]\n",
    "  name = str(i.columns[0].split('_')[0])\n",
    "  temp_df = i.diff(axis = 1).values\n",
    "  temp_df = pd.DataFrame(temp_df, columns=['diff_' +name + '_' + str(i) for i in range(1, 122)])\n",
    "  diff_dfs.append(temp_df)\n",
    "\n",
    "diff_dfs.append(comb)\n",
    "comb = reduce(lambda  left,right: pd.merge(left,right, right_index=True, left_index=True, how='outer'), diff_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "TUeOD_AMUMyQ",
    "outputId": "ebd84e15-b073-45aa-d096-c2f270ce2e10"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diff_temp_1</th>\n",
       "      <th>diff_temp_2</th>\n",
       "      <th>diff_temp_3</th>\n",
       "      <th>diff_temp_4</th>\n",
       "      <th>diff_temp_5</th>\n",
       "      <th>diff_temp_6</th>\n",
       "      <th>diff_temp_7</th>\n",
       "      <th>diff_temp_8</th>\n",
       "      <th>diff_temp_9</th>\n",
       "      <th>diff_temp_10</th>\n",
       "      <th>...</th>\n",
       "      <th>add_tp_112</th>\n",
       "      <th>add_tp_113</th>\n",
       "      <th>add_tp_114</th>\n",
       "      <th>add_tp_115</th>\n",
       "      <th>add_tp_116</th>\n",
       "      <th>add_tp_117</th>\n",
       "      <th>add_tp_118</th>\n",
       "      <th>add_tp_119</th>\n",
       "      <th>add_tp_120</th>\n",
       "      <th>add_tp_121</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>111.009</td>\n",
       "      <td>110.396</td>\n",
       "      <td>109.991</td>\n",
       "      <td>110.031</td>\n",
       "      <td>110.085</td>\n",
       "      <td>109.618</td>\n",
       "      <td>109.161</td>\n",
       "      <td>108.627</td>\n",
       "      <td>108.839</td>\n",
       "      <td>109.081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.816667</td>\n",
       "      <td>-0.883333</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>-0.108333</td>\n",
       "      <td>-0.733333</td>\n",
       "      <td>-0.766667</td>\n",
       "      <td>-0.583333</td>\n",
       "      <td>-0.016667</td>\n",
       "      <td>-0.133333</td>\n",
       "      <td>...</td>\n",
       "      <td>118.268</td>\n",
       "      <td>119.334</td>\n",
       "      <td>120.191</td>\n",
       "      <td>122.078</td>\n",
       "      <td>122.734</td>\n",
       "      <td>123.072</td>\n",
       "      <td>121.885</td>\n",
       "      <td>119.338</td>\n",
       "      <td>118.589</td>\n",
       "      <td>114.357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.025000</td>\n",
       "      <td>1.650000</td>\n",
       "      <td>-3.175000</td>\n",
       "      <td>-4.333333</td>\n",
       "      <td>-0.316667</td>\n",
       "      <td>0.558333</td>\n",
       "      <td>-0.383333</td>\n",
       "      <td>-1.508333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>111.332</td>\n",
       "      <td>110.389</td>\n",
       "      <td>110.174</td>\n",
       "      <td>110.092</td>\n",
       "      <td>110.746</td>\n",
       "      <td>113.401</td>\n",
       "      <td>116.527</td>\n",
       "      <td>118.449</td>\n",
       "      <td>119.659</td>\n",
       "      <td>120.692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.300000</td>\n",
       "      <td>1.008333</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>0.241667</td>\n",
       "      <td>-0.775000</td>\n",
       "      <td>-1.066667</td>\n",
       "      <td>-1.600000</td>\n",
       "      <td>-2.191667</td>\n",
       "      <td>-2.816667</td>\n",
       "      <td>...</td>\n",
       "      <td>109.088</td>\n",
       "      <td>108.214</td>\n",
       "      <td>108.176</td>\n",
       "      <td>107.477</td>\n",
       "      <td>107.332</td>\n",
       "      <td>108.305</td>\n",
       "      <td>110.405</td>\n",
       "      <td>112.255</td>\n",
       "      <td>114.158</td>\n",
       "      <td>115.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.300000</td>\n",
       "      <td>-0.050000</td>\n",
       "      <td>-0.091667</td>\n",
       "      <td>-0.925000</td>\n",
       "      <td>-0.591667</td>\n",
       "      <td>-0.925000</td>\n",
       "      <td>-0.400000</td>\n",
       "      <td>-0.541667</td>\n",
       "      <td>-0.133333</td>\n",
       "      <td>...</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "      <td>110.649</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1656 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   diff_temp_1  diff_temp_2  diff_temp_3  diff_temp_4  diff_temp_5  \\\n",
       "0          NaN          NaN          NaN          NaN          NaN   \n",
       "1          NaN    -0.816667    -0.883333     0.150000    -0.108333   \n",
       "2          NaN    -1.025000     1.650000    -3.175000    -4.333333   \n",
       "3          NaN     1.300000     1.008333     0.350000     0.241667   \n",
       "4          NaN    -0.300000    -0.050000    -0.091667    -0.925000   \n",
       "\n",
       "   diff_temp_6  diff_temp_7  diff_temp_8  diff_temp_9  diff_temp_10  ...  \\\n",
       "0          NaN          NaN          NaN          NaN           NaN  ...   \n",
       "1    -0.733333    -0.766667    -0.583333    -0.016667     -0.133333  ...   \n",
       "2    -0.316667     0.558333    -0.383333    -1.508333      0.000000  ...   \n",
       "3    -0.775000    -1.066667    -1.600000    -2.191667     -2.816667  ...   \n",
       "4    -0.591667    -0.925000    -0.400000    -0.541667     -0.133333  ...   \n",
       "\n",
       "   add_tp_112  add_tp_113  add_tp_114  add_tp_115  add_tp_116  add_tp_117  \\\n",
       "0     111.009     110.396     109.991     110.031     110.085     109.618   \n",
       "1     118.268     119.334     120.191     122.078     122.734     123.072   \n",
       "2     111.332     110.389     110.174     110.092     110.746     113.401   \n",
       "3     109.088     108.214     108.176     107.477     107.332     108.305   \n",
       "4     110.649     110.649     110.649     110.649     110.649     110.649   \n",
       "\n",
       "   add_tp_118  add_tp_119  add_tp_120  add_tp_121  \n",
       "0     109.161     108.627     108.839     109.081  \n",
       "1     121.885     119.338     118.589     114.357  \n",
       "2     116.527     118.449     119.659     120.692  \n",
       "3     110.405     112.255     114.158      115.68  \n",
       "4     110.649     110.649     110.649     110.649  \n",
       "\n",
       "[5 rows x 1656 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def apply_qcut(feat):\n",
    "    return pd.qcut(comb[feat], 24, labels=False, duplicates='drop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "Z1eJlRcDUdrq",
    "outputId": "4b5cf990-88bb-4147-b6cc-27f8f9af991a"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3567c0b91c18424a881e581f27602992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1653.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 2.11 s, sys: 1.36 s, total: 3.48 s\n",
      "Wall time: 18.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Binning feaures\n",
    "other_features = [x for x in comb.columns if x not in ['separator', 'ID', 'location']]\n",
    "\n",
    "# Multiprocessing trick: 15 seconds instead of 7 minutes !\n",
    "binned_data = joblib.Parallel(n_jobs=-1, backend='multiprocessing')(\n",
    "    joblib.delayed(apply_qcut)(feat) for feat in tqdm_notebook(other_features))\n",
    "\n",
    "comb_binned_data = pd.concat(binned_data, axis=1)\n",
    "comb = pd.concat([comb[['separator', 'ID', 'location']], comb_binned_data], axis=1)\n",
    "\n",
    "# Separating train and test from the combined dataframe\n",
    "train = comb[comb.separator == 0]\n",
    "test = comb[comb.separator == 1]\n",
    "train.drop('separator', axis = 1, inplace = True)\n",
    "test.drop('separator', axis = 1, inplace = True)\n",
    "\n",
    "# Creating a list of test ids in the order that they will be trained\n",
    "testA =  test[test.location == 'A']\n",
    "testB =  test[test.location == 'B']\n",
    "testC =  test[test.location == 'C']\n",
    "testD =  test[test.location == 'D']\n",
    "testE =  test[test.location == 'E']\n",
    "\n",
    "tA, tD, tE, tBC = testA.ID, testD.ID, testE.ID, test[(test.location == 'B') | (test.location == 'C')].ID\n",
    "test_id = pd.concat([tA, tD, tE, tBC])\n",
    "\n",
    "# Adding back target to the train set\n",
    "train['target'] = target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "95P-K-ZljXYv",
    "outputId": "0e8d98eb-fbcb-4100-a4a8-fc434398a399"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd94cfe7d8b84087bd429024bb637184",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=25.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 2h 35min 4s, sys: 5min 2s, total: 2h 40min 7s\n",
      "Wall time: 42min 26s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Creating X and y values\n",
    "X = train.drop(['ID', 'location', 'target'], axis = 1)\n",
    "y = target.values\n",
    "\n",
    "# Shuffling the X, y values\n",
    "X, y = shuffle(X, y, random_state = 0)\n",
    "tes = test.drop(['ID', 'location'], axis = 1)\n",
    "\n",
    "# Traing the model across multiple seeds\n",
    "predictions = []\n",
    "for i in tqdm_notebook(range(25)):\n",
    "  cat = CatBoostRegressor(verbose = False, random_seed=i)\n",
    "  cat.fit(X, y)\n",
    "\n",
    "  preds = cat.predict(tes)\n",
    "  predictions.append(preds)\n",
    "\n",
    "# Averaging the predictions\n",
    "avg_preds = np.mean(predictions, axis = 0)\n",
    "\n",
    "# Post processing of the predictions\n",
    "post_proc = [((((((((((x-0.85)*1.015)-0.85)*1.012)-0.75)*1.0095)-0.55)*1.0065)-0.8)*1.007) for x in avg_preds]\n",
    "post_proc = predzz = [((x-0.85)*1.015) for x in post_proc]\n",
    "\n",
    "# Creating a submission file\n",
    "sub_df = pd.DataFrame({'ID': test.ID, 'target':post_proc})\n",
    "sub_df.to_csv('model_2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15539</th>\n",
       "      <td>ID_test_0</td>\n",
       "      <td>154.512605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15540</th>\n",
       "      <td>ID_test_1</td>\n",
       "      <td>117.225585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15541</th>\n",
       "      <td>ID_test_10</td>\n",
       "      <td>26.247779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15542</th>\n",
       "      <td>ID_test_100</td>\n",
       "      <td>63.167372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15543</th>\n",
       "      <td>ID_test_1000</td>\n",
       "      <td>92.044408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15544</th>\n",
       "      <td>ID_test_1001</td>\n",
       "      <td>41.451966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15545</th>\n",
       "      <td>ID_test_1002</td>\n",
       "      <td>84.857269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15546</th>\n",
       "      <td>ID_test_1003</td>\n",
       "      <td>37.807807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15547</th>\n",
       "      <td>ID_test_1004</td>\n",
       "      <td>30.820292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15548</th>\n",
       "      <td>ID_test_1005</td>\n",
       "      <td>45.990773</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID      target\n",
       "15539     ID_test_0  154.512605\n",
       "15540     ID_test_1  117.225585\n",
       "15541    ID_test_10   26.247779\n",
       "15542   ID_test_100   63.167372\n",
       "15543  ID_test_1000   92.044408\n",
       "15544  ID_test_1001   41.451966\n",
       "15545  ID_test_1002   84.857269\n",
       "15546  ID_test_1003   37.807807\n",
       "15547  ID_test_1004   30.820292\n",
       "15548  ID_test_1005   45.990773"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}