resnet-keras-4(submit).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:43.015165Z",
     "iopub.status.busy": "2021-02-21T07:04:43.014291Z",
     "iopub.status.idle": "2021-02-21T07:04:45.345903Z",
     "shell.execute_reply": "2021-02-21T07:04:45.344560Z"
    },
    "papermill": {
     "duration": 2.347805,
     "end_time": "2021-02-21T07:04:45.346146",
     "exception": false,
     "start_time": "2021-02-21T07:04:42.998341",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from torch.nn.modules.loss import _WeightedLoss\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:45.367859Z",
     "iopub.status.busy": "2021-02-21T07:04:45.367124Z",
     "iopub.status.idle": "2021-02-21T07:04:45.381344Z",
     "shell.execute_reply": "2021-02-21T07:04:45.380762Z"
    },
    "papermill": {
     "duration": 0.026482,
     "end_time": "2021-02-21T07:04:45.381479",
     "exception": false,
     "start_time": "2021-02-21T07:04:45.354997",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "NFOLDS = 5\n",
    "\n",
    "CACHE_PATH = '../input/dropoutrate01hiddensize128'\n",
    "\n",
    "feat_cols = [f'feature_{i}' for i in range(130)]\n",
    "\n",
    "target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']\n",
    "\n",
    "f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')\n",
    "\n",
    "all_feat_cols = [col for col in feat_cols]\n",
    "all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:45.413733Z",
     "iopub.status.busy": "2021-02-21T07:04:45.412722Z",
     "iopub.status.idle": "2021-02-21T07:04:45.415959Z",
     "shell.execute_reply": "2021-02-21T07:04:45.415440Z"
    },
    "papermill": {
     "duration": 0.027559,
     "end_time": "2021-02-21T07:04:45.416116",
     "exception": false,
     "start_time": "2021-02-21T07:04:45.388557",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))\n",
    "        self.dropout0 = nn.Dropout(0.1)\n",
    "\n",
    "        dropout_rate = 0.1\n",
    "        hidden_size = 128\n",
    "        self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout1 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout2 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout3 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm4 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout4 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))\n",
    "\n",
    "        self.Relu = nn.ReLU(inplace=True)\n",
    "        self.PReLU = nn.PReLU()\n",
    "        self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)\n",
    "        self.RReLU = nn.RReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.batch_norm0(x)\n",
    "        x = self.dropout0(x)\n",
    "\n",
    "        x1 = self.dense1(x)\n",
    "        x1 = self.batch_norm1(x1)\n",
    "        x1 = self.LeakyReLU(x1)\n",
    "        x1 = self.dropout1(x1)\n",
    "\n",
    "        x = torch.cat([x, x1], 1)\n",
    "\n",
    "        x2 = self.dense2(x)\n",
    "        x2 = self.batch_norm2(x2)\n",
    "        x2 = self.LeakyReLU(x2)\n",
    "        x2 = self.dropout2(x2)\n",
    "\n",
    "        x = torch.cat([x1, x2], 1)\n",
    "\n",
    "        x3 = self.dense3(x)\n",
    "        x3 = self.batch_norm3(x3)\n",
    "        x3 = self.LeakyReLU(x3)\n",
    "        x3 = self.dropout3(x3)\n",
    "\n",
    "        x = torch.cat([x2, x3], 1)\n",
    "\n",
    "        x4 = self.dense4(x)\n",
    "        x4 = self.batch_norm4(x4)\n",
    "        x4 = self.LeakyReLU(x4)\n",
    "        x4 = self.dropout4(x4)\n",
    "\n",
    "        x = torch.cat([x3, x4], 1)\n",
    "\n",
    "        x = self.dense5(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:45.439087Z",
     "iopub.status.busy": "2021-02-21T07:04:45.438316Z",
     "iopub.status.idle": "2021-02-21T07:04:50.430724Z",
     "shell.execute_reply": "2021-02-21T07:04:50.429666Z"
    },
    "papermill": {
     "duration": 5.007369,
     "end_time": "2021-02-21T07:04:50.430899",
     "exception": false,
     "start_time": "2021-02-21T07:04:45.423530",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\")\n",
    "resnet_model_list = []\n",
    "for _fold in range(NFOLDS):\n",
    "    torch.cuda.empty_cache()\n",
    "    model = Model()\n",
    "    model.to(device)\n",
    "    model_weights = f\"{CACHE_PATH}/online_model{_fold}.pth\"\n",
    "    model.load_state_dict(torch.load(model_weights))\n",
    "    model.eval()\n",
    "    resnet_model_list.append(model)\n",
    "    \n",
    "resnet_model_list=resnet_model_list[-2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:50.453719Z",
     "iopub.status.busy": "2021-02-21T07:04:50.452939Z",
     "iopub.status.idle": "2021-02-21T07:04:55.436184Z",
     "shell.execute_reply": "2021-02-21T07:04:55.435471Z"
    },
    "papermill": {
     "duration": 4.997107,
     "end_time": "2021-02-21T07:04:55.436401",
     "exception": false,
     "start_time": "2021-02-21T07:04:50.439294",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "from tensorflow.keras.losses import BinaryCrossentropy\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.layers.experimental.preprocessing import Normalization\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from random import choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:55.457432Z",
     "iopub.status.busy": "2021-02-21T07:04:55.456707Z",
     "iopub.status.idle": "2021-02-21T07:04:55.468607Z",
     "shell.execute_reply": "2021-02-21T07:04:55.467984Z"
    },
    "papermill": {
     "duration": 0.024382,
     "end_time": "2021-02-21T07:04:55.468773",
     "exception": false,
     "start_time": "2021-02-21T07:04:55.444391",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_f_mean = np.load('../input/kerasfmeanonline/keras_f_mean_online.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:55.487688Z",
     "iopub.status.busy": "2021-02-21T07:04:55.487000Z",
     "iopub.status.idle": "2021-02-21T07:04:58.070111Z",
     "shell.execute_reply": "2021-02-21T07:04:58.070947Z"
    },
    "papermill": {
     "duration": 2.594967,
     "end_time": "2021-02-21T07:04:58.071198",
     "exception": false,
     "start_time": "2021-02-21T07:04:55.476231",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "keras_model = load_model('../input/newkeras/model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T07:04:58.108375Z",
     "iopub.status.busy": "2021-02-21T07:04:58.107192Z",
     "iopub.status.idle": "2021-02-21T07:09:51.955117Z",
     "shell.execute_reply": "2021-02-21T07:09:51.954354Z"
    },
    "papermill": {
     "duration": 293.866894,
     "end_time": "2021-02-21T07:09:51.955393",
     "exception": false,
     "start_time": "2021-02-21T07:04:58.088499",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15219it [04:53, 51.80it/s]\n"
     ]
    }
   ],
   "source": [
    "import janestreet\n",
    "env = janestreet.make_env()\n",
    "env_iter = env.iter_test()\n",
    "\n",
    "test_df_columns = ['weight'] + [f'feature_{i}' for i in range(130)] + ['date']\n",
    "index_features = [n for n, col in enumerate(test_df_columns) if col in feat_cols]\n",
    "\n",
    "import gc\n",
    "for (test_df, pred_df) in tqdm(env_iter):\n",
    "    if test_df['weight'].values[0]>0:\n",
    "        \n",
    "        resnet_x_tt = test_df.values[0][index_features].reshape(1, -1)\n",
    "        if np.isnan(resnet_x_tt.sum()):\n",
    "            resnet_x_tt = np.nan_to_num(resnet_x_tt) + np.isnan(resnet_x_tt) * f_mean\n",
    "        cross_41_42_43 =resnet_x_tt[:, 41] +resnet_x_tt[:, 42] +resnet_x_tt[:, 43]\n",
    "        cross_1_2 =resnet_x_tt[:, 1] / (resnet_x_tt[:, 2] + 1e-5)\n",
    "        feature_inp = np.concatenate((\n",
    "            resnet_x_tt,\n",
    "            np.array(cross_41_42_43).reshape(resnet_x_tt.shape[0], 1),\n",
    "            np.array(cross_1_2).reshape(resnet_x_tt.shape[0], 1),\n",
    "            ), axis=1)\n",
    "        resnet_pred = np.zeros((1, len(target_cols)))\n",
    "        for model in resnet_model_list:\n",
    "            resnet_pred += model(torch.tensor(feature_inp, dtype=torch.float).to(device)).sigmoid().detach().cpu().numpy() / 2\n",
    "        resnet_pred = np.median(resnet_pred)\n",
    "        \n",
    "        keras_x_tt = test_df.values[0][index_features].reshape(1, -1)\n",
    "        if np.isnan(keras_x_tt.sum()):\n",
    "            keras_x_tt = np.nan_to_num(keras_x_tt) + np.isnan(keras_x_tt) * keras_f_mean\n",
    "        keras_pred = keras_model(keras_x_tt, training = False).numpy() \n",
    "        keras_pred =  np.median(keras_pred)\n",
    "        \n",
    "        final_pred = 0.48*resnet_pred + 0.52*keras_pred \n",
    "        pred_df.action = int(final_pred >= 0.5) \n",
    "    else:\n",
    "        pred_df['action'].values[0] = 0\n",
    "    env.predict(pred_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 318.7965,
   "end_time": "2021-02-21T07:09:56.148015",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-02-21T07:04:37.351515",
   "version": "2.2.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}