resnet-2keras-lgb(submit).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-02-21T18:17:20.269086Z",
     "iopub.status.busy": "2021-02-21T18:17:20.268500Z",
     "iopub.status.idle": "2021-02-21T18:17:27.403468Z",
     "shell.execute_reply": "2021-02-21T18:17:27.402414Z"
    },
    "papermill": {
     "duration": 7.160109,
     "end_time": "2021-02-21T18:17:27.403667",
     "exception": false,
     "start_time": "2021-02-21T18:17:20.243558",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from torch.nn.modules.loss import _WeightedLoss\n",
    "import torch.nn.functional as F\n",
    "\n",
    "NFOLDS = 5\n",
    "\n",
    "CACHE_PATH = '../input/dropoutrate01hiddensize128'\n",
    "\n",
    "feat_cols = [f'feature_{i}' for i in range(130)]\n",
    "\n",
    "target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']\n",
    "\n",
    "f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')\n",
    "\n",
    "all_feat_cols = [col for col in feat_cols]\n",
    "all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))\n",
    "        self.dropout0 = nn.Dropout(0.1)#\n",
    "\n",
    "        dropout_rate = 0.1\n",
    "        hidden_size = 128\n",
    "        self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout1 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout2 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout3 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm4 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout4 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))\n",
    "\n",
    "        self.Relu = nn.ReLU(inplace=True)\n",
    "        self.PReLU = nn.PReLU()\n",
    "        self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)\n",
    "        self.RReLU = nn.RReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.batch_norm0(x)\n",
    "        x = self.dropout0(x)\n",
    "\n",
    "        x1 = self.dense1(x)\n",
    "        x1 = self.batch_norm1(x1)\n",
    "        x1 = self.LeakyReLU(x1)\n",
    "        x1 = self.dropout1(x1)\n",
    "\n",
    "        x = torch.cat([x, x1], 1)\n",
    "\n",
    "        x2 = self.dense2(x)\n",
    "        x2 = self.batch_norm2(x2)\n",
    "        x2 = self.LeakyReLU(x2)\n",
    "        x2 = self.dropout2(x2)\n",
    "\n",
    "        x = torch.cat([x1, x2], 1)\n",
    "\n",
    "        x3 = self.dense3(x)\n",
    "        x3 = self.batch_norm3(x3)\n",
    "        x3 = self.LeakyReLU(x3)\n",
    "        x3 = self.dropout3(x3)\n",
    "\n",
    "        x = torch.cat([x2, x3], 1)\n",
    "\n",
    "        x4 = self.dense4(x)\n",
    "        x4 = self.batch_norm4(x4)\n",
    "        x4 = self.LeakyReLU(x4)\n",
    "        x4 = self.dropout4(x4)\n",
    "\n",
    "        x = torch.cat([x3, x4], 1)\n",
    "\n",
    "        x = self.dense5(x)\n",
    "\n",
    "        return x\n",
    "    \n",
    "device = torch.device(\"cuda:0\")\n",
    "resnet_model_list = []\n",
    "for _fold in range(NFOLDS):\n",
    "    torch.cuda.empty_cache()\n",
    "    model = Model()\n",
    "    model.to(device)\n",
    "    model_weights = f\"{CACHE_PATH}/online_model{_fold}.pth\"\n",
    "    model.load_state_dict(torch.load(model_weights))\n",
    "    model.eval()\n",
    "    resnet_model_list.append(model)\n",
    "    \n",
    "resnet_model_list=resnet_model_list[-2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T18:17:27.417400Z",
     "iopub.status.busy": "2021-02-21T18:17:27.416742Z",
     "iopub.status.idle": "2021-02-21T18:17:27.420738Z",
     "shell.execute_reply": "2021-02-21T18:17:27.420252Z"
    },
    "papermill": {
     "duration": 0.012156,
     "end_time": "2021-02-21T18:17:27.420850",
     "exception": false,
     "start_time": "2021-02-21T18:17:27.408694",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation\n",
    "# from tensorflow.keras.models import Model, Sequential\n",
    "# from tensorflow.keras.losses import BinaryCrossentropy\n",
    "# from tensorflow.keras.optimizers import Adam\n",
    "# from tensorflow.keras.callbacks import EarlyStopping\n",
    "# from tensorflow.keras.layers.experimental.preprocessing import Normalization\n",
    "# import tensorflow as tf\n",
    "# import numpy as np\n",
    "# import pandas as pd\n",
    "# from sklearn.model_selection import GroupKFold\n",
    "\n",
    "# from tqdm import tqdm\n",
    "# from random import choices\n",
    "# import random\n",
    "\n",
    "# import kerastuner as kt\n",
    "# keras_f_mean = np.load('../input/kerasfmeanonline/keras_f_mean_online.npy')\n",
    "\n",
    "# def create_autoencoder(input_dim,output_dim,noise=0.05):\n",
    "#     i = Input(input_dim)\n",
    "#     encoded = BatchNormalization()(i)\n",
    "#     encoded = GaussianNoise(noise)(encoded)\n",
    "#     encoded = Dense(64,activation='relu')(encoded)\n",
    "#     decoded = Dropout(0.2)(encoded)\n",
    "#     decoded = Dense(input_dim,name='decoded')(decoded)\n",
    "#     x = Dense(32,activation='relu')(decoded)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dropout(0.2)(x)\n",
    "#     #\n",
    "#     x = Dense(32,activation='relu')(decoded)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dropout(0.2)(x)\n",
    "#     #\n",
    "#     x = Dense(output_dim,activation='sigmoid',name='label_output')(x)\n",
    "    \n",
    "#     encoder = Model(inputs=i,outputs=[decoded,x])\n",
    "#     autoencoder = Model(inputs=i,outputs=[decoded,x])\n",
    "    \n",
    "#     autoencoder.compile(optimizer=Adam(0.005),loss={'decoded':'mse','label_output':'binary_crossentropy'})\n",
    "#     return autoencoder, encoder\n",
    "\n",
    "# def create_model(hp,input_dim,output_dim,encoder):\n",
    "#     inputs = Input(input_dim)\n",
    "    \n",
    "#     x = encoder(inputs)\n",
    "#     x = Concatenate()([x,inputs]) \n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dropout(hp.Float('init_dropout',0.0,0.5,step=0.01,default=0.25))(x)#\n",
    "    \n",
    "#     for i in range(hp.Int('num_layers',1,3)):\n",
    "#         x = Dense(hp.Int(f'num_units_{i}',64,256))(x)\n",
    "#         x = BatchNormalization()(x)\n",
    "#         x = Lambda(tf.keras.activations.swish)(x)\n",
    "#         x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5,step=0.01,default=0.25))(x)#\n",
    "#     x = Dense(output_dim,activation='sigmoid')(x)\n",
    "#     model = Model(inputs=inputs,outputs=x)\n",
    "#     model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.0005)),loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1,default=0.05)),metrics=[tf.keras.metrics.AUC(name = 'auc')])#\n",
    "#     return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T18:17:27.435060Z",
     "iopub.status.busy": "2021-02-21T18:17:27.434501Z",
     "iopub.status.idle": "2021-02-21T18:17:32.474357Z",
     "shell.execute_reply": "2021-02-21T18:17:32.473343Z"
    },
    "papermill": {
     "duration": 5.049118,
     "end_time": "2021-02-21T18:17:32.474503",
     "exception": false,
     "start_time": "2021-02-21T18:17:27.425385",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "from tensorflow.keras.losses import BinaryCrossentropy\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.layers.experimental.preprocessing import Normalization\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from random import choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T18:17:32.488090Z",
     "iopub.status.busy": "2021-02-21T18:17:32.487489Z",
     "iopub.status.idle": "2021-02-21T18:17:32.497715Z",
     "shell.execute_reply": "2021-02-21T18:17:32.497018Z"
    },
    "papermill": {
     "duration": 0.018272,
     "end_time": "2021-02-21T18:17:32.497847",
     "exception": false,
     "start_time": "2021-02-21T18:17:32.479575",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "keras_f_mean = np.load('../input/kerasfmeanonline/keras_f_mean_online.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T18:17:32.511493Z",
     "iopub.status.busy": "2021-02-21T18:17:32.510830Z",
     "iopub.status.idle": "2021-02-21T18:17:35.118325Z",
     "shell.execute_reply": "2021-02-21T18:17:35.116790Z"
    },
    "papermill": {
     "duration": 2.615572,
     "end_time": "2021-02-21T18:17:35.118470",
     "exception": false,
     "start_time": "2021-02-21T18:17:32.502898",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "keras_model = load_model('../input/newkeras/model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-21T18:17:35.144031Z",
     "iopub.status.busy": "2021-02-21T18:17:35.142083Z",
     "iopub.status.idle": "2021-02-21T18:22:36.193639Z",
     "shell.execute_reply": "2021-02-21T18:22:36.194031Z"
    },
    "papermill": {
     "duration": 301.070778,
     "end_time": "2021-02-21T18:22:36.194251",
     "exception": false,
     "start_time": "2021-02-21T18:17:35.123473",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15219it [04:53, 51.77it/s] \n"
     ]
    }
   ],
   "source": [
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from random import choices\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgbm\n",
    "\n",
    "features = [col for col in feat_cols]\n",
    "features.extend(['feature_stock_id_sum', 'feature_1_2_cross'])\n",
    "\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "lgb_models = []\n",
    "for i in range(5):\n",
    "    model_path = '../input/lgbm300055055001/lgb_model_'+str(i)+'.bin' #700 0.5 0.5 0.005\n",
    "    clf = pickle.load(open(model_path, 'rb'))\n",
    "    lgb_models.append(clf)\n",
    "    \n",
    "import janestreet\n",
    "env = janestreet.make_env()\n",
    "env_iter = env.iter_test()\n",
    "\n",
    "test_df_columns = ['weight'] + [f'feature_{i}' for i in range(130)] + ['date']\n",
    "index_features = [n for n, col in enumerate(test_df_columns) if col in feat_cols]\n",
    "\n",
    "import gc\n",
    "for (test_df, pred_df) in tqdm(env_iter):\n",
    "    if test_df['weight'].values[0]>0:\n",
    "   \n",
    "        resnet_x_tt = test_df.values[0][index_features].reshape(1, -1)\n",
    "        if np.isnan(resnet_x_tt.sum()):\n",
    "            resnet_x_tt = np.nan_to_num(resnet_x_tt) + np.isnan(resnet_x_tt) * f_mean\n",
    "        cross_41_42_43 =resnet_x_tt[:, 41] +resnet_x_tt[:, 42] +resnet_x_tt[:, 43]\n",
    "        cross_1_2 =resnet_x_tt[:, 1] / (resnet_x_tt[:, 2] + 1e-5)\n",
    "        feature_inp = np.concatenate((\n",
    "            resnet_x_tt,\n",
    "            np.array(cross_41_42_43).reshape(resnet_x_tt.shape[0], 1),\n",
    "            np.array(cross_1_2).reshape(resnet_x_tt.shape[0], 1),\n",
    "            ), axis=1)\n",
    "        resnet_pred = np.zeros((1, len(target_cols)))\n",
    "        for model in resnet_model_list:\n",
    "            resnet_pred += model(torch.tensor(feature_inp, dtype=torch.float).to(device)).sigmoid().detach().cpu().numpy() / 2\n",
    "        resnet_pred = np.median(resnet_pred)\n",
    "\n",
    "        keras_x_tt = test_df.values[0][index_features].reshape(1, -1)\n",
    "        if np.isnan(keras_x_tt.sum()):\n",
    "            keras_x_tt = np.nan_to_num(keras_x_tt) + np.isnan(keras_x_tt) * keras_f_mean\n",
    "        keras_pred = keras_model(keras_x_tt, training = False).numpy() \n",
    "        keras_pred =  np.median(keras_pred)\n",
    "  \n",
    "        test_df['feature_stock_id_sum'] = test_df['feature_41'] + test_df['feature_42'] + test_df['feature_43']\n",
    "        test_df['feature_1_2_cross'] = test_df['feature_1']/(test_df['feature_2']+1e-5)\n",
    "        test_df.fillna(-9999,inplace=True)\n",
    "        lgb_x_tt = test_df.loc[:, features].values\n",
    "        lgb_pred = np.median([model.predict(lgb_x_tt) for model in lgb_models])\n",
    "     \n",
    "        final_pred=(1/3)*resnet_pred +(1/3)*keras_pred+(1/3)*lgb_pred \n",
    "        pred_df.action = int(final_pred >= 0.499) #+-0.001\n",
    "    else:\n",
    "        pred_df['action'].values[0] = 0\n",
    "    env.predict(pred_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 324.853004,
   "end_time": "2021-02-21T18:22:40.051503",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-02-21T18:17:15.198499",
   "version": "2.2.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}