ensemble_baseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "ensemble-baseline.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "papermill": {
      "duration": 5955.149668,
      "end_time": "2020-11-30T01:40:42.062682",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2020-11-30T00:01:26.913014",
      "version": "2.1.0"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F0fdkwGB7OS3",
        "papermill": {
          "duration": 0.074782,
          "end_time": "2020-11-30T00:01:31.595038",
          "exception": false,
          "start_time": "2020-11-30T00:01:31.520256",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Strategy\n",
        "\n",
        "- Preprocessing\n",
        "    - Remove ctrl_vehicle\n",
        "    - RankGauss\n",
        "    - PCA + Existing Features\n",
        "    - KMeans\n",
        "    - Basic stats\n",
        "- Model\n",
        "    - Multi head ResNet (tensorflow)\n",
        "    - TabNet (pytorch)\n",
        "    - NODE - Neural Oblivious Decision Ensembles (tensorflow)\n",
        "- Training\n",
        "    - Pre-train with scored and non-scored target\n",
        "    - Train with public test pseudo label\n",
        "    - Add swap noise for train data\n",
        "    - Optimizer: Adam/AdamW with weight_decay\n",
        "    - Loss: BCE with Label smoothing + Logits\n",
        "- Prediction\n",
        "    - Ensemble above with weight optimization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uQVXKcKnx3VV",
        "papermill": {
          "duration": 0.073298,
          "end_time": "2020-11-30T00:01:32.057865",
          "exception": false,
          "start_time": "2020-11-30T00:01:31.984567",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gaR8v1Yr23Ml",
        "papermill": {
          "duration": 0.075069,
          "end_time": "2020-11-30T00:01:32.211052",
          "exception": false,
          "start_time": "2020-11-30T00:01:32.135983",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:32.364036Z",
          "iopub.status.busy": "2020-11-30T00:01:32.363178Z",
          "iopub.status.idle": "2020-11-30T00:01:32.366352Z",
          "shell.execute_reply": "2020-11-30T00:01:32.365860Z"
        },
        "id": "d6l6s1ka7OS5",
        "papermill": {
          "duration": 0.081463,
          "end_time": "2020-11-30T00:01:32.366473",
          "exception": false,
          "start_time": "2020-11-30T00:01:32.285010",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "import sys\n",
        "\n",
        "IN_COLAB = \"google.colab\" in sys.modules"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1JjflPsCGkyx",
        "papermill": {
          "duration": 0.072213,
          "end_time": "2020-11-30T00:01:32.510302",
          "exception": false,
          "start_time": "2020-11-30T00:01:32.438089",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "### Datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:32.661088Z",
          "iopub.status.busy": "2020-11-30T00:01:32.660166Z",
          "iopub.status.idle": "2020-11-30T00:01:32.663117Z",
          "shell.execute_reply": "2020-11-30T00:01:32.662507Z"
        },
        "id": "o0_nQy6bUb42",
        "papermill": {
          "duration": 0.082032,
          "end_time": "2020-11-30T00:01:32.663218",
          "exception": false,
          "start_time": "2020-11-30T00:01:32.581186",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "CTRL_VEHICLE = \"remove\"  # \"use\", \"keep\", \"remove\"\n",
        "# use: Use cp_type column for training.\n",
        "# keep: Remove cp_type column, but keep ctrl_vehicle row.\n",
        "# remove: Remove cp_type column and ctrl_vehicle row."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:32.813145Z",
          "iopub.status.busy": "2020-11-30T00:01:32.812234Z",
          "iopub.status.idle": "2020-11-30T00:01:32.814693Z",
          "shell.execute_reply": "2020-11-30T00:01:32.815172Z"
        },
        "id": "2aiL-BRW3Hfd",
        "papermill": {
          "duration": 0.080058,
          "end_time": "2020-11-30T00:01:32.815287",
          "exception": false,
          "start_time": "2020-11-30T00:01:32.735229",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "USE_PUBLIC_TEST_PSEUDO_LABEL = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2yVTOrEVGmjL",
        "papermill": {
          "duration": 0.071092,
          "end_time": "2020-11-30T00:01:32.957952",
          "exception": false,
          "start_time": "2020-11-30T00:01:32.886860",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "### Pre training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:33.200144Z",
          "iopub.status.busy": "2020-11-30T00:01:33.198985Z",
          "iopub.status.idle": "2020-11-30T00:01:33.201940Z",
          "shell.execute_reply": "2020-11-30T00:01:33.201081Z"
        },
        "id": "TqJFpiv74iX-",
        "papermill": {
          "duration": 0.12732,
          "end_time": "2020-11-30T00:01:33.202093",
          "exception": false,
          "start_time": "2020-11-30T00:01:33.074773",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "PRE_TRAIN_MODEL = \"load-others\"  # \"in-notebook\", \"load-others\", \"no\"\n",
        "# in-notebook: Pre-train in this notebook.\n",
        "# load-others: Load pre-train model that is trained in other notebook.\n",
        "# no: Disable pre-training.\n",
        "\n",
        "PRE_TRAIN_MODEL_DIR_RESNET = \"../input/pretrain-with-non-scored-target-baseline\"\n",
        "PRE_TRAIN_MODEL_DIR_TABNET = \"../input/pretrain-tabnet\"\n",
        "PRE_TRAIN_MODEL_DIR_NODE = \"../input/pretrain-node\"\n",
        "PRE_TRAIN_MODEL_DIR_NO_FIT = \"../input/ensemble-baseline-pre-training\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SAlO_jYUGpbn",
        "papermill": {
          "duration": 0.106476,
          "end_time": "2020-11-30T00:01:33.411893",
          "exception": false,
          "start_time": "2020-11-30T00:01:33.305417",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "### Hyper parameter tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:33.625161Z",
          "iopub.status.busy": "2020-11-30T00:01:33.624227Z",
          "iopub.status.idle": "2020-11-30T00:01:33.628263Z",
          "shell.execute_reply": "2020-11-30T00:01:33.628957Z"
        },
        "id": "e-zw-Hv822uk",
        "papermill": {
          "duration": 0.113552,
          "end_time": "2020-11-30T00:01:33.629137",
          "exception": false,
          "start_time": "2020-11-30T00:01:33.515585",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "HYPER_PARAMETER_TUNING = False\n",
        "\n",
        "if HYPER_PARAMETER_TUNING:\n",
        "    TUNING_RESNET = False\n",
        "    TUNING_TABNET = False\n",
        "    TUNING_NODE = True\n",
        "\n",
        "    USE_PUBLIC_TEST_PSEUDO_LABEL = False\n",
        "    PRE_TRAIN_MODEL = \"in-notebook\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N9iRNw4qGrry",
        "papermill": {
          "duration": 0.103695,
          "end_time": "2020-11-30T00:01:33.832906",
          "exception": false,
          "start_time": "2020-11-30T00:01:33.729211",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "### Model & KFold"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:34.002607Z",
          "iopub.status.busy": "2020-11-30T00:01:34.001680Z",
          "iopub.status.idle": "2020-11-30T00:01:34.004686Z",
          "shell.execute_reply": "2020-11-30T00:01:34.004197Z"
        },
        "id": "tL4Oe0sQHmud",
        "papermill": {
          "duration": 0.081594,
          "end_time": "2020-11-30T00:01:34.004788",
          "exception": false,
          "start_time": "2020-11-30T00:01:33.923194",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "DRUG_KFOLD = \"hard\"  # \"hard\", \"soft\"\n",
        "# hard: [vc <= 19]\n",
        "# soft: [(vc == 6) | (vc == 12) | (vc == 18)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:34.160225Z",
          "iopub.status.busy": "2020-11-30T00:01:34.159341Z",
          "iopub.status.idle": "2020-11-30T00:01:34.162591Z",
          "shell.execute_reply": "2020-11-30T00:01:34.162053Z"
        },
        "id": "8OpG3f96GehI",
        "papermill": {
          "duration": 0.08585,
          "end_time": "2020-11-30T00:01:34.162694",
          "exception": false,
          "start_time": "2020-11-30T00:01:34.076844",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "models = [\n",
        "    #{\"model_name\": \"ResNet\", \"cv\": \"with_drug_id\", \"fit\": True},\n",
        "    {\"model_name\": \"ResNet\", \"cv\": \"without_drug_id\", \"fit\": True},\n",
        "    #{\"model_name\": \"TabNet\", \"cv\": \"with_drug_id\", \"fit\": True},\n",
        "    {\"model_name\": \"TabNet\", \"cv\": \"without_drug_id\", \"fit\": True},\n",
        "    #{\"model_name\": \"NODE\", \"cv\": \"with_drug_id\", \"fit\": True},\n",
        "    {\"model_name\": \"NODE\", \"cv\": \"without_drug_id\", \"fit\": True},\n",
        "\n",
        "    #{\"model_name\": \"ResNet\", \"cv\": \"with_drug_id\", \"fit\": False},\n",
        "    #{\"model_name\": \"ResNet\", \"cv\": \"without_drug_id\", \"fit\": False},\n",
        "    #{\"model_name\": \"TabNet\", \"cv\": \"with_drug_id\", \"fit\": False},\n",
        "    #{\"model_name\": \"TabNet\", \"cv\": \"without_drug_id\", \"fit\": False},\n",
        "]\n",
        "\n",
        "N_SEED = 2\n",
        "N_STARTS = len(models) * N_SEED\n",
        "N_SPLITS = 7\n",
        "pre_train_models = [\"ResNet\", \"TabNet\", \"NODE\"]\n",
        "\n",
        "if IN_COLAB:\n",
        "    models = [\n",
        "        {\"model_name\": \"ResNet\", \"cv\": \"with_drug_id\", \"fit\": True},\n",
        "        #{\"model_name\": \"ResNet\", \"cv\": \"without_drug_id\", \"fit\": True},\n",
        "        {\"model_name\": \"TabNet\", \"cv\": \"with_drug_id\", \"fit\": True},\n",
        "        #{\"model_name\": \"TabNet\", \"cv\": \"without_drug_id\", \"fit\": True},\n",
        "        {\"model_name\": \"NODE\", \"cv\": \"with_drug_id\", \"fit\": True},\n",
        "        #{\"model_name\": \"NODE\", \"cv\": \"without_drug_id\", \"fit\": True},\n",
        "\n",
        "        #{\"model_name\": \"ResNet\", \"cv\": \"with_drug_id\", \"fit\": False},\n",
        "        #{\"model_name\": \"ResNet\", \"cv\": \"without_drug_id\", \"fit\": False},\n",
        "        #{\"model_name\": \"TabNet\", \"cv\": \"with_drug_id\", \"fit\": False},\n",
        "        #{\"model_name\": \"TabNet\", \"cv\": \"without_drug_id\", \"fit\": False},\n",
        "    ]\n",
        "\n",
        "    N_SEED = 3\n",
        "    N_STARTS = len(models) * N_SEED\n",
        "    N_SPLITS = 5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z8ZwGkFjIX9j",
        "papermill": {
          "duration": 0.072989,
          "end_time": "2020-11-30T00:01:34.310537",
          "exception": false,
          "start_time": "2020-11-30T00:01:34.237548",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:34.471089Z",
          "iopub.status.busy": "2020-11-30T00:01:34.470170Z",
          "iopub.status.idle": "2020-11-30T00:01:34.472925Z",
          "shell.execute_reply": "2020-11-30T00:01:34.472417Z"
        },
        "id": "xAYNK0IwIZ72",
        "papermill": {
          "duration": 0.086024,
          "end_time": "2020-11-30T00:01:34.473026",
          "exception": false,
          "start_time": "2020-11-30T00:01:34.387002",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "SAVE_WEIGHT = False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V1FRwKFHGxm_",
        "papermill": {
          "duration": 0.073782,
          "end_time": "2020-11-30T00:01:34.619317",
          "exception": false,
          "start_time": "2020-11-30T00:01:34.545535",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "### Cross Validation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:34.776278Z",
          "iopub.status.busy": "2020-11-30T00:01:34.775323Z",
          "iopub.status.idle": "2020-11-30T00:01:34.778596Z",
          "shell.execute_reply": "2020-11-30T00:01:34.778031Z"
        },
        "id": "Bxq-qlAvGBiv",
        "papermill": {
          "duration": 0.08242,
          "end_time": "2020-11-30T00:01:34.778705",
          "exception": false,
          "start_time": "2020-11-30T00:01:34.696285",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "optimize = \"average\"\n",
        "fixed_weight = [0.3, 0.2, 0.3, 0.2]  # if fixed\n",
        "\n",
        "if IN_COLAB:\n",
        "    optimize = \"lagrange\"  # \"lagrange\", \"fixed\", \"average\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZ1uTTKS7OS4",
        "papermill": {
          "duration": 0.072152,
          "end_time": "2020-11-30T00:01:34.924516",
          "exception": false,
          "start_time": "2020-11-30T00:01:34.852364",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## for Google Colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:35.108220Z",
          "iopub.status.busy": "2020-11-30T00:01:35.107348Z",
          "iopub.status.idle": "2020-11-30T00:01:35.109935Z",
          "shell.execute_reply": "2020-11-30T00:01:35.110478Z"
        },
        "id": "-u2KbV-T7OS8",
        "papermill": {
          "duration": 0.09925,
          "end_time": "2020-11-30T00:01:35.110611",
          "exception": false,
          "start_time": "2020-11-30T00:01:35.011361",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "COMPETE = \"lish-moa\"\n",
        "DATASETS = [\n",
        "    \"imokuri/moapublictestpredictions\",\n",
        "    \"optimo/pytorchtabnetpretraining\",\n",
        "    \"rahulsd91/moapublictest\",\n",
        "    \"tolgadincer/autograd\",\n",
        "    \"yasufuminakama/iterative-stratification\",\n",
        "]\n",
        "KERNEL_OUTPUTS = [\n",
        "    \"imokuri/pretrain-with-non-scored-target-baseline\",\n",
        "    \"imokuri/pretrain-tabnet\",\n",
        "    \"imokuri/pretrain-node\",\n",
        "]\n",
        "PACKAGES = [\"optuna\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:35.304284Z",
          "iopub.status.busy": "2020-11-30T00:01:35.299078Z",
          "iopub.status.idle": "2020-11-30T00:01:35.370424Z",
          "shell.execute_reply": "2020-11-30T00:01:35.369861Z"
        },
        "id": "MCI1aEn_7OTA",
        "papermill": {
          "duration": 0.183175,
          "end_time": "2020-11-30T00:01:35.370575",
          "exception": false,
          "start_time": "2020-11-30T00:01:35.187400",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if IN_COLAB:\n",
        "    !python2 -m pip uninstall kaggle -y\n",
        "    !python3 -m pip uninstall kaggle -y\n",
        "    !python3 -m pip install -U -q kaggle\n",
        "\n",
        "    !pip install -q -U git+https://github.com/IMOKURI/kaggle_on_google_colab.git\n",
        "\n",
        "    from kaggle_on_google_colab import setup\n",
        "    kaggle = setup.Setup()\n",
        "    kaggle.dirs(COMPETE)\n",
        "\n",
        "    !kaggle competitions download -p /content/zip {COMPETE}\n",
        "    !unzip -q -n /content/zip/{COMPETE}.zip -d /content/{COMPETE}/input/{COMPETE}\n",
        "    #for line in setup.exec_get_lines(cmd=f\"kaggle competitions files --csv {COMPETE} | egrep -v \\\"Warning: Looks like you're using an outdated API Version|name,size,creationDate\\\" | cut -d , -f 1\"):\n",
        "    #    !unzip -q -n /content/zip/{line.decode().strip()}.zip -d /content/{COMPETE}/input/{COMPETE}\n",
        "\n",
        "    for dataset in DATASETS:\n",
        "        dataset_name = dataset.split(\"/\")[-1]\n",
        "\n",
        "        !kaggle datasets download -p /content/zip {dataset}\n",
        "        !unzip -q -n /content/zip/{dataset_name}.zip -d /content/{COMPETE}/input/{dataset_name}\n",
        "\n",
        "    for kernel in KERNEL_OUTPUTS:\n",
        "        kernel_name = kernel.split(\"/\")[-1]\n",
        "\n",
        "        !kaggle kernels output -p /content/{COMPETE}/input/{kernel_name} {kernel}\n",
        "\n",
        "    for package_ in PACKAGES:\n",
        "        !pip install -q {package_}\n",
        "\n",
        "    !pip install -q -U tensorflow-addons\n",
        "    #!mv /content/zip/train_drug.csv /content/{COMPETE}/input/{COMPETE}/\n",
        "\n",
        "    %cd /content/{COMPETE}/output\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zfsIxBee7OTC",
        "papermill": {
          "duration": 0.072462,
          "end_time": "2020-11-30T00:01:35.516283",
          "exception": false,
          "start_time": "2020-11-30T00:01:35.443821",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:35.667712Z",
          "iopub.status.busy": "2020-11-30T00:01:35.666872Z",
          "iopub.status.idle": "2020-11-30T00:01:35.669949Z",
          "shell.execute_reply": "2020-11-30T00:01:35.669368Z"
        },
        "id": "ZSt3tVRO7OTD",
        "papermill": {
          "duration": 0.081502,
          "end_time": "2020-11-30T00:01:35.670076",
          "exception": false,
          "start_time": "2020-11-30T00:01:35.588574",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "import warnings\n",
        "\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:01:35.825921Z",
          "iopub.status.busy": "2020-11-30T00:01:35.825265Z",
          "iopub.status.idle": "2020-11-30T00:02:07.588302Z",
          "shell.execute_reply": "2020-11-30T00:02:07.587570Z"
        },
        "id": "Di-6b3xS7OTF",
        "papermill": {
          "duration": 31.845124,
          "end_time": "2020-11-30T00:02:07.588471",
          "exception": false,
          "start_time": "2020-11-30T00:01:35.743347",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "import sys\n",
        "\n",
        "sys.path.append(\"../input/iterative-stratification/iterative-stratification-master\")\n",
        "from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
        "\n",
        "sys.path.append(\"../input/autograd\")\n",
        "import autograd.numpy as np\n",
        "from autograd import grad\n",
        "\n",
        "#sys.path.append(\"../input/pytorchtabnetpretraining/pytorch_tabnet-2.0.1/pytorch_tabnet-2.0.1\")\n",
        "!pip install -q ../input/pytorchtabnetpretraining/pytorch_tabnet-2.0.1-py3-none-any.whl\n",
        "from pytorch_tabnet.metrics import Metric\n",
        "from pytorch_tabnet.pretraining import TabNetPretrainer\n",
        "from pytorch_tabnet.tab_model import TabNetRegressor"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:07.759848Z",
          "iopub.status.busy": "2020-11-30T00:02:07.758958Z",
          "iopub.status.idle": "2020-11-30T00:02:14.305128Z",
          "shell.execute_reply": "2020-11-30T00:02:14.304547Z"
        },
        "id": "PZIdM0rJ7OTH",
        "papermill": {
          "duration": 6.636517,
          "end_time": "2020-11-30T00:02:14.305277",
          "exception": false,
          "start_time": "2020-11-30T00:02:07.668760",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "import datetime\n",
        "import gc\n",
        "import json\n",
        "import io\n",
        "import itertools\n",
        "import os\n",
        "import random\n",
        "import zipfile\n",
        "from collections import defaultdict\n",
        "from time import time\n",
        "from typing import Optional\n",
        "\n",
        "# import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras.backend as K\n",
        "import tensorflow.keras.layers as L\n",
        "import tensorflow.keras.regularizers as R\n",
        "import tensorflow_addons as tfa\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from scipy.optimize import fsolve, minimize\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.decomposition import PCA, KernelPCA\n",
        "from sklearn.feature_selection import VarianceThreshold\n",
        "from sklearn.manifold import TSNE\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.preprocessing import QuantileTransformer\n",
        "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
        "from tensorflow_probability import distributions as tfp_distributions\n",
        "from tensorflow_probability import stats as tfp_stats\n",
        "from torch import nn\n",
        "from torch.nn.modules.loss import _WeightedLoss\n",
        "from torch.optim.lr_scheduler import ReduceLROnPlateau as torch_ReduceLROnPlateau"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:14.460678Z",
          "iopub.status.busy": "2020-11-30T00:02:14.459974Z",
          "iopub.status.idle": "2020-11-30T00:02:14.464710Z",
          "shell.execute_reply": "2020-11-30T00:02:14.464157Z"
        },
        "id": "pBCQf4N63tLr",
        "papermill": {
          "duration": 0.083908,
          "end_time": "2020-11-30T00:02:14.464837",
          "exception": false,
          "start_time": "2020-11-30T00:02:14.380929",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if HYPER_PARAMETER_TUNING:\n",
        "    import optuna"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:14.618059Z",
          "iopub.status.busy": "2020-11-30T00:02:14.617461Z",
          "iopub.status.idle": "2020-11-30T00:02:14.621250Z",
          "shell.execute_reply": "2020-11-30T00:02:14.620771Z"
        },
        "id": "Gn3NWKbmPWRj",
        "papermill": {
          "duration": 0.082428,
          "end_time": "2020-11-30T00:02:14.621348",
          "exception": false,
          "start_time": "2020-11-30T00:02:14.538920",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if IN_COLAB:\n",
        "    from IPython.display import SVG, display_svg\n",
        "    from tensorflow.keras.utils import model_to_dot"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:14.780073Z",
          "iopub.status.busy": "2020-11-30T00:02:14.779371Z",
          "iopub.status.idle": "2020-11-30T00:02:14.785968Z",
          "shell.execute_reply": "2020-11-30T00:02:14.785301Z"
        },
        "id": "nrm75bEQ7OTL",
        "papermill": {
          "duration": 0.087839,
          "end_time": "2020-11-30T00:02:14.786070",
          "exception": false,
          "start_time": "2020-11-30T00:02:14.698231",
          "status": "completed"
        },
        "tags": [],
        "outputId": "7c087246-fbdf-46d4-937d-dc060ab20637"
      },
      "source": [
        "MIXED_PRECISION = False\n",
        "XLA_ACCELERATE = True\n",
        "\n",
        "if MIXED_PRECISION:\n",
        "    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n",
        "\n",
        "    if tpu:\n",
        "        policy = tf.keras.mixed_precision.experimental.Policy(\"mixed_bfloat16\")\n",
        "    else:\n",
        "        policy = tf.keras.mixed_precision.experimental.Policy(\"mixed_float16\")\n",
        "    mixed_precision.set_policy(policy)\n",
        "    print(\"Mixed precision enabled\")\n",
        "\n",
        "if XLA_ACCELERATE:\n",
        "    tf.config.optimizer.set_jit(True)\n",
        "    print(\"Accelerated Linear Algebra enabled\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accelerated Linear Algebra enabled\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:14.943728Z",
          "iopub.status.busy": "2020-11-30T00:02:14.943009Z",
          "iopub.status.idle": "2020-11-30T00:02:14.947217Z",
          "shell.execute_reply": "2020-11-30T00:02:14.946622Z"
        },
        "id": "8aMe4rbK7OTN",
        "papermill": {
          "duration": 0.085098,
          "end_time": "2020-11-30T00:02:14.947332",
          "exception": false,
          "start_time": "2020-11-30T00:02:14.862234",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ff-lKRvA7OTP",
        "papermill": {
          "duration": 0.114718,
          "end_time": "2020-11-30T00:02:15.137335",
          "exception": false,
          "start_time": "2020-11-30T00:02:15.022617",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:15.305386Z",
          "iopub.status.busy": "2020-11-30T00:02:15.304591Z",
          "iopub.status.idle": "2020-11-30T00:02:15.665959Z",
          "shell.execute_reply": "2020-11-30T00:02:15.664373Z"
        },
        "id": "vYlmC0dI7OTP",
        "papermill": {
          "duration": 0.451988,
          "end_time": "2020-11-30T00:02:15.666090",
          "exception": false,
          "start_time": "2020-11-30T00:02:15.214102",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def fix_seed(seed=2020):\n",
        "    random.seed(seed)\n",
        "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    tf.random.set_seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.manual_seed(seed)\n",
        "        torch.cuda.manual_seed_all(seed)\n",
        "        torch.backends.cudnn.deterministic = True\n",
        "        torch.backends.cudnn.benchmark = False\n",
        "\n",
        "\n",
        "random_seed = 2222\n",
        "fix_seed(random_seed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:15.838943Z",
          "iopub.status.busy": "2020-11-30T00:02:15.831569Z",
          "iopub.status.idle": "2020-11-30T00:02:15.841337Z",
          "shell.execute_reply": "2020-11-30T00:02:15.841901Z"
        },
        "id": "NRRBictUC72K",
        "papermill": {
          "duration": 0.099258,
          "end_time": "2020-11-30T00:02:15.842045",
          "exception": false,
          "start_time": "2020-11-30T00:02:15.742787",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def reduce_mem_usage(df):\n",
        "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
        "    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n",
        "\n",
        "    for col in df.columns:\n",
        "        col_type = df[col].dtype\n",
        "\n",
        "        if col_type != object:\n",
        "            c_min = df[col].min()\n",
        "            c_max = df[col].max()\n",
        "            if str(col_type)[:3] == \"int\":\n",
        "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
        "                    df[col] = df[col].astype(np.int8)\n",
        "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
        "                    df[col] = df[col].astype(np.int16)\n",
        "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
        "                    df[col] = df[col].astype(np.int32)\n",
        "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
        "                    df[col] = df[col].astype(np.int64)\n",
        "            else:\n",
        "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
        "                    df[col] = df[col].astype(np.float16)\n",
        "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
        "                    df[col] = df[col].astype(np.float32)\n",
        "                else:\n",
        "                    df[col] = df[col].astype(np.float64)\n",
        "        else:\n",
        "            df[col] = df[col].astype(\"category\")\n",
        "\n",
        "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
        "    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n",
        "    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n",
        "\n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tS_yBdE7tdSR",
        "papermill": {
          "duration": 0.073894,
          "end_time": "2020-11-30T00:02:15.991430",
          "exception": false,
          "start_time": "2020-11-30T00:02:15.917536",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:16.159366Z",
          "iopub.status.busy": "2020-11-30T00:02:16.157544Z",
          "iopub.status.idle": "2020-11-30T00:02:16.160080Z",
          "shell.execute_reply": "2020-11-30T00:02:16.160603Z"
        },
        "id": "CWfvin6dtein",
        "papermill": {
          "duration": 0.091447,
          "end_time": "2020-11-30T00:02:16.160726",
          "exception": false,
          "start_time": "2020-11-30T00:02:16.069279",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# https://www.kaggle.com/markpeng/deepinsight-efficientnet-b3-noisystudent/comments#1075013\n",
        "\n",
        "def add_swap_noise(index, X, train_):\n",
        "    swap_prob=0.15\n",
        "    swap_portion=0.1\n",
        "\n",
        "    for i in range(len(index)):\n",
        "        if np.random.rand() < swap_prob:\n",
        "            swap_index = np.random.randint(train_.shape[0], size=1)[0]\n",
        "            # Select only gene expression and cell viability features\n",
        "            swap_features = np.random.choice(\n",
        "                np.array(range(2, train_.shape[1])),\n",
        "                size=int(train_.shape[1] * swap_portion),\n",
        "                replace=False\n",
        "            )\n",
        "            X[i, swap_features] = train_[swap_index, swap_features]\n",
        "\n",
        "    return X"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zyamWSs0M93B",
        "papermill": {
          "duration": 0.077006,
          "end_time": "2020-11-30T00:02:16.317691",
          "exception": false,
          "start_time": "2020-11-30T00:02:16.240685",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:16.485861Z",
          "iopub.status.busy": "2020-11-30T00:02:16.485059Z",
          "iopub.status.idle": "2020-11-30T00:02:16.488941Z",
          "shell.execute_reply": "2020-11-30T00:02:16.489432Z"
        },
        "id": "Ysy4T3ZM7OTR",
        "papermill": {
          "duration": 0.093245,
          "end_time": "2020-11-30T00:02:16.489607",
          "exception": false,
          "start_time": "2020-11-30T00:02:16.396362",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Evaluation Metric with sigmoid applied and clipping\n",
        "\n",
        "## for tensorflow\n",
        "def logloss(y_true, y_pred):\n",
        "    logits = 1 / (1 + K.exp(-y_pred))\n",
        "    aux = (1 - y_true) * K.log(1 - logits + 1e-15) + y_true * K.log(logits + 1e-15)\n",
        "    return K.mean(-aux)\n",
        "\n",
        "\n",
        "## for pytorch\n",
        "class LogitsLogLoss(Metric):\n",
        "    def __init__(self):\n",
        "        self._name = \"logits_ll\"\n",
        "        self._maximize = False\n",
        "\n",
        "    def __call__(self, y_true, y_pred):\n",
        "        logits = 1 / (1 + np.exp(-y_pred))\n",
        "        aux = (1 - y_true) * np.log(1 - logits + 1e-15) + y_true * np.log(logits + 1e-15)\n",
        "        return np.mean(-aux)\n",
        "\n",
        "\n",
        "## for overall\n",
        "## [Fast Numpy Log Loss] https://www.kaggle.com/gogo827jz/optimise-blending-weights-4-5x-faster-log-loss\n",
        "def metric(y_true, y_pred):\n",
        "    loss = 0\n",
        "    for i in range(y_pred.shape[1]):\n",
        "        loss += -np.mean(\n",
        "            y_true[:, i] * np.log(y_pred[:, i] + 1e-15) + (1 - y_true[:, i]) * np.log(1 - y_pred[:, i] + 1e-15)\n",
        "        )\n",
        "    return loss / y_pred.shape[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rAAH9Q1VNc9m",
        "papermill": {
          "duration": 0.08615,
          "end_time": "2020-11-30T00:02:16.652242",
          "exception": false,
          "start_time": "2020-11-30T00:02:16.566092",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Loss functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:16.833088Z",
          "iopub.status.busy": "2020-11-30T00:02:16.832050Z",
          "iopub.status.idle": "2020-11-30T00:02:16.835602Z",
          "shell.execute_reply": "2020-11-30T00:02:16.835031Z"
        },
        "id": "KtIMMq6Y7OTY",
        "papermill": {
          "duration": 0.099675,
          "end_time": "2020-11-30T00:02:16.835716",
          "exception": false,
          "start_time": "2020-11-30T00:02:16.736041",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# https://www.kaggle.com/felipebihaiek/torch-continued-from-auxiliary-targets-smoothing\n",
        "class SmoothBCEwLogits(_WeightedLoss):\n",
        "    def __init__(self, weight=None, reduction=\"mean\", smoothing=0.0):\n",
        "        super().__init__(weight=weight, reduction=reduction)\n",
        "        self.smoothing = smoothing\n",
        "        self.weight = weight\n",
        "        self.reduction = reduction\n",
        "\n",
        "    @staticmethod\n",
        "    def _smooth(targets: torch.Tensor, n_labels: int, smoothing=0.0):\n",
        "        assert 0 <= smoothing < 1\n",
        "        with torch.no_grad():\n",
        "            targets = targets * (1.0 - smoothing) + 0.5 * smoothing\n",
        "        return targets\n",
        "\n",
        "    def forward(self, inputs, targets):\n",
        "        targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1), self.smoothing)\n",
        "        loss = F.binary_cross_entropy_with_logits(inputs, targets, self.weight)\n",
        "\n",
        "        if self.reduction == \"sum\":\n",
        "            loss = loss.sum()\n",
        "        elif self.reduction == \"mean\":\n",
        "            loss = loss.mean()\n",
        "\n",
        "        return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_HEx7QqGNqZ8",
        "papermill": {
          "duration": 0.083917,
          "end_time": "2020-11-30T00:02:17.002898",
          "exception": false,
          "start_time": "2020-11-30T00:02:16.918981",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Cross Validation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:17.173252Z",
          "iopub.status.busy": "2020-11-30T00:02:17.171306Z",
          "iopub.status.idle": "2020-11-30T00:02:17.174041Z",
          "shell.execute_reply": "2020-11-30T00:02:17.174578Z"
        },
        "id": "fgO6t6pg7OTV",
        "papermill": {
          "duration": 0.087643,
          "end_time": "2020-11-30T00:02:17.174711",
          "exception": false,
          "start_time": "2020-11-30T00:02:17.087068",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Blend oof predictions\n",
        "def blend(size, weights, oof):\n",
        "    blend_ = np.zeros(size)\n",
        "    for i, key in enumerate(oof.keys()):\n",
        "        blend_ += weights[i] * oof[key].values[: blend_.shape[0], : blend_.shape[1]]\n",
        "    return blend_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:17.381682Z",
          "iopub.status.busy": "2020-11-30T00:02:17.380751Z",
          "iopub.status.idle": "2020-11-30T00:02:17.385312Z",
          "shell.execute_reply": "2020-11-30T00:02:17.386051Z"
        },
        "id": "kcsCWnZeovLR",
        "papermill": {
          "duration": 0.131677,
          "end_time": "2020-11-30T00:02:17.386241",
          "exception": false,
          "start_time": "2020-11-30T00:02:17.254564",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def cross_validation(size, weight, y_true, oof):\n",
        "    x = size[0]\n",
        "    blend_ = blend(y_true[:x].shape, weight, oof)\n",
        "\n",
        "    aucs = []\n",
        "    for task_id in range(blend_.shape[1]):\n",
        "        aucs.append(roc_auc_score(y_true=y_true[:x, task_id], y_score=blend_[:, task_id]))\n",
        "\n",
        "    CV = metric(y_true[:x], blend_)\n",
        "    AUC = np.mean(aucs)\n",
        "    print(f\"Blended CV: {CV}, AUC : {AUC}\")\n",
        "\n",
        "    return CV, AUC, pd.DataFrame(blend_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iotU3PZB7OTa",
        "papermill": {
          "duration": 0.10581,
          "end_time": "2020-11-30T00:02:17.602144",
          "exception": false,
          "start_time": "2020-11-30T00:02:17.496334",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Load Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:17.832205Z",
          "iopub.status.busy": "2020-11-30T00:02:17.830977Z",
          "iopub.status.idle": "2020-11-30T00:02:25.511244Z",
          "shell.execute_reply": "2020-11-30T00:02:25.510720Z"
        },
        "id": "p03erQnA7OTa",
        "papermill": {
          "duration": 7.802652,
          "end_time": "2020-11-30T00:02:25.511358",
          "exception": false,
          "start_time": "2020-11-30T00:02:17.708706",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "train_df = pd.read_csv(\"../input/lish-moa/train_features.csv\")\n",
        "test_df = pd.read_csv(\"../input/lish-moa/test_features.csv\")\n",
        "target_df = pd.read_csv(\"../input/lish-moa/train_targets_scored.csv\")\n",
        "non_target_df = pd.read_csv(\"../input/lish-moa/train_targets_nonscored.csv\")\n",
        "submit_df = pd.read_csv(\"../input/lish-moa/sample_submission.csv\")\n",
        "drug_df = pd.read_csv(\"../input/lish-moa/train_drug.csv\")\n",
        "\n",
        "pub_test_df = pd.read_csv(\"../input/moapublictest/test_features.csv\")\n",
        "# pub_submit_df = pd.read_csv(\"../input/moapublictestpredictions/submission-blendblendblend.csv\")\n",
        "pub_submit_df = pd.read_csv(\"../input/moapublictestpredictions/submission-v65-old_best_lb.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:25.671191Z",
          "iopub.status.busy": "2020-11-30T00:02:25.669935Z",
          "iopub.status.idle": "2020-11-30T00:02:25.804503Z",
          "shell.execute_reply": "2020-11-30T00:02:25.803909Z"
        },
        "id": "hYN5WkgA7OTc",
        "papermill": {
          "duration": 0.217583,
          "end_time": "2020-11-30T00:02:25.804639",
          "exception": false,
          "start_time": "2020-11-30T00:02:25.587056",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "train = train_df.copy()\n",
        "test = test_df.copy()\n",
        "target = target_df.copy()\n",
        "non_target = non_target_df.copy()\n",
        "ss = submit_df.copy()\n",
        "drug = drug_df.copy()\n",
        "\n",
        "pub_test = pub_test_df.copy()\n",
        "pub_ss = pub_submit_df.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U_HmJN8tnpob",
        "papermill": {
          "duration": 0.073224,
          "end_time": "2020-11-30T00:02:25.952341",
          "exception": false,
          "start_time": "2020-11-30T00:02:25.879117",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Use public test data for training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:26.108701Z",
          "iopub.status.busy": "2020-11-30T00:02:26.104887Z",
          "iopub.status.idle": "2020-11-30T00:02:26.360583Z",
          "shell.execute_reply": "2020-11-30T00:02:26.359974Z"
        },
        "id": "1ETxnt8B7OTe",
        "papermill": {
          "duration": 0.336005,
          "end_time": "2020-11-30T00:02:26.360698",
          "exception": false,
          "start_time": "2020-11-30T00:02:26.024693",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Merge public test data (and pseudo label) into train data\n",
        "if USE_PUBLIC_TEST_PSEUDO_LABEL:\n",
        "    train = pd.concat([train, pub_test]).reset_index(drop=True)\n",
        "    target = pd.concat([target, pub_ss]).reset_index(drop=True)\n",
        "\n",
        "# This is used for CV with ctrl_vehicle.\n",
        "train_pub_test = train.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:26.521469Z",
          "iopub.status.busy": "2020-11-30T00:02:26.520522Z",
          "iopub.status.idle": "2020-11-30T00:02:26.563365Z",
          "shell.execute_reply": "2020-11-30T00:02:26.563984Z"
        },
        "id": "Cn_3SXzg7OTg",
        "papermill": {
          "duration": 0.129853,
          "end_time": "2020-11-30T00:02:26.564146",
          "exception": false,
          "start_time": "2020-11-30T00:02:26.434293",
          "status": "completed"
        },
        "tags": [],
        "outputId": "242d3074-5587-41f7-aa26-b294a9021032"
      },
      "source": [
        "target"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sig_id</th>\n",
              "      <th>5-alpha_reductase_inhibitor</th>\n",
              "      <th>11-beta-hsd1_inhibitor</th>\n",
              "      <th>acat_inhibitor</th>\n",
              "      <th>acetylcholine_receptor_agonist</th>\n",
              "      <th>acetylcholine_receptor_antagonist</th>\n",
              "      <th>acetylcholinesterase_inhibitor</th>\n",
              "      <th>adenosine_receptor_agonist</th>\n",
              "      <th>adenosine_receptor_antagonist</th>\n",
              "      <th>adenylyl_cyclase_activator</th>\n",
              "      <th>...</th>\n",
              "      <th>tropomyosin_receptor_kinase_inhibitor</th>\n",
              "      <th>trpv_agonist</th>\n",
              "      <th>trpv_antagonist</th>\n",
              "      <th>tubulin_inhibitor</th>\n",
              "      <th>tyrosine_kinase_inhibitor</th>\n",
              "      <th>ubiquitin_specific_protease_inhibitor</th>\n",
              "      <th>vegfr_inhibitor</th>\n",
              "      <th>vitamin_b</th>\n",
              "      <th>vitamin_d_receptor_agonist</th>\n",
              "      <th>wnt_inhibitor</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_000644bb2</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_000779bfc</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_000a6266a</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_0015fd391</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_001626bd3</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27791</th>\n",
              "      <td>id_ff7004b87</td>\n",
              "      <td>0.001187</td>\n",
              "      <td>0.001471</td>\n",
              "      <td>0.001790</td>\n",
              "      <td>0.003865</td>\n",
              "      <td>0.006655</td>\n",
              "      <td>0.003512</td>\n",
              "      <td>0.001677</td>\n",
              "      <td>0.003756</td>\n",
              "      <td>0.001046</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001112</td>\n",
              "      <td>0.004794</td>\n",
              "      <td>0.002926</td>\n",
              "      <td>0.211817</td>\n",
              "      <td>0.011809</td>\n",
              "      <td>0.001845</td>\n",
              "      <td>0.006745</td>\n",
              "      <td>0.002509</td>\n",
              "      <td>0.001243</td>\n",
              "      <td>0.003638</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27792</th>\n",
              "      <td>id_ff925dd0d</td>\n",
              "      <td>0.006804</td>\n",
              "      <td>0.004367</td>\n",
              "      <td>0.001244</td>\n",
              "      <td>0.007929</td>\n",
              "      <td>0.025647</td>\n",
              "      <td>0.006798</td>\n",
              "      <td>0.004036</td>\n",
              "      <td>0.004191</td>\n",
              "      <td>0.001031</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000985</td>\n",
              "      <td>0.001218</td>\n",
              "      <td>0.002999</td>\n",
              "      <td>0.002630</td>\n",
              "      <td>0.001974</td>\n",
              "      <td>0.001638</td>\n",
              "      <td>0.001790</td>\n",
              "      <td>0.002058</td>\n",
              "      <td>0.000697</td>\n",
              "      <td>0.001785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27793</th>\n",
              "      <td>id_ffb710450</td>\n",
              "      <td>0.002167</td>\n",
              "      <td>0.001374</td>\n",
              "      <td>0.001716</td>\n",
              "      <td>0.014372</td>\n",
              "      <td>0.040319</td>\n",
              "      <td>0.005958</td>\n",
              "      <td>0.004267</td>\n",
              "      <td>0.003298</td>\n",
              "      <td>0.000597</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001039</td>\n",
              "      <td>0.001140</td>\n",
              "      <td>0.003067</td>\n",
              "      <td>0.001776</td>\n",
              "      <td>0.001588</td>\n",
              "      <td>0.001091</td>\n",
              "      <td>0.001032</td>\n",
              "      <td>0.001740</td>\n",
              "      <td>0.000927</td>\n",
              "      <td>0.001551</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27794</th>\n",
              "      <td>id_ffbb869f2</td>\n",
              "      <td>0.002340</td>\n",
              "      <td>0.001462</td>\n",
              "      <td>0.001797</td>\n",
              "      <td>0.022218</td>\n",
              "      <td>0.021918</td>\n",
              "      <td>0.005618</td>\n",
              "      <td>0.008658</td>\n",
              "      <td>0.002461</td>\n",
              "      <td>0.001359</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000921</td>\n",
              "      <td>0.000901</td>\n",
              "      <td>0.002803</td>\n",
              "      <td>0.001641</td>\n",
              "      <td>0.001624</td>\n",
              "      <td>0.000881</td>\n",
              "      <td>0.001040</td>\n",
              "      <td>0.002210</td>\n",
              "      <td>0.000746</td>\n",
              "      <td>0.003842</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27795</th>\n",
              "      <td>id_ffd5800b6</td>\n",
              "      <td>0.001093</td>\n",
              "      <td>0.001305</td>\n",
              "      <td>0.002075</td>\n",
              "      <td>0.017827</td>\n",
              "      <td>0.019495</td>\n",
              "      <td>0.004582</td>\n",
              "      <td>0.003291</td>\n",
              "      <td>0.004849</td>\n",
              "      <td>0.000556</td>\n",
              "      <td>...</td>\n",
              "      <td>0.001120</td>\n",
              "      <td>0.001956</td>\n",
              "      <td>0.002954</td>\n",
              "      <td>0.003283</td>\n",
              "      <td>0.001833</td>\n",
              "      <td>0.001072</td>\n",
              "      <td>0.001663</td>\n",
              "      <td>0.002474</td>\n",
              "      <td>0.001145</td>\n",
              "      <td>0.001950</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>27796 rows × 207 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             sig_id  5-alpha_reductase_inhibitor  11-beta-hsd1_inhibitor  \\\n",
              "0      id_000644bb2                     0.000000                0.000000   \n",
              "1      id_000779bfc                     0.000000                0.000000   \n",
              "2      id_000a6266a                     0.000000                0.000000   \n",
              "3      id_0015fd391                     0.000000                0.000000   \n",
              "4      id_001626bd3                     0.000000                0.000000   \n",
              "...             ...                          ...                     ...   \n",
              "27791  id_ff7004b87                     0.001187                0.001471   \n",
              "27792  id_ff925dd0d                     0.006804                0.004367   \n",
              "27793  id_ffb710450                     0.002167                0.001374   \n",
              "27794  id_ffbb869f2                     0.002340                0.001462   \n",
              "27795  id_ffd5800b6                     0.001093                0.001305   \n",
              "\n",
              "       acat_inhibitor  acetylcholine_receptor_agonist  \\\n",
              "0            0.000000                        0.000000   \n",
              "1            0.000000                        0.000000   \n",
              "2            0.000000                        0.000000   \n",
              "3            0.000000                        0.000000   \n",
              "4            0.000000                        0.000000   \n",
              "...               ...                             ...   \n",
              "27791        0.001790                        0.003865   \n",
              "27792        0.001244                        0.007929   \n",
              "27793        0.001716                        0.014372   \n",
              "27794        0.001797                        0.022218   \n",
              "27795        0.002075                        0.017827   \n",
              "\n",
              "       acetylcholine_receptor_antagonist  acetylcholinesterase_inhibitor  \\\n",
              "0                               0.000000                        0.000000   \n",
              "1                               0.000000                        0.000000   \n",
              "2                               0.000000                        0.000000   \n",
              "3                               0.000000                        0.000000   \n",
              "4                               0.000000                        0.000000   \n",
              "...                                  ...                             ...   \n",
              "27791                           0.006655                        0.003512   \n",
              "27792                           0.025647                        0.006798   \n",
              "27793                           0.040319                        0.005958   \n",
              "27794                           0.021918                        0.005618   \n",
              "27795                           0.019495                        0.004582   \n",
              "\n",
              "       adenosine_receptor_agonist  adenosine_receptor_antagonist  \\\n",
              "0                        0.000000                       0.000000   \n",
              "1                        0.000000                       0.000000   \n",
              "2                        0.000000                       0.000000   \n",
              "3                        0.000000                       0.000000   \n",
              "4                        0.000000                       0.000000   \n",
              "...                           ...                            ...   \n",
              "27791                    0.001677                       0.003756   \n",
              "27792                    0.004036                       0.004191   \n",
              "27793                    0.004267                       0.003298   \n",
              "27794                    0.008658                       0.002461   \n",
              "27795                    0.003291                       0.004849   \n",
              "\n",
              "       adenylyl_cyclase_activator  ...  tropomyosin_receptor_kinase_inhibitor  \\\n",
              "0                        0.000000  ...                               0.000000   \n",
              "1                        0.000000  ...                               0.000000   \n",
              "2                        0.000000  ...                               0.000000   \n",
              "3                        0.000000  ...                               0.000000   \n",
              "4                        0.000000  ...                               0.000000   \n",
              "...                           ...  ...                                    ...   \n",
              "27791                    0.001046  ...                               0.001112   \n",
              "27792                    0.001031  ...                               0.000985   \n",
              "27793                    0.000597  ...                               0.001039   \n",
              "27794                    0.001359  ...                               0.000921   \n",
              "27795                    0.000556  ...                               0.001120   \n",
              "\n",
              "       trpv_agonist  trpv_antagonist  tubulin_inhibitor  \\\n",
              "0          0.000000         0.000000           0.000000   \n",
              "1          0.000000         0.000000           0.000000   \n",
              "2          0.000000         0.000000           0.000000   \n",
              "3          0.000000         0.000000           0.000000   \n",
              "4          0.000000         0.000000           0.000000   \n",
              "...             ...              ...                ...   \n",
              "27791      0.004794         0.002926           0.211817   \n",
              "27792      0.001218         0.002999           0.002630   \n",
              "27793      0.001140         0.003067           0.001776   \n",
              "27794      0.000901         0.002803           0.001641   \n",
              "27795      0.001956         0.002954           0.003283   \n",
              "\n",
              "       tyrosine_kinase_inhibitor  ubiquitin_specific_protease_inhibitor  \\\n",
              "0                       0.000000                               0.000000   \n",
              "1                       0.000000                               0.000000   \n",
              "2                       0.000000                               0.000000   \n",
              "3                       0.000000                               0.000000   \n",
              "4                       0.000000                               0.000000   \n",
              "...                          ...                                    ...   \n",
              "27791                   0.011809                               0.001845   \n",
              "27792                   0.001974                               0.001638   \n",
              "27793                   0.001588                               0.001091   \n",
              "27794                   0.001624                               0.000881   \n",
              "27795                   0.001833                               0.001072   \n",
              "\n",
              "       vegfr_inhibitor  vitamin_b  vitamin_d_receptor_agonist  wnt_inhibitor  \n",
              "0             0.000000   0.000000                    0.000000       0.000000  \n",
              "1             0.000000   0.000000                    0.000000       0.000000  \n",
              "2             0.000000   0.000000                    0.000000       0.000000  \n",
              "3             0.000000   0.000000                    0.000000       0.000000  \n",
              "4             0.000000   0.000000                    0.000000       0.000000  \n",
              "...                ...        ...                         ...            ...  \n",
              "27791         0.006745   0.002509                    0.001243       0.003638  \n",
              "27792         0.001790   0.002058                    0.000697       0.001785  \n",
              "27793         0.001032   0.001740                    0.000927       0.001551  \n",
              "27794         0.001040   0.002210                    0.000746       0.003842  \n",
              "27795         0.001663   0.002474                    0.001145       0.001950  \n",
              "\n",
              "[27796 rows x 207 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7LA6ekI07OTi",
        "papermill": {
          "duration": 0.083718,
          "end_time": "2020-11-30T00:02:26.731937",
          "exception": false,
          "start_time": "2020-11-30T00:02:26.648219",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:26.896953Z",
          "iopub.status.busy": "2020-11-30T00:02:26.895868Z",
          "iopub.status.idle": "2020-11-30T00:02:26.904319Z",
          "shell.execute_reply": "2020-11-30T00:02:26.903782Z"
        },
        "id": "3DM9pkDt7OTj",
        "papermill": {
          "duration": 0.0968,
          "end_time": "2020-11-30T00:02:26.904422",
          "exception": false,
          "start_time": "2020-11-30T00:02:26.807622",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "train.loc[:, \"cp_dose\"] = train.loc[:, \"cp_dose\"].map({\"D1\": 0, \"D2\": 1})\n",
        "test.loc[:, \"cp_dose\"] = test.loc[:, \"cp_dose\"].map({\"D1\": 0, \"D2\": 1})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:27.063826Z",
          "iopub.status.busy": "2020-11-30T00:02:27.062848Z",
          "iopub.status.idle": "2020-11-30T00:02:27.069290Z",
          "shell.execute_reply": "2020-11-30T00:02:27.068784Z"
        },
        "id": "jx46CYog7OTk",
        "papermill": {
          "duration": 0.090668,
          "end_time": "2020-11-30T00:02:27.069396",
          "exception": false,
          "start_time": "2020-11-30T00:02:26.978728",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "train.loc[:, \"cp_time\"] = train.loc[:, \"cp_time\"].map({24: 0, 48: 1, 72: 2})\n",
        "test.loc[:, \"cp_time\"] = test.loc[:, \"cp_time\"].map({24: 0, 48: 1, 72: 2})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aQWxpszg7OTm",
        "papermill": {
          "duration": 0.078959,
          "end_time": "2020-11-30T00:02:27.223902",
          "exception": false,
          "start_time": "2020-11-30T00:02:27.144943",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Remove ctrl_vehicle\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:27.393587Z",
          "iopub.status.busy": "2020-11-30T00:02:27.392617Z",
          "iopub.status.idle": "2020-11-30T00:02:27.682999Z",
          "shell.execute_reply": "2020-11-30T00:02:27.682490Z"
        },
        "id": "0HYbBuVW7OTm",
        "papermill": {
          "duration": 0.381138,
          "end_time": "2020-11-30T00:02:27.683110",
          "exception": false,
          "start_time": "2020-11-30T00:02:27.301972",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if CTRL_VEHICLE == \"use\":\n",
        "    train.loc[:, \"cp_type\"] = train.loc[:, \"cp_type\"].map({\"ctl_vehicle\": 0, \"trt_cp\": 1})\n",
        "    test.loc[:, \"cp_type\"] = test.loc[:, \"cp_type\"].map({\"ctl_vehicle\": 0, \"trt_cp\": 1})\n",
        "\n",
        "else:\n",
        "    if CTRL_VEHICLE == \"remove\":\n",
        "        target = target.loc[train[\"cp_type\"] != \"ctl_vehicle\"].reset_index(drop=True)\n",
        "        non_target = non_target.loc[train[: train_df.shape[0]][\"cp_type\"] != \"ctl_vehicle\"].reset_index(drop=True)\n",
        "\n",
        "        train = train.loc[train[\"cp_type\"] != \"ctl_vehicle\"].reset_index(drop=True)\n",
        "\n",
        "    train = train.drop(\"cp_type\", axis=1)\n",
        "    test = test.drop(\"cp_type\", axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LCzLe0NasfYe",
        "papermill": {
          "duration": 0.077719,
          "end_time": "2020-11-30T00:02:27.835983",
          "exception": false,
          "start_time": "2020-11-30T00:02:27.758264",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Merge drug_id into training data\n",
        "\n",
        "https://www.kaggle.com/c/lish-moa/discussion/195195"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:28.144422Z",
          "iopub.status.busy": "2020-11-30T00:02:28.137956Z",
          "iopub.status.idle": "2020-11-30T00:02:28.186526Z",
          "shell.execute_reply": "2020-11-30T00:02:28.187299Z"
        },
        "id": "fLso2JtqsdQi",
        "papermill": {
          "duration": 0.275187,
          "end_time": "2020-11-30T00:02:28.187559",
          "exception": false,
          "start_time": "2020-11-30T00:02:27.912372",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "target_drug = pd.DataFrame(target.loc[:, \"sig_id\"]).merge(drug, on=\"sig_id\", how=\"left\")\n",
        "non_target_drug = pd.DataFrame(non_target.loc[:, \"sig_id\"]).merge(drug, on=\"sig_id\", how=\"left\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:28.444068Z",
          "iopub.status.busy": "2020-11-30T00:02:28.438328Z",
          "iopub.status.idle": "2020-11-30T00:02:28.448704Z",
          "shell.execute_reply": "2020-11-30T00:02:28.449423Z"
        },
        "id": "PtxsR6XoeM0T",
        "papermill": {
          "duration": 0.137296,
          "end_time": "2020-11-30T00:02:28.449651",
          "exception": false,
          "start_time": "2020-11-30T00:02:28.312355",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "target_drug = target_drug.fillna(\"xxxxxxxxx\")\n",
        "non_target_drug = non_target_drug.fillna(\"xxxxxxxxx\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:28.687728Z",
          "iopub.status.busy": "2020-11-30T00:02:28.686783Z",
          "iopub.status.idle": "2020-11-30T00:02:28.697844Z",
          "shell.execute_reply": "2020-11-30T00:02:28.698947Z"
        },
        "id": "d4w0OboIswhr",
        "papermill": {
          "duration": 0.139714,
          "end_time": "2020-11-30T00:02:28.699123",
          "exception": false,
          "start_time": "2020-11-30T00:02:28.559409",
          "status": "completed"
        },
        "tags": [],
        "outputId": "6a795287-9b05-4fb2-d5f4-dd683b4c674b"
      },
      "source": [
        "target_drug"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sig_id</th>\n",
              "      <th>drug_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_000644bb2</td>\n",
              "      <td>b68db1d53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_000779bfc</td>\n",
              "      <td>df89a8e5a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_000a6266a</td>\n",
              "      <td>18bb41b2c</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_0015fd391</td>\n",
              "      <td>8c7f86626</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_001626bd3</td>\n",
              "      <td>7cbed3131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>id_ff7004b87</td>\n",
              "      <td>xxxxxxxxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>id_ff925dd0d</td>\n",
              "      <td>xxxxxxxxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>id_ffb710450</td>\n",
              "      <td>xxxxxxxxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>id_ffbb869f2</td>\n",
              "      <td>xxxxxxxxx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>id_ffd5800b6</td>\n",
              "      <td>xxxxxxxxx</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             sig_id    drug_id\n",
              "0      id_000644bb2  b68db1d53\n",
              "1      id_000779bfc  df89a8e5a\n",
              "2      id_000a6266a  18bb41b2c\n",
              "3      id_0015fd391  8c7f86626\n",
              "4      id_001626bd3  7cbed3131\n",
              "...             ...        ...\n",
              "25567  id_ff7004b87  xxxxxxxxx\n",
              "25568  id_ff925dd0d  xxxxxxxxx\n",
              "25569  id_ffb710450  xxxxxxxxx\n",
              "25570  id_ffbb869f2  xxxxxxxxx\n",
              "25571  id_ffd5800b6  xxxxxxxxx\n",
              "\n",
              "[25572 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MvkK0m0xsoTz",
        "papermill": {
          "duration": 0.082895,
          "end_time": "2020-11-30T00:02:28.891612",
          "exception": false,
          "start_time": "2020-11-30T00:02:28.808717",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Remove sig_id"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:29.085254Z",
          "iopub.status.busy": "2020-11-30T00:02:29.083921Z",
          "iopub.status.idle": "2020-11-30T00:02:29.087917Z",
          "shell.execute_reply": "2020-11-30T00:02:29.087422Z"
        },
        "id": "5KBPZw3p7OTq",
        "papermill": {
          "duration": 0.110375,
          "end_time": "2020-11-30T00:02:29.088020",
          "exception": false,
          "start_time": "2020-11-30T00:02:28.977645",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "del train[\"sig_id\"]\n",
        "del target[\"sig_id\"]\n",
        "del non_target[\"sig_id\"]\n",
        "del test[\"sig_id\"]\n",
        "del ss[\"sig_id\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:29.253968Z",
          "iopub.status.busy": "2020-11-30T00:02:29.252989Z",
          "iopub.status.idle": "2020-11-30T00:02:29.288928Z",
          "shell.execute_reply": "2020-11-30T00:02:29.288255Z"
        },
        "id": "GCSu_X1_7OTs",
        "papermill": {
          "duration": 0.124714,
          "end_time": "2020-11-30T00:02:29.289058",
          "exception": false,
          "start_time": "2020-11-30T00:02:29.164344",
          "status": "completed"
        },
        "tags": [],
        "outputId": "2067fe5b-1aed-4bb7-ad16-6e42a40b707e"
      },
      "source": [
        "train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cp_time</th>\n",
              "      <th>cp_dose</th>\n",
              "      <th>g-0</th>\n",
              "      <th>g-1</th>\n",
              "      <th>g-2</th>\n",
              "      <th>g-3</th>\n",
              "      <th>g-4</th>\n",
              "      <th>g-5</th>\n",
              "      <th>g-6</th>\n",
              "      <th>g-7</th>\n",
              "      <th>...</th>\n",
              "      <th>c-90</th>\n",
              "      <th>c-91</th>\n",
              "      <th>c-92</th>\n",
              "      <th>c-93</th>\n",
              "      <th>c-94</th>\n",
              "      <th>c-95</th>\n",
              "      <th>c-96</th>\n",
              "      <th>c-97</th>\n",
              "      <th>c-98</th>\n",
              "      <th>c-99</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0620</td>\n",
              "      <td>0.5577</td>\n",
              "      <td>-0.2479</td>\n",
              "      <td>-0.6208</td>\n",
              "      <td>-0.1944</td>\n",
              "      <td>-1.0120</td>\n",
              "      <td>-1.0220</td>\n",
              "      <td>-0.0326</td>\n",
              "      <td>...</td>\n",
              "      <td>0.2862</td>\n",
              "      <td>0.2584</td>\n",
              "      <td>0.8076</td>\n",
              "      <td>0.5523</td>\n",
              "      <td>-0.1912</td>\n",
              "      <td>0.6584</td>\n",
              "      <td>-0.3981</td>\n",
              "      <td>0.2139</td>\n",
              "      <td>0.3801</td>\n",
              "      <td>0.4176</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0743</td>\n",
              "      <td>0.4087</td>\n",
              "      <td>0.2991</td>\n",
              "      <td>0.0604</td>\n",
              "      <td>1.0190</td>\n",
              "      <td>0.5207</td>\n",
              "      <td>0.2341</td>\n",
              "      <td>0.3372</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.4265</td>\n",
              "      <td>0.7543</td>\n",
              "      <td>0.4708</td>\n",
              "      <td>0.0230</td>\n",
              "      <td>0.2957</td>\n",
              "      <td>0.4899</td>\n",
              "      <td>0.1522</td>\n",
              "      <td>0.1241</td>\n",
              "      <td>0.6077</td>\n",
              "      <td>0.7371</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.6280</td>\n",
              "      <td>0.5817</td>\n",
              "      <td>1.5540</td>\n",
              "      <td>-0.0764</td>\n",
              "      <td>-0.0323</td>\n",
              "      <td>1.2390</td>\n",
              "      <td>0.1715</td>\n",
              "      <td>0.2155</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.7250</td>\n",
              "      <td>-0.6297</td>\n",
              "      <td>0.6103</td>\n",
              "      <td>0.0223</td>\n",
              "      <td>-1.3240</td>\n",
              "      <td>-0.3174</td>\n",
              "      <td>-0.6417</td>\n",
              "      <td>-0.2187</td>\n",
              "      <td>-1.4080</td>\n",
              "      <td>0.6931</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.5138</td>\n",
              "      <td>-0.2491</td>\n",
              "      <td>-0.2656</td>\n",
              "      <td>0.5288</td>\n",
              "      <td>4.0620</td>\n",
              "      <td>-0.8095</td>\n",
              "      <td>-1.9590</td>\n",
              "      <td>0.1792</td>\n",
              "      <td>...</td>\n",
              "      <td>-2.0990</td>\n",
              "      <td>-0.6441</td>\n",
              "      <td>-5.6300</td>\n",
              "      <td>-1.3780</td>\n",
              "      <td>-0.8632</td>\n",
              "      <td>-1.2880</td>\n",
              "      <td>-1.6210</td>\n",
              "      <td>-0.8784</td>\n",
              "      <td>-0.3876</td>\n",
              "      <td>-0.8154</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.3254</td>\n",
              "      <td>-0.4009</td>\n",
              "      <td>0.9700</td>\n",
              "      <td>0.6919</td>\n",
              "      <td>1.4180</td>\n",
              "      <td>-0.8244</td>\n",
              "      <td>-0.2800</td>\n",
              "      <td>-0.1498</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0042</td>\n",
              "      <td>0.0048</td>\n",
              "      <td>0.6670</td>\n",
              "      <td>1.0690</td>\n",
              "      <td>0.5523</td>\n",
              "      <td>-0.3031</td>\n",
              "      <td>0.1094</td>\n",
              "      <td>0.2885</td>\n",
              "      <td>-0.3786</td>\n",
              "      <td>0.7125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.4571</td>\n",
              "      <td>-0.5743</td>\n",
              "      <td>3.3930</td>\n",
              "      <td>-0.6202</td>\n",
              "      <td>0.8557</td>\n",
              "      <td>1.6240</td>\n",
              "      <td>0.0640</td>\n",
              "      <td>-0.6316</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.1790</td>\n",
              "      <td>-0.6422</td>\n",
              "      <td>-0.4367</td>\n",
              "      <td>0.0159</td>\n",
              "      <td>-0.6539</td>\n",
              "      <td>-0.4791</td>\n",
              "      <td>-1.2680</td>\n",
              "      <td>-1.1280</td>\n",
              "      <td>-0.4167</td>\n",
              "      <td>-0.6600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.5885</td>\n",
              "      <td>-0.2548</td>\n",
              "      <td>2.5850</td>\n",
              "      <td>0.3456</td>\n",
              "      <td>0.4401</td>\n",
              "      <td>0.3107</td>\n",
              "      <td>-0.7437</td>\n",
              "      <td>-0.0143</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0210</td>\n",
              "      <td>0.5780</td>\n",
              "      <td>-0.5888</td>\n",
              "      <td>0.8057</td>\n",
              "      <td>0.9312</td>\n",
              "      <td>1.2730</td>\n",
              "      <td>0.2614</td>\n",
              "      <td>-0.2790</td>\n",
              "      <td>-0.0131</td>\n",
              "      <td>-0.0934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.3985</td>\n",
              "      <td>-0.1554</td>\n",
              "      <td>0.2677</td>\n",
              "      <td>-0.6813</td>\n",
              "      <td>0.0152</td>\n",
              "      <td>0.4791</td>\n",
              "      <td>-0.0166</td>\n",
              "      <td>0.7501</td>\n",
              "      <td>...</td>\n",
              "      <td>0.4418</td>\n",
              "      <td>0.9153</td>\n",
              "      <td>-0.1862</td>\n",
              "      <td>0.4049</td>\n",
              "      <td>0.9568</td>\n",
              "      <td>0.4666</td>\n",
              "      <td>0.0461</td>\n",
              "      <td>0.5888</td>\n",
              "      <td>-0.4205</td>\n",
              "      <td>-0.1504</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-1.0960</td>\n",
              "      <td>-1.7750</td>\n",
              "      <td>-0.3977</td>\n",
              "      <td>1.0160</td>\n",
              "      <td>-1.3350</td>\n",
              "      <td>-0.2207</td>\n",
              "      <td>-0.3611</td>\n",
              "      <td>-1.3020</td>\n",
              "      <td>...</td>\n",
              "      <td>0.3079</td>\n",
              "      <td>-0.4473</td>\n",
              "      <td>-0.8192</td>\n",
              "      <td>0.7785</td>\n",
              "      <td>0.3133</td>\n",
              "      <td>0.1286</td>\n",
              "      <td>-0.2618</td>\n",
              "      <td>0.5074</td>\n",
              "      <td>0.7430</td>\n",
              "      <td>-0.0484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.5174</td>\n",
              "      <td>0.2953</td>\n",
              "      <td>0.3286</td>\n",
              "      <td>-0.0428</td>\n",
              "      <td>-0.0800</td>\n",
              "      <td>0.8702</td>\n",
              "      <td>-0.8724</td>\n",
              "      <td>0.3883</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0363</td>\n",
              "      <td>0.1708</td>\n",
              "      <td>0.5939</td>\n",
              "      <td>-0.0507</td>\n",
              "      <td>0.2811</td>\n",
              "      <td>-0.4041</td>\n",
              "      <td>-0.4948</td>\n",
              "      <td>0.0757</td>\n",
              "      <td>-0.1356</td>\n",
              "      <td>0.5280</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 874 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       cp_time  cp_dose     g-0     g-1     g-2     g-3     g-4     g-5  \\\n",
              "0            0        0  1.0620  0.5577 -0.2479 -0.6208 -0.1944 -1.0120   \n",
              "1            2        0  0.0743  0.4087  0.2991  0.0604  1.0190  0.5207   \n",
              "2            1        0  0.6280  0.5817  1.5540 -0.0764 -0.0323  1.2390   \n",
              "3            1        0 -0.5138 -0.2491 -0.2656  0.5288  4.0620 -0.8095   \n",
              "4            2        1 -0.3254 -0.4009  0.9700  0.6919  1.4180 -0.8244   \n",
              "...        ...      ...     ...     ...     ...     ...     ...     ...   \n",
              "25567        0        0  0.4571 -0.5743  3.3930 -0.6202  0.8557  1.6240   \n",
              "25568        0        0 -0.5885 -0.2548  2.5850  0.3456  0.4401  0.3107   \n",
              "25569        2        0 -0.3985 -0.1554  0.2677 -0.6813  0.0152  0.4791   \n",
              "25570        1        1 -1.0960 -1.7750 -0.3977  1.0160 -1.3350 -0.2207   \n",
              "25571        2        0 -0.5174  0.2953  0.3286 -0.0428 -0.0800  0.8702   \n",
              "\n",
              "          g-6     g-7  ...    c-90    c-91    c-92    c-93    c-94    c-95  \\\n",
              "0     -1.0220 -0.0326  ...  0.2862  0.2584  0.8076  0.5523 -0.1912  0.6584   \n",
              "1      0.2341  0.3372  ... -0.4265  0.7543  0.4708  0.0230  0.2957  0.4899   \n",
              "2      0.1715  0.2155  ... -0.7250 -0.6297  0.6103  0.0223 -1.3240 -0.3174   \n",
              "3     -1.9590  0.1792  ... -2.0990 -0.6441 -5.6300 -1.3780 -0.8632 -1.2880   \n",
              "4     -0.2800 -0.1498  ...  0.0042  0.0048  0.6670  1.0690  0.5523 -0.3031   \n",
              "...       ...     ...  ...     ...     ...     ...     ...     ...     ...   \n",
              "25567  0.0640 -0.6316  ... -1.1790 -0.6422 -0.4367  0.0159 -0.6539 -0.4791   \n",
              "25568 -0.7437 -0.0143  ...  0.0210  0.5780 -0.5888  0.8057  0.9312  1.2730   \n",
              "25569 -0.0166  0.7501  ...  0.4418  0.9153 -0.1862  0.4049  0.9568  0.4666   \n",
              "25570 -0.3611 -1.3020  ...  0.3079 -0.4473 -0.8192  0.7785  0.3133  0.1286   \n",
              "25571 -0.8724  0.3883  ...  0.0363  0.1708  0.5939 -0.0507  0.2811 -0.4041   \n",
              "\n",
              "         c-96    c-97    c-98    c-99  \n",
              "0     -0.3981  0.2139  0.3801  0.4176  \n",
              "1      0.1522  0.1241  0.6077  0.7371  \n",
              "2     -0.6417 -0.2187 -1.4080  0.6931  \n",
              "3     -1.6210 -0.8784 -0.3876 -0.8154  \n",
              "4      0.1094  0.2885 -0.3786  0.7125  \n",
              "...       ...     ...     ...     ...  \n",
              "25567 -1.2680 -1.1280 -0.4167 -0.6600  \n",
              "25568  0.2614 -0.2790 -0.0131 -0.0934  \n",
              "25569  0.0461  0.5888 -0.4205 -0.1504  \n",
              "25570 -0.2618  0.5074  0.7430 -0.0484  \n",
              "25571 -0.4948  0.0757 -0.1356  0.5280  \n",
              "\n",
              "[25572 rows x 874 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:29.480690Z",
          "iopub.status.busy": "2020-11-30T00:02:29.479725Z",
          "iopub.status.idle": "2020-11-30T00:02:29.484932Z",
          "shell.execute_reply": "2020-11-30T00:02:29.485617Z"
        },
        "id": "JZaiV9v_7OTv",
        "papermill": {
          "duration": 0.099596,
          "end_time": "2020-11-30T00:02:29.485763",
          "exception": false,
          "start_time": "2020-11-30T00:02:29.386167",
          "status": "completed"
        },
        "tags": [],
        "outputId": "b82fb9f0-ffda-4fb9-915f-518c29d98a63"
      },
      "source": [
        "print(train.shape)\n",
        "print(target.shape)\n",
        "print(non_target.shape)\n",
        "\n",
        "print(test.shape)\n",
        "print(ss.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(25572, 874)\n",
            "(25572, 206)\n",
            "(21948, 402)\n",
            "(3982, 874)\n",
            "(3982, 206)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6rjhO-b7OTx",
        "papermill": {
          "duration": 0.084357,
          "end_time": "2020-11-30T00:02:29.659686",
          "exception": false,
          "start_time": "2020-11-30T00:02:29.575329",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Rank Gauss\n",
        "\n",
        "https://www.kaggle.com/nayuts/moa-pytorch-nn-pca-rankgauss\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:29.831589Z",
          "iopub.status.busy": "2020-11-30T00:02:29.830532Z",
          "iopub.status.idle": "2020-11-30T00:02:40.071503Z",
          "shell.execute_reply": "2020-11-30T00:02:40.072070Z"
        },
        "id": "jUsSMkTK7OTx",
        "papermill": {
          "duration": 10.332947,
          "end_time": "2020-11-30T00:02:40.072225",
          "exception": false,
          "start_time": "2020-11-30T00:02:29.739278",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "g_cols = [col for col in train_df.columns if col.startswith(\"g-\")]\n",
        "c_cols = [col for col in train_df.columns if col.startswith(\"c-\")]\n",
        "\n",
        "for col in g_cols + c_cols:\n",
        "    transformer = QuantileTransformer(n_quantiles=100, random_state=random_seed, output_distribution=\"normal\")\n",
        "\n",
        "    vec_len = len(train[col].values)\n",
        "    vec_len_test = len(test[col].values)\n",
        "\n",
        "    raw_vec = train[col].values.reshape(vec_len, 1)\n",
        "    transformer.fit(raw_vec)\n",
        "\n",
        "    train[col] = transformer.transform(raw_vec).reshape(1, vec_len)[0]\n",
        "    test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:40.239577Z",
          "iopub.status.busy": "2020-11-30T00:02:40.238665Z",
          "iopub.status.idle": "2020-11-30T00:02:40.270945Z",
          "shell.execute_reply": "2020-11-30T00:02:40.271499Z"
        },
        "id": "VY4_EigR7OT0",
        "papermill": {
          "duration": 0.121205,
          "end_time": "2020-11-30T00:02:40.271639",
          "exception": false,
          "start_time": "2020-11-30T00:02:40.150434",
          "status": "completed"
        },
        "tags": [],
        "outputId": "4c1ffe3f-13fa-4938-c924-603cbe1f7786"
      },
      "source": [
        "train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cp_time</th>\n",
              "      <th>cp_dose</th>\n",
              "      <th>g-0</th>\n",
              "      <th>g-1</th>\n",
              "      <th>g-2</th>\n",
              "      <th>g-3</th>\n",
              "      <th>g-4</th>\n",
              "      <th>g-5</th>\n",
              "      <th>g-6</th>\n",
              "      <th>g-7</th>\n",
              "      <th>...</th>\n",
              "      <th>c-90</th>\n",
              "      <th>c-91</th>\n",
              "      <th>c-92</th>\n",
              "      <th>c-93</th>\n",
              "      <th>c-94</th>\n",
              "      <th>c-95</th>\n",
              "      <th>c-96</th>\n",
              "      <th>c-97</th>\n",
              "      <th>c-98</th>\n",
              "      <th>c-99</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.124260</td>\n",
              "      <td>0.896698</td>\n",
              "      <td>-0.436214</td>\n",
              "      <td>-0.965311</td>\n",
              "      <td>-0.287443</td>\n",
              "      <td>-1.016437</td>\n",
              "      <td>-1.360774</td>\n",
              "      <td>-0.045876</td>\n",
              "      <td>...</td>\n",
              "      <td>0.428869</td>\n",
              "      <td>0.384250</td>\n",
              "      <td>1.300482</td>\n",
              "      <td>0.879422</td>\n",
              "      <td>-0.206096</td>\n",
              "      <td>1.046155</td>\n",
              "      <td>-0.479268</td>\n",
              "      <td>0.339234</td>\n",
              "      <td>0.583214</td>\n",
              "      <td>0.696712</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0.117451</td>\n",
              "      <td>0.667759</td>\n",
              "      <td>0.260124</td>\n",
              "      <td>0.097531</td>\n",
              "      <td>1.204172</td>\n",
              "      <td>0.692876</td>\n",
              "      <td>0.356691</td>\n",
              "      <td>0.559630</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.499745</td>\n",
              "      <td>1.147297</td>\n",
              "      <td>0.728062</td>\n",
              "      <td>0.089253</td>\n",
              "      <td>0.453665</td>\n",
              "      <td>0.770909</td>\n",
              "      <td>0.226300</td>\n",
              "      <td>0.202945</td>\n",
              "      <td>0.955497</td>\n",
              "      <td>1.219730</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.777229</td>\n",
              "      <td>0.935347</td>\n",
              "      <td>1.414044</td>\n",
              "      <td>-0.113563</td>\n",
              "      <td>-0.025489</td>\n",
              "      <td>1.494313</td>\n",
              "      <td>0.277364</td>\n",
              "      <td>0.357917</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.800373</td>\n",
              "      <td>-0.721883</td>\n",
              "      <td>0.960080</td>\n",
              "      <td>0.088259</td>\n",
              "      <td>-1.182700</td>\n",
              "      <td>-0.358059</td>\n",
              "      <td>-0.732238</td>\n",
              "      <td>-0.253014</td>\n",
              "      <td>-1.085791</td>\n",
              "      <td>1.140342</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.749489</td>\n",
              "      <td>-0.299404</td>\n",
              "      <td>-0.459100</td>\n",
              "      <td>0.774708</td>\n",
              "      <td>2.344556</td>\n",
              "      <td>-0.856449</td>\n",
              "      <td>-2.323390</td>\n",
              "      <td>0.298781</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.391931</td>\n",
              "      <td>-0.736149</td>\n",
              "      <td>-1.612415</td>\n",
              "      <td>-1.219207</td>\n",
              "      <td>-0.912980</td>\n",
              "      <td>-1.194806</td>\n",
              "      <td>-1.288428</td>\n",
              "      <td>-0.950502</td>\n",
              "      <td>-0.445204</td>\n",
              "      <td>-0.884754</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.460555</td>\n",
              "      <td>-0.508226</td>\n",
              "      <td>0.959313</td>\n",
              "      <td>0.984009</td>\n",
              "      <td>1.451890</td>\n",
              "      <td>-0.867329</td>\n",
              "      <td>-0.342599</td>\n",
              "      <td>-0.234770</td>\n",
              "      <td>...</td>\n",
              "      <td>0.038727</td>\n",
              "      <td>0.021330</td>\n",
              "      <td>1.056779</td>\n",
              "      <td>1.734597</td>\n",
              "      <td>0.843756</td>\n",
              "      <td>-0.341198</td>\n",
              "      <td>0.169668</td>\n",
              "      <td>0.451146</td>\n",
              "      <td>-0.434772</td>\n",
              "      <td>1.174162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.599819</td>\n",
              "      <td>-0.723658</td>\n",
              "      <td>2.333805</td>\n",
              "      <td>-0.964385</td>\n",
              "      <td>1.075072</td>\n",
              "      <td>1.786462</td>\n",
              "      <td>0.131113</td>\n",
              "      <td>-0.812649</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.126325</td>\n",
              "      <td>-0.734273</td>\n",
              "      <td>-0.505298</td>\n",
              "      <td>0.079622</td>\n",
              "      <td>-0.730794</td>\n",
              "      <td>-0.547424</td>\n",
              "      <td>-1.172338</td>\n",
              "      <td>-1.131847</td>\n",
              "      <td>-0.476173</td>\n",
              "      <td>-0.740089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.865375</td>\n",
              "      <td>-0.307222</td>\n",
              "      <td>1.988666</td>\n",
              "      <td>0.523322</td>\n",
              "      <td>0.644383</td>\n",
              "      <td>0.437283</td>\n",
              "      <td>-0.989775</td>\n",
              "      <td>-0.015526</td>\n",
              "      <td>...</td>\n",
              "      <td>0.062985</td>\n",
              "      <td>0.868007</td>\n",
              "      <td>-0.660578</td>\n",
              "      <td>1.300272</td>\n",
              "      <td>1.418983</td>\n",
              "      <td>2.041320</td>\n",
              "      <td>0.385704</td>\n",
              "      <td>-0.325379</td>\n",
              "      <td>0.004373</td>\n",
              "      <td>-0.059129</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.575003</td>\n",
              "      <td>-0.170529</td>\n",
              "      <td>0.221620</td>\n",
              "      <td>-1.053665</td>\n",
              "      <td>0.049099</td>\n",
              "      <td>0.641676</td>\n",
              "      <td>0.016354</td>\n",
              "      <td>1.208632</td>\n",
              "      <td>...</td>\n",
              "      <td>0.670012</td>\n",
              "      <td>1.400519</td>\n",
              "      <td>-0.220545</td>\n",
              "      <td>0.641140</td>\n",
              "      <td>1.461002</td>\n",
              "      <td>0.734826</td>\n",
              "      <td>0.081096</td>\n",
              "      <td>0.924382</td>\n",
              "      <td>-0.479982</td>\n",
              "      <td>-0.136455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-1.635710</td>\n",
              "      <td>-1.976022</td>\n",
              "      <td>-0.636890</td>\n",
              "      <td>1.330359</td>\n",
              "      <td>-1.719197</td>\n",
              "      <td>-0.259467</td>\n",
              "      <td>-0.455103</td>\n",
              "      <td>-1.308468</td>\n",
              "      <td>...</td>\n",
              "      <td>0.463639</td>\n",
              "      <td>-0.527331</td>\n",
              "      <td>-0.857951</td>\n",
              "      <td>1.256633</td>\n",
              "      <td>0.481844</td>\n",
              "      <td>0.236350</td>\n",
              "      <td>-0.320390</td>\n",
              "      <td>0.796356</td>\n",
              "      <td>1.191530</td>\n",
              "      <td>-0.001347</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.754997</td>\n",
              "      <td>0.494512</td>\n",
              "      <td>0.294161</td>\n",
              "      <td>-0.062373</td>\n",
              "      <td>-0.102483</td>\n",
              "      <td>1.097825</td>\n",
              "      <td>-1.164675</td>\n",
              "      <td>0.639927</td>\n",
              "      <td>...</td>\n",
              "      <td>0.083709</td>\n",
              "      <td>0.256954</td>\n",
              "      <td>0.932040</td>\n",
              "      <td>-0.010455</td>\n",
              "      <td>0.433234</td>\n",
              "      <td>-0.460940</td>\n",
              "      <td>-0.593303</td>\n",
              "      <td>0.135958</td>\n",
              "      <td>-0.151625</td>\n",
              "      <td>0.872828</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 874 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       cp_time  cp_dose       g-0       g-1       g-2       g-3       g-4  \\\n",
              "0            0        0  1.124260  0.896698 -0.436214 -0.965311 -0.287443   \n",
              "1            2        0  0.117451  0.667759  0.260124  0.097531  1.204172   \n",
              "2            1        0  0.777229  0.935347  1.414044 -0.113563 -0.025489   \n",
              "3            1        0 -0.749489 -0.299404 -0.459100  0.774708  2.344556   \n",
              "4            2        1 -0.460555 -0.508226  0.959313  0.984009  1.451890   \n",
              "...        ...      ...       ...       ...       ...       ...       ...   \n",
              "25567        0        0  0.599819 -0.723658  2.333805 -0.964385  1.075072   \n",
              "25568        0        0 -0.865375 -0.307222  1.988666  0.523322  0.644383   \n",
              "25569        2        0 -0.575003 -0.170529  0.221620 -1.053665  0.049099   \n",
              "25570        1        1 -1.635710 -1.976022 -0.636890  1.330359 -1.719197   \n",
              "25571        2        0 -0.754997  0.494512  0.294161 -0.062373 -0.102483   \n",
              "\n",
              "            g-5       g-6       g-7  ...      c-90      c-91      c-92  \\\n",
              "0     -1.016437 -1.360774 -0.045876  ...  0.428869  0.384250  1.300482   \n",
              "1      0.692876  0.356691  0.559630  ... -0.499745  1.147297  0.728062   \n",
              "2      1.494313  0.277364  0.357917  ... -0.800373 -0.721883  0.960080   \n",
              "3     -0.856449 -2.323390  0.298781  ... -1.391931 -0.736149 -1.612415   \n",
              "4     -0.867329 -0.342599 -0.234770  ...  0.038727  0.021330  1.056779   \n",
              "...         ...       ...       ...  ...       ...       ...       ...   \n",
              "25567  1.786462  0.131113 -0.812649  ... -1.126325 -0.734273 -0.505298   \n",
              "25568  0.437283 -0.989775 -0.015526  ...  0.062985  0.868007 -0.660578   \n",
              "25569  0.641676  0.016354  1.208632  ...  0.670012  1.400519 -0.220545   \n",
              "25570 -0.259467 -0.455103 -1.308468  ...  0.463639 -0.527331 -0.857951   \n",
              "25571  1.097825 -1.164675  0.639927  ...  0.083709  0.256954  0.932040   \n",
              "\n",
              "           c-93      c-94      c-95      c-96      c-97      c-98      c-99  \n",
              "0      0.879422 -0.206096  1.046155 -0.479268  0.339234  0.583214  0.696712  \n",
              "1      0.089253  0.453665  0.770909  0.226300  0.202945  0.955497  1.219730  \n",
              "2      0.088259 -1.182700 -0.358059 -0.732238 -0.253014 -1.085791  1.140342  \n",
              "3     -1.219207 -0.912980 -1.194806 -1.288428 -0.950502 -0.445204 -0.884754  \n",
              "4      1.734597  0.843756 -0.341198  0.169668  0.451146 -0.434772  1.174162  \n",
              "...         ...       ...       ...       ...       ...       ...       ...  \n",
              "25567  0.079622 -0.730794 -0.547424 -1.172338 -1.131847 -0.476173 -0.740089  \n",
              "25568  1.300272  1.418983  2.041320  0.385704 -0.325379  0.004373 -0.059129  \n",
              "25569  0.641140  1.461002  0.734826  0.081096  0.924382 -0.479982 -0.136455  \n",
              "25570  1.256633  0.481844  0.236350 -0.320390  0.796356  1.191530 -0.001347  \n",
              "25571 -0.010455  0.433234 -0.460940 -0.593303  0.135958 -0.151625  0.872828  \n",
              "\n",
              "[25572 rows x 874 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DHiSL9lh7OT2",
        "papermill": {
          "duration": 0.080125,
          "end_time": "2020-11-30T00:02:40.441664",
          "exception": false,
          "start_time": "2020-11-30T00:02:40.361539",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## PCA features (+ Existing features)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:40.675375Z",
          "iopub.status.busy": "2020-11-30T00:02:40.616777Z",
          "iopub.status.idle": "2020-11-30T00:02:43.040824Z",
          "shell.execute_reply": "2020-11-30T00:02:43.040217Z"
        },
        "id": "vSiyBS5s7OT2",
        "papermill": {
          "duration": 2.519036,
          "end_time": "2020-11-30T00:02:43.040955",
          "exception": false,
          "start_time": "2020-11-30T00:02:40.521919",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# g-\n",
        "n_comp = 50\n",
        "\n",
        "data = pd.concat([pd.DataFrame(train[g_cols]), pd.DataFrame(test[g_cols])])\n",
        "data2 = PCA(n_components=n_comp, random_state=random_seed).fit_transform(data[g_cols])\n",
        "train2 = data2[: train.shape[0]]\n",
        "test2 = data2[-test.shape[0] :]\n",
        "\n",
        "train2 = pd.DataFrame(train2, columns=[f\"pca_G-{i}\" for i in range(n_comp)])\n",
        "test2 = pd.DataFrame(test2, columns=[f\"pca_G-{i}\" for i in range(n_comp)])\n",
        "\n",
        "train = pd.concat((train, train2), axis=1)\n",
        "test = pd.concat((test, test2), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:43.211901Z",
          "iopub.status.busy": "2020-11-30T00:02:43.210590Z",
          "iopub.status.idle": "2020-11-30T00:02:43.758705Z",
          "shell.execute_reply": "2020-11-30T00:02:43.757870Z"
        },
        "id": "4m1Xajv77OT4",
        "papermill": {
          "duration": 0.639469,
          "end_time": "2020-11-30T00:02:43.758901",
          "exception": false,
          "start_time": "2020-11-30T00:02:43.119432",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# c-\n",
        "n_comp = 15\n",
        "\n",
        "data = pd.concat([pd.DataFrame(train[c_cols]), pd.DataFrame(test[c_cols])])\n",
        "data2 = PCA(n_components=n_comp, random_state=random_seed).fit_transform(data[c_cols])\n",
        "train2 = data2[: train.shape[0]]\n",
        "test2 = data2[-test.shape[0] :]\n",
        "\n",
        "train2 = pd.DataFrame(train2, columns=[f\"pca_C-{i}\" for i in range(n_comp)])\n",
        "test2 = pd.DataFrame(test2, columns=[f\"pca_C-{i}\" for i in range(n_comp)])\n",
        "\n",
        "train = pd.concat((train, train2), axis=1)\n",
        "test = pd.concat((test, test2), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:43.998294Z",
          "iopub.status.busy": "2020-11-30T00:02:43.997138Z",
          "iopub.status.idle": "2020-11-30T00:02:44.047433Z",
          "shell.execute_reply": "2020-11-30T00:02:44.044476Z"
        },
        "id": "2vVAG1aL7OT6",
        "papermill": {
          "duration": 0.173018,
          "end_time": "2020-11-30T00:02:44.047611",
          "exception": false,
          "start_time": "2020-11-30T00:02:43.874593",
          "status": "completed"
        },
        "tags": [],
        "outputId": "0e6062ad-8b29-4f2f-8f19-be8e34515f72"
      },
      "source": [
        "train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cp_time</th>\n",
              "      <th>cp_dose</th>\n",
              "      <th>g-0</th>\n",
              "      <th>g-1</th>\n",
              "      <th>g-2</th>\n",
              "      <th>g-3</th>\n",
              "      <th>g-4</th>\n",
              "      <th>g-5</th>\n",
              "      <th>g-6</th>\n",
              "      <th>g-7</th>\n",
              "      <th>...</th>\n",
              "      <th>pca_C-5</th>\n",
              "      <th>pca_C-6</th>\n",
              "      <th>pca_C-7</th>\n",
              "      <th>pca_C-8</th>\n",
              "      <th>pca_C-9</th>\n",
              "      <th>pca_C-10</th>\n",
              "      <th>pca_C-11</th>\n",
              "      <th>pca_C-12</th>\n",
              "      <th>pca_C-13</th>\n",
              "      <th>pca_C-14</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.124260</td>\n",
              "      <td>0.896698</td>\n",
              "      <td>-0.436214</td>\n",
              "      <td>-0.965311</td>\n",
              "      <td>-0.287443</td>\n",
              "      <td>-1.016437</td>\n",
              "      <td>-1.360774</td>\n",
              "      <td>-0.045876</td>\n",
              "      <td>...</td>\n",
              "      <td>0.984003</td>\n",
              "      <td>-0.450318</td>\n",
              "      <td>0.336426</td>\n",
              "      <td>0.015646</td>\n",
              "      <td>0.244094</td>\n",
              "      <td>-0.336008</td>\n",
              "      <td>0.062116</td>\n",
              "      <td>-0.398600</td>\n",
              "      <td>-0.044412</td>\n",
              "      <td>-1.430930</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0.117451</td>\n",
              "      <td>0.667759</td>\n",
              "      <td>0.260124</td>\n",
              "      <td>0.097531</td>\n",
              "      <td>1.204172</td>\n",
              "      <td>0.692876</td>\n",
              "      <td>0.356691</td>\n",
              "      <td>0.559630</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.505982</td>\n",
              "      <td>0.099885</td>\n",
              "      <td>0.527028</td>\n",
              "      <td>1.249065</td>\n",
              "      <td>-0.497161</td>\n",
              "      <td>-1.017310</td>\n",
              "      <td>0.180813</td>\n",
              "      <td>-0.877796</td>\n",
              "      <td>1.173441</td>\n",
              "      <td>-0.533822</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.777229</td>\n",
              "      <td>0.935347</td>\n",
              "      <td>1.414044</td>\n",
              "      <td>-0.113563</td>\n",
              "      <td>-0.025489</td>\n",
              "      <td>1.494313</td>\n",
              "      <td>0.277364</td>\n",
              "      <td>0.357917</td>\n",
              "      <td>...</td>\n",
              "      <td>0.972868</td>\n",
              "      <td>0.623104</td>\n",
              "      <td>0.151891</td>\n",
              "      <td>0.986884</td>\n",
              "      <td>-0.530927</td>\n",
              "      <td>-0.623886</td>\n",
              "      <td>-0.024449</td>\n",
              "      <td>-0.122968</td>\n",
              "      <td>-0.843806</td>\n",
              "      <td>0.147631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.749489</td>\n",
              "      <td>-0.299404</td>\n",
              "      <td>-0.459100</td>\n",
              "      <td>0.774708</td>\n",
              "      <td>2.344556</td>\n",
              "      <td>-0.856449</td>\n",
              "      <td>-2.323390</td>\n",
              "      <td>0.298781</td>\n",
              "      <td>...</td>\n",
              "      <td>0.968908</td>\n",
              "      <td>-0.512865</td>\n",
              "      <td>0.940177</td>\n",
              "      <td>0.797558</td>\n",
              "      <td>0.696072</td>\n",
              "      <td>-2.188802</td>\n",
              "      <td>-0.587456</td>\n",
              "      <td>-1.290193</td>\n",
              "      <td>0.755513</td>\n",
              "      <td>0.681749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.460555</td>\n",
              "      <td>-0.508226</td>\n",
              "      <td>0.959313</td>\n",
              "      <td>0.984009</td>\n",
              "      <td>1.451890</td>\n",
              "      <td>-0.867329</td>\n",
              "      <td>-0.342599</td>\n",
              "      <td>-0.234770</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.408597</td>\n",
              "      <td>0.222136</td>\n",
              "      <td>-0.328591</td>\n",
              "      <td>0.115461</td>\n",
              "      <td>-0.613891</td>\n",
              "      <td>-0.266831</td>\n",
              "      <td>1.858980</td>\n",
              "      <td>0.385555</td>\n",
              "      <td>0.229644</td>\n",
              "      <td>0.888441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.599819</td>\n",
              "      <td>-0.723658</td>\n",
              "      <td>2.333805</td>\n",
              "      <td>-0.964385</td>\n",
              "      <td>1.075072</td>\n",
              "      <td>1.786462</td>\n",
              "      <td>0.131113</td>\n",
              "      <td>-0.812649</td>\n",
              "      <td>...</td>\n",
              "      <td>1.074865</td>\n",
              "      <td>-0.015002</td>\n",
              "      <td>0.154258</td>\n",
              "      <td>-0.425031</td>\n",
              "      <td>0.060053</td>\n",
              "      <td>-0.950791</td>\n",
              "      <td>0.272566</td>\n",
              "      <td>-0.709891</td>\n",
              "      <td>0.183526</td>\n",
              "      <td>-0.035548</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.865375</td>\n",
              "      <td>-0.307222</td>\n",
              "      <td>1.988666</td>\n",
              "      <td>0.523322</td>\n",
              "      <td>0.644383</td>\n",
              "      <td>0.437283</td>\n",
              "      <td>-0.989775</td>\n",
              "      <td>-0.015526</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.420880</td>\n",
              "      <td>-0.435511</td>\n",
              "      <td>1.148459</td>\n",
              "      <td>0.378976</td>\n",
              "      <td>-1.053755</td>\n",
              "      <td>-1.137182</td>\n",
              "      <td>-0.378525</td>\n",
              "      <td>0.875301</td>\n",
              "      <td>2.879233</td>\n",
              "      <td>-2.048089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.575003</td>\n",
              "      <td>-0.170529</td>\n",
              "      <td>0.221620</td>\n",
              "      <td>-1.053665</td>\n",
              "      <td>0.049099</td>\n",
              "      <td>0.641676</td>\n",
              "      <td>0.016354</td>\n",
              "      <td>1.208632</td>\n",
              "      <td>...</td>\n",
              "      <td>0.902644</td>\n",
              "      <td>-1.134607</td>\n",
              "      <td>0.028057</td>\n",
              "      <td>-0.469159</td>\n",
              "      <td>-1.043663</td>\n",
              "      <td>0.210421</td>\n",
              "      <td>-0.909664</td>\n",
              "      <td>-0.352517</td>\n",
              "      <td>-0.612421</td>\n",
              "      <td>-0.247028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-1.635710</td>\n",
              "      <td>-1.976022</td>\n",
              "      <td>-0.636890</td>\n",
              "      <td>1.330359</td>\n",
              "      <td>-1.719197</td>\n",
              "      <td>-0.259467</td>\n",
              "      <td>-0.455103</td>\n",
              "      <td>-1.308468</td>\n",
              "      <td>...</td>\n",
              "      <td>0.193346</td>\n",
              "      <td>-0.775846</td>\n",
              "      <td>-1.107324</td>\n",
              "      <td>0.405084</td>\n",
              "      <td>0.886731</td>\n",
              "      <td>-0.560762</td>\n",
              "      <td>-0.039118</td>\n",
              "      <td>0.349216</td>\n",
              "      <td>-0.017659</td>\n",
              "      <td>-0.805854</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.754997</td>\n",
              "      <td>0.494512</td>\n",
              "      <td>0.294161</td>\n",
              "      <td>-0.062373</td>\n",
              "      <td>-0.102483</td>\n",
              "      <td>1.097825</td>\n",
              "      <td>-1.164675</td>\n",
              "      <td>0.639927</td>\n",
              "      <td>...</td>\n",
              "      <td>1.485049</td>\n",
              "      <td>0.226634</td>\n",
              "      <td>-0.335871</td>\n",
              "      <td>-0.979968</td>\n",
              "      <td>0.111319</td>\n",
              "      <td>0.310239</td>\n",
              "      <td>-0.072833</td>\n",
              "      <td>1.234607</td>\n",
              "      <td>-0.962043</td>\n",
              "      <td>0.383007</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 939 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       cp_time  cp_dose       g-0       g-1       g-2       g-3       g-4  \\\n",
              "0            0        0  1.124260  0.896698 -0.436214 -0.965311 -0.287443   \n",
              "1            2        0  0.117451  0.667759  0.260124  0.097531  1.204172   \n",
              "2            1        0  0.777229  0.935347  1.414044 -0.113563 -0.025489   \n",
              "3            1        0 -0.749489 -0.299404 -0.459100  0.774708  2.344556   \n",
              "4            2        1 -0.460555 -0.508226  0.959313  0.984009  1.451890   \n",
              "...        ...      ...       ...       ...       ...       ...       ...   \n",
              "25567        0        0  0.599819 -0.723658  2.333805 -0.964385  1.075072   \n",
              "25568        0        0 -0.865375 -0.307222  1.988666  0.523322  0.644383   \n",
              "25569        2        0 -0.575003 -0.170529  0.221620 -1.053665  0.049099   \n",
              "25570        1        1 -1.635710 -1.976022 -0.636890  1.330359 -1.719197   \n",
              "25571        2        0 -0.754997  0.494512  0.294161 -0.062373 -0.102483   \n",
              "\n",
              "            g-5       g-6       g-7  ...   pca_C-5   pca_C-6   pca_C-7  \\\n",
              "0     -1.016437 -1.360774 -0.045876  ...  0.984003 -0.450318  0.336426   \n",
              "1      0.692876  0.356691  0.559630  ... -0.505982  0.099885  0.527028   \n",
              "2      1.494313  0.277364  0.357917  ...  0.972868  0.623104  0.151891   \n",
              "3     -0.856449 -2.323390  0.298781  ...  0.968908 -0.512865  0.940177   \n",
              "4     -0.867329 -0.342599 -0.234770  ... -0.408597  0.222136 -0.328591   \n",
              "...         ...       ...       ...  ...       ...       ...       ...   \n",
              "25567  1.786462  0.131113 -0.812649  ...  1.074865 -0.015002  0.154258   \n",
              "25568  0.437283 -0.989775 -0.015526  ... -1.420880 -0.435511  1.148459   \n",
              "25569  0.641676  0.016354  1.208632  ...  0.902644 -1.134607  0.028057   \n",
              "25570 -0.259467 -0.455103 -1.308468  ...  0.193346 -0.775846 -1.107324   \n",
              "25571  1.097825 -1.164675  0.639927  ...  1.485049  0.226634 -0.335871   \n",
              "\n",
              "        pca_C-8   pca_C-9  pca_C-10  pca_C-11  pca_C-12  pca_C-13  pca_C-14  \n",
              "0      0.015646  0.244094 -0.336008  0.062116 -0.398600 -0.044412 -1.430930  \n",
              "1      1.249065 -0.497161 -1.017310  0.180813 -0.877796  1.173441 -0.533822  \n",
              "2      0.986884 -0.530927 -0.623886 -0.024449 -0.122968 -0.843806  0.147631  \n",
              "3      0.797558  0.696072 -2.188802 -0.587456 -1.290193  0.755513  0.681749  \n",
              "4      0.115461 -0.613891 -0.266831  1.858980  0.385555  0.229644  0.888441  \n",
              "...         ...       ...       ...       ...       ...       ...       ...  \n",
              "25567 -0.425031  0.060053 -0.950791  0.272566 -0.709891  0.183526 -0.035548  \n",
              "25568  0.378976 -1.053755 -1.137182 -0.378525  0.875301  2.879233 -2.048089  \n",
              "25569 -0.469159 -1.043663  0.210421 -0.909664 -0.352517 -0.612421 -0.247028  \n",
              "25570  0.405084  0.886731 -0.560762 -0.039118  0.349216 -0.017659 -0.805854  \n",
              "25571 -0.979968  0.111319  0.310239 -0.072833  1.234607 -0.962043  0.383007  \n",
              "\n",
              "[25572 rows x 939 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:44.237017Z",
          "iopub.status.busy": "2020-11-30T00:02:44.235747Z",
          "iopub.status.idle": "2020-11-30T00:02:44.487257Z",
          "shell.execute_reply": "2020-11-30T00:02:44.486707Z"
        },
        "id": "GIHl5F3x7OT7",
        "papermill": {
          "duration": 0.34031,
          "end_time": "2020-11-30T00:02:44.487365",
          "exception": false,
          "start_time": "2020-11-30T00:02:44.147055",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "train_pca = train.copy()\n",
        "test_pca = test.copy()\n",
        "\n",
        "train_pca.drop(g_cols, axis=1, inplace=True)\n",
        "test_pca.drop(g_cols, axis=1, inplace=True)\n",
        "\n",
        "train_pca.drop(c_cols, axis=1, inplace=True)\n",
        "test_pca.drop(c_cols, axis=1, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:44.663130Z",
          "iopub.status.busy": "2020-11-30T00:02:44.662149Z",
          "iopub.status.idle": "2020-11-30T00:02:44.695809Z",
          "shell.execute_reply": "2020-11-30T00:02:44.696332Z"
        },
        "id": "nJyx4dSR7OT9",
        "papermill": {
          "duration": 0.125696,
          "end_time": "2020-11-30T00:02:44.696488",
          "exception": false,
          "start_time": "2020-11-30T00:02:44.570792",
          "status": "completed"
        },
        "tags": [],
        "outputId": "fe18631b-6fb4-4e6c-c04c-f64d6ce91427"
      },
      "source": [
        "train_pca"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cp_time</th>\n",
              "      <th>cp_dose</th>\n",
              "      <th>pca_G-0</th>\n",
              "      <th>pca_G-1</th>\n",
              "      <th>pca_G-2</th>\n",
              "      <th>pca_G-3</th>\n",
              "      <th>pca_G-4</th>\n",
              "      <th>pca_G-5</th>\n",
              "      <th>pca_G-6</th>\n",
              "      <th>pca_G-7</th>\n",
              "      <th>...</th>\n",
              "      <th>pca_C-5</th>\n",
              "      <th>pca_C-6</th>\n",
              "      <th>pca_C-7</th>\n",
              "      <th>pca_C-8</th>\n",
              "      <th>pca_C-9</th>\n",
              "      <th>pca_C-10</th>\n",
              "      <th>pca_C-11</th>\n",
              "      <th>pca_C-12</th>\n",
              "      <th>pca_C-13</th>\n",
              "      <th>pca_C-14</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-5.778899</td>\n",
              "      <td>6.154613</td>\n",
              "      <td>8.561315</td>\n",
              "      <td>-7.442511</td>\n",
              "      <td>4.386002</td>\n",
              "      <td>1.258147</td>\n",
              "      <td>3.520685</td>\n",
              "      <td>1.828526</td>\n",
              "      <td>...</td>\n",
              "      <td>0.984003</td>\n",
              "      <td>-0.450318</td>\n",
              "      <td>0.336426</td>\n",
              "      <td>0.015646</td>\n",
              "      <td>0.244094</td>\n",
              "      <td>-0.336008</td>\n",
              "      <td>0.062116</td>\n",
              "      <td>-0.398600</td>\n",
              "      <td>-0.044412</td>\n",
              "      <td>-1.430930</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-5.035246</td>\n",
              "      <td>1.003536</td>\n",
              "      <td>-12.642795</td>\n",
              "      <td>4.682019</td>\n",
              "      <td>0.934481</td>\n",
              "      <td>0.017921</td>\n",
              "      <td>0.817860</td>\n",
              "      <td>-1.085419</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.505982</td>\n",
              "      <td>0.099885</td>\n",
              "      <td>0.527028</td>\n",
              "      <td>1.249065</td>\n",
              "      <td>-0.497161</td>\n",
              "      <td>-1.017310</td>\n",
              "      <td>0.180813</td>\n",
              "      <td>-0.877796</td>\n",
              "      <td>1.173441</td>\n",
              "      <td>-0.533822</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.849837</td>\n",
              "      <td>-8.534120</td>\n",
              "      <td>-2.961085</td>\n",
              "      <td>0.234691</td>\n",
              "      <td>0.712903</td>\n",
              "      <td>3.226471</td>\n",
              "      <td>-1.540530</td>\n",
              "      <td>3.543482</td>\n",
              "      <td>...</td>\n",
              "      <td>0.972868</td>\n",
              "      <td>0.623104</td>\n",
              "      <td>0.151891</td>\n",
              "      <td>0.986884</td>\n",
              "      <td>-0.530927</td>\n",
              "      <td>-0.623886</td>\n",
              "      <td>-0.024449</td>\n",
              "      <td>-0.122968</td>\n",
              "      <td>-0.843806</td>\n",
              "      <td>0.147631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>11.053726</td>\n",
              "      <td>-10.088315</td>\n",
              "      <td>-0.812731</td>\n",
              "      <td>-4.941979</td>\n",
              "      <td>-7.323094</td>\n",
              "      <td>-2.490876</td>\n",
              "      <td>-2.273711</td>\n",
              "      <td>6.357738</td>\n",
              "      <td>...</td>\n",
              "      <td>0.968908</td>\n",
              "      <td>-0.512865</td>\n",
              "      <td>0.940177</td>\n",
              "      <td>0.797558</td>\n",
              "      <td>0.696072</td>\n",
              "      <td>-2.188802</td>\n",
              "      <td>-0.587456</td>\n",
              "      <td>-1.290193</td>\n",
              "      <td>0.755513</td>\n",
              "      <td>0.681749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-6.813030</td>\n",
              "      <td>-5.481174</td>\n",
              "      <td>-9.282727</td>\n",
              "      <td>-4.827295</td>\n",
              "      <td>-7.899419</td>\n",
              "      <td>-8.227711</td>\n",
              "      <td>-3.362620</td>\n",
              "      <td>-3.581454</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.408597</td>\n",
              "      <td>0.222136</td>\n",
              "      <td>-0.328591</td>\n",
              "      <td>0.115461</td>\n",
              "      <td>-0.613891</td>\n",
              "      <td>-0.266831</td>\n",
              "      <td>1.858980</td>\n",
              "      <td>0.385555</td>\n",
              "      <td>0.229644</td>\n",
              "      <td>0.888441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3.034423</td>\n",
              "      <td>-2.604996</td>\n",
              "      <td>0.378983</td>\n",
              "      <td>1.080481</td>\n",
              "      <td>4.262611</td>\n",
              "      <td>2.492815</td>\n",
              "      <td>3.584577</td>\n",
              "      <td>-0.012864</td>\n",
              "      <td>...</td>\n",
              "      <td>1.074865</td>\n",
              "      <td>-0.015002</td>\n",
              "      <td>0.154258</td>\n",
              "      <td>-0.425031</td>\n",
              "      <td>0.060053</td>\n",
              "      <td>-0.950791</td>\n",
              "      <td>0.272566</td>\n",
              "      <td>-0.709891</td>\n",
              "      <td>0.183526</td>\n",
              "      <td>-0.035548</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-7.989301</td>\n",
              "      <td>-0.778425</td>\n",
              "      <td>-4.860383</td>\n",
              "      <td>0.376690</td>\n",
              "      <td>-1.113370</td>\n",
              "      <td>-2.287973</td>\n",
              "      <td>-5.796794</td>\n",
              "      <td>1.580867</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.420880</td>\n",
              "      <td>-0.435511</td>\n",
              "      <td>1.148459</td>\n",
              "      <td>0.378976</td>\n",
              "      <td>-1.053755</td>\n",
              "      <td>-1.137182</td>\n",
              "      <td>-0.378525</td>\n",
              "      <td>0.875301</td>\n",
              "      <td>2.879233</td>\n",
              "      <td>-2.048089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-6.872688</td>\n",
              "      <td>6.782780</td>\n",
              "      <td>1.654480</td>\n",
              "      <td>-7.876308</td>\n",
              "      <td>1.163434</td>\n",
              "      <td>2.100182</td>\n",
              "      <td>4.330693</td>\n",
              "      <td>-0.996571</td>\n",
              "      <td>...</td>\n",
              "      <td>0.902644</td>\n",
              "      <td>-1.134607</td>\n",
              "      <td>0.028057</td>\n",
              "      <td>-0.469159</td>\n",
              "      <td>-1.043663</td>\n",
              "      <td>0.210421</td>\n",
              "      <td>-0.909664</td>\n",
              "      <td>-0.352517</td>\n",
              "      <td>-0.612421</td>\n",
              "      <td>-0.247028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-1.134083</td>\n",
              "      <td>-9.890526</td>\n",
              "      <td>11.790893</td>\n",
              "      <td>7.032540</td>\n",
              "      <td>2.695275</td>\n",
              "      <td>-2.669481</td>\n",
              "      <td>2.486435</td>\n",
              "      <td>-0.267855</td>\n",
              "      <td>...</td>\n",
              "      <td>0.193346</td>\n",
              "      <td>-0.775846</td>\n",
              "      <td>-1.107324</td>\n",
              "      <td>0.405084</td>\n",
              "      <td>0.886731</td>\n",
              "      <td>-0.560762</td>\n",
              "      <td>-0.039118</td>\n",
              "      <td>0.349216</td>\n",
              "      <td>-0.017659</td>\n",
              "      <td>-0.805854</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.079644</td>\n",
              "      <td>3.391940</td>\n",
              "      <td>-0.898045</td>\n",
              "      <td>-4.511296</td>\n",
              "      <td>1.969612</td>\n",
              "      <td>-5.755667</td>\n",
              "      <td>-6.406180</td>\n",
              "      <td>0.154235</td>\n",
              "      <td>...</td>\n",
              "      <td>1.485049</td>\n",
              "      <td>0.226634</td>\n",
              "      <td>-0.335871</td>\n",
              "      <td>-0.979968</td>\n",
              "      <td>0.111319</td>\n",
              "      <td>0.310239</td>\n",
              "      <td>-0.072833</td>\n",
              "      <td>1.234607</td>\n",
              "      <td>-0.962043</td>\n",
              "      <td>0.383007</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 67 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       cp_time  cp_dose    pca_G-0    pca_G-1    pca_G-2   pca_G-3   pca_G-4  \\\n",
              "0            0        0  -5.778899   6.154613   8.561315 -7.442511  4.386002   \n",
              "1            2        0  -5.035246   1.003536 -12.642795  4.682019  0.934481   \n",
              "2            1        0   0.849837  -8.534120  -2.961085  0.234691  0.712903   \n",
              "3            1        0  11.053726 -10.088315  -0.812731 -4.941979 -7.323094   \n",
              "4            2        1  -6.813030  -5.481174  -9.282727 -4.827295 -7.899419   \n",
              "...        ...      ...        ...        ...        ...       ...       ...   \n",
              "25567        0        0   3.034423  -2.604996   0.378983  1.080481  4.262611   \n",
              "25568        0        0  -7.989301  -0.778425  -4.860383  0.376690 -1.113370   \n",
              "25569        2        0  -6.872688   6.782780   1.654480 -7.876308  1.163434   \n",
              "25570        1        1  -1.134083  -9.890526  11.790893  7.032540  2.695275   \n",
              "25571        2        0  -1.079644   3.391940  -0.898045 -4.511296  1.969612   \n",
              "\n",
              "        pca_G-5   pca_G-6   pca_G-7  ...   pca_C-5   pca_C-6   pca_C-7  \\\n",
              "0      1.258147  3.520685  1.828526  ...  0.984003 -0.450318  0.336426   \n",
              "1      0.017921  0.817860 -1.085419  ... -0.505982  0.099885  0.527028   \n",
              "2      3.226471 -1.540530  3.543482  ...  0.972868  0.623104  0.151891   \n",
              "3     -2.490876 -2.273711  6.357738  ...  0.968908 -0.512865  0.940177   \n",
              "4     -8.227711 -3.362620 -3.581454  ... -0.408597  0.222136 -0.328591   \n",
              "...         ...       ...       ...  ...       ...       ...       ...   \n",
              "25567  2.492815  3.584577 -0.012864  ...  1.074865 -0.015002  0.154258   \n",
              "25568 -2.287973 -5.796794  1.580867  ... -1.420880 -0.435511  1.148459   \n",
              "25569  2.100182  4.330693 -0.996571  ...  0.902644 -1.134607  0.028057   \n",
              "25570 -2.669481  2.486435 -0.267855  ...  0.193346 -0.775846 -1.107324   \n",
              "25571 -5.755667 -6.406180  0.154235  ...  1.485049  0.226634 -0.335871   \n",
              "\n",
              "        pca_C-8   pca_C-9  pca_C-10  pca_C-11  pca_C-12  pca_C-13  pca_C-14  \n",
              "0      0.015646  0.244094 -0.336008  0.062116 -0.398600 -0.044412 -1.430930  \n",
              "1      1.249065 -0.497161 -1.017310  0.180813 -0.877796  1.173441 -0.533822  \n",
              "2      0.986884 -0.530927 -0.623886 -0.024449 -0.122968 -0.843806  0.147631  \n",
              "3      0.797558  0.696072 -2.188802 -0.587456 -1.290193  0.755513  0.681749  \n",
              "4      0.115461 -0.613891 -0.266831  1.858980  0.385555  0.229644  0.888441  \n",
              "...         ...       ...       ...       ...       ...       ...       ...  \n",
              "25567 -0.425031  0.060053 -0.950791  0.272566 -0.709891  0.183526 -0.035548  \n",
              "25568  0.378976 -1.053755 -1.137182 -0.378525  0.875301  2.879233 -2.048089  \n",
              "25569 -0.469159 -1.043663  0.210421 -0.909664 -0.352517 -0.612421 -0.247028  \n",
              "25570  0.405084  0.886731 -0.560762 -0.039118  0.349216 -0.017659 -0.805854  \n",
              "25571 -0.979968  0.111319  0.310239 -0.072833  1.234607 -0.962043  0.383007  \n",
              "\n",
              "[25572 rows x 67 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ujMOcMFG7OUD",
        "papermill": {
          "duration": 0.07971,
          "end_time": "2020-11-30T00:02:44.863605",
          "exception": false,
          "start_time": "2020-11-30T00:02:44.783895",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## feature Selection using Variance Encoding\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:45.034013Z",
          "iopub.status.busy": "2020-11-30T00:02:45.033250Z",
          "iopub.status.idle": "2020-11-30T00:02:45.036568Z",
          "shell.execute_reply": "2020-11-30T00:02:45.037003Z"
        },
        "id": "OR4sYLWT7OUE",
        "papermill": {
          "duration": 0.093507,
          "end_time": "2020-11-30T00:02:45.037124",
          "exception": false,
          "start_time": "2020-11-30T00:02:44.943617",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# https://www.kaggle.com/c/lish-moa/discussion/194973#1067941\n",
        "if False:\n",
        "\n",
        "    var_threshold = 0.5\n",
        "\n",
        "    data = train.append(test)\n",
        "    ve_columns = (data.iloc[:, 2:].var() >= var_threshold).values\n",
        "    ve_data = data.iloc[:, 2:].loc[:, ve_columns]\n",
        "\n",
        "    ve_train = ve_data[: train.shape[0]]\n",
        "    ve_test = ve_data[-test.shape[0] :]\n",
        "\n",
        "    train = pd.DataFrame(train[[\"cp_time\", \"cp_dose\"]].values.reshape(-1, 2), columns=[\"cp_time\", \"cp_dose\"])\n",
        "    train = pd.concat([train, ve_train], axis=1)\n",
        "\n",
        "    test = pd.DataFrame(test[[\"cp_time\", \"cp_dose\"]].values.reshape(-1, 2), columns=[\"cp_time\", \"cp_dose\"])\n",
        "    test = pd.concat([test, ve_test], axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:45.202781Z",
          "iopub.status.busy": "2020-11-30T00:02:45.201932Z",
          "iopub.status.idle": "2020-11-30T00:02:45.207044Z",
          "shell.execute_reply": "2020-11-30T00:02:45.206415Z"
        },
        "id": "lvb-RhDK7OUG",
        "papermill": {
          "duration": 0.090649,
          "end_time": "2020-11-30T00:02:45.207154",
          "exception": false,
          "start_time": "2020-11-30T00:02:45.116505",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ekFLgTbpKfA4",
        "papermill": {
          "duration": 0.079906,
          "end_time": "2020-11-30T00:02:45.378368",
          "exception": false,
          "start_time": "2020-11-30T00:02:45.298462",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## KMeans"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:02:45.552624Z",
          "iopub.status.busy": "2020-11-30T00:02:45.551366Z",
          "iopub.status.idle": "2020-11-30T00:04:13.008249Z",
          "shell.execute_reply": "2020-11-30T00:04:13.008950Z"
        },
        "id": "MA5PfEXjKg5O",
        "papermill": {
          "duration": 87.549513,
          "end_time": "2020-11-30T00:04:13.009132",
          "exception": false,
          "start_time": "2020-11-30T00:02:45.459619",
          "status": "completed"
        },
        "tags": [],
        "outputId": "e590ac84-6990-470d-aa7e-180bd379c8f4"
      },
      "source": [
        "%%time\n",
        "\n",
        "features_g = [col for col in train.columns if col.startswith(\"g-\")]\n",
        "features_c = [col for col in train.columns if col.startswith(\"c-\")]\n",
        "\n",
        "\n",
        "def fe_cluster(train_, test_, n_clusters_g=35, n_clusters_c=5):\n",
        "    def create_cluster(tr, te, features, kind=\"g\", n_clusters=n_clusters_g):\n",
        "        tmp_train_ = tr[features].copy()\n",
        "        tmp_test_ = te[features].copy()\n",
        "        data = pd.concat([tmp_train_, tmp_test_], axis=0)\n",
        "\n",
        "        kmeans = KMeans(n_clusters=n_clusters, random_state=random_seed).fit(data)\n",
        "\n",
        "        tr[f\"clusters_{kind}\"] = kmeans.labels_[: tr.shape[0]]\n",
        "        te[f\"clusters_{kind}\"] = kmeans.labels_[-te.shape[0] :]\n",
        "        tr = pd.get_dummies(tr, columns=[f\"clusters_{kind}\"])\n",
        "        te = pd.get_dummies(te, columns=[f\"clusters_{kind}\"])\n",
        "        return tr, te\n",
        "\n",
        "    train_, test_ = create_cluster(train_, test_, features_g, kind=\"g\", n_clusters=n_clusters_g)\n",
        "    train_, test_ = create_cluster(train_, test_, features_c, kind=\"c\", n_clusters=n_clusters_c)\n",
        "    return train_, test_\n",
        "\n",
        "\n",
        "train, test = fe_cluster(train, test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1min 26s, sys: 449 ms, total: 1min 26s\n",
            "Wall time: 1min 27s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:13.177631Z",
          "iopub.status.busy": "2020-11-30T00:04:13.176233Z",
          "iopub.status.idle": "2020-11-30T00:04:13.209352Z",
          "shell.execute_reply": "2020-11-30T00:04:13.210018Z"
        },
        "id": "EZ7Cetl7KjfY",
        "papermill": {
          "duration": 0.120599,
          "end_time": "2020-11-30T00:04:13.210171",
          "exception": false,
          "start_time": "2020-11-30T00:04:13.089572",
          "status": "completed"
        },
        "tags": [],
        "outputId": "5a3eaae9-9592-4519-be27-b8c14e4a9426"
      },
      "source": [
        "train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cp_time</th>\n",
              "      <th>cp_dose</th>\n",
              "      <th>g-0</th>\n",
              "      <th>g-1</th>\n",
              "      <th>g-2</th>\n",
              "      <th>g-3</th>\n",
              "      <th>g-4</th>\n",
              "      <th>g-5</th>\n",
              "      <th>g-6</th>\n",
              "      <th>g-7</th>\n",
              "      <th>...</th>\n",
              "      <th>clusters_g_30</th>\n",
              "      <th>clusters_g_31</th>\n",
              "      <th>clusters_g_32</th>\n",
              "      <th>clusters_g_33</th>\n",
              "      <th>clusters_g_34</th>\n",
              "      <th>clusters_c_0</th>\n",
              "      <th>clusters_c_1</th>\n",
              "      <th>clusters_c_2</th>\n",
              "      <th>clusters_c_3</th>\n",
              "      <th>clusters_c_4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.124260</td>\n",
              "      <td>0.896698</td>\n",
              "      <td>-0.436214</td>\n",
              "      <td>-0.965311</td>\n",
              "      <td>-0.287443</td>\n",
              "      <td>-1.016437</td>\n",
              "      <td>-1.360774</td>\n",
              "      <td>-0.045876</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0.117451</td>\n",
              "      <td>0.667759</td>\n",
              "      <td>0.260124</td>\n",
              "      <td>0.097531</td>\n",
              "      <td>1.204172</td>\n",
              "      <td>0.692876</td>\n",
              "      <td>0.356691</td>\n",
              "      <td>0.559630</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.777229</td>\n",
              "      <td>0.935347</td>\n",
              "      <td>1.414044</td>\n",
              "      <td>-0.113563</td>\n",
              "      <td>-0.025489</td>\n",
              "      <td>1.494313</td>\n",
              "      <td>0.277364</td>\n",
              "      <td>0.357917</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.749489</td>\n",
              "      <td>-0.299404</td>\n",
              "      <td>-0.459100</td>\n",
              "      <td>0.774708</td>\n",
              "      <td>2.344556</td>\n",
              "      <td>-0.856449</td>\n",
              "      <td>-2.323390</td>\n",
              "      <td>0.298781</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.460555</td>\n",
              "      <td>-0.508226</td>\n",
              "      <td>0.959313</td>\n",
              "      <td>0.984009</td>\n",
              "      <td>1.451890</td>\n",
              "      <td>-0.867329</td>\n",
              "      <td>-0.342599</td>\n",
              "      <td>-0.234770</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.599819</td>\n",
              "      <td>-0.723658</td>\n",
              "      <td>2.333805</td>\n",
              "      <td>-0.964385</td>\n",
              "      <td>1.075072</td>\n",
              "      <td>1.786462</td>\n",
              "      <td>0.131113</td>\n",
              "      <td>-0.812649</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.865375</td>\n",
              "      <td>-0.307222</td>\n",
              "      <td>1.988666</td>\n",
              "      <td>0.523322</td>\n",
              "      <td>0.644383</td>\n",
              "      <td>0.437283</td>\n",
              "      <td>-0.989775</td>\n",
              "      <td>-0.015526</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.575003</td>\n",
              "      <td>-0.170529</td>\n",
              "      <td>0.221620</td>\n",
              "      <td>-1.053665</td>\n",
              "      <td>0.049099</td>\n",
              "      <td>0.641676</td>\n",
              "      <td>0.016354</td>\n",
              "      <td>1.208632</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-1.635710</td>\n",
              "      <td>-1.976022</td>\n",
              "      <td>-0.636890</td>\n",
              "      <td>1.330359</td>\n",
              "      <td>-1.719197</td>\n",
              "      <td>-0.259467</td>\n",
              "      <td>-0.455103</td>\n",
              "      <td>-1.308468</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.754997</td>\n",
              "      <td>0.494512</td>\n",
              "      <td>0.294161</td>\n",
              "      <td>-0.062373</td>\n",
              "      <td>-0.102483</td>\n",
              "      <td>1.097825</td>\n",
              "      <td>-1.164675</td>\n",
              "      <td>0.639927</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 979 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       cp_time  cp_dose       g-0       g-1       g-2       g-3       g-4  \\\n",
              "0            0        0  1.124260  0.896698 -0.436214 -0.965311 -0.287443   \n",
              "1            2        0  0.117451  0.667759  0.260124  0.097531  1.204172   \n",
              "2            1        0  0.777229  0.935347  1.414044 -0.113563 -0.025489   \n",
              "3            1        0 -0.749489 -0.299404 -0.459100  0.774708  2.344556   \n",
              "4            2        1 -0.460555 -0.508226  0.959313  0.984009  1.451890   \n",
              "...        ...      ...       ...       ...       ...       ...       ...   \n",
              "25567        0        0  0.599819 -0.723658  2.333805 -0.964385  1.075072   \n",
              "25568        0        0 -0.865375 -0.307222  1.988666  0.523322  0.644383   \n",
              "25569        2        0 -0.575003 -0.170529  0.221620 -1.053665  0.049099   \n",
              "25570        1        1 -1.635710 -1.976022 -0.636890  1.330359 -1.719197   \n",
              "25571        2        0 -0.754997  0.494512  0.294161 -0.062373 -0.102483   \n",
              "\n",
              "            g-5       g-6       g-7  ...  clusters_g_30  clusters_g_31  \\\n",
              "0     -1.016437 -1.360774 -0.045876  ...              0              0   \n",
              "1      0.692876  0.356691  0.559630  ...              0              0   \n",
              "2      1.494313  0.277364  0.357917  ...              0              0   \n",
              "3     -0.856449 -2.323390  0.298781  ...              0              0   \n",
              "4     -0.867329 -0.342599 -0.234770  ...              0              0   \n",
              "...         ...       ...       ...  ...            ...            ...   \n",
              "25567  1.786462  0.131113 -0.812649  ...              0              0   \n",
              "25568  0.437283 -0.989775 -0.015526  ...              0              0   \n",
              "25569  0.641676  0.016354  1.208632  ...              0              0   \n",
              "25570 -0.259467 -0.455103 -1.308468  ...              0              1   \n",
              "25571  1.097825 -1.164675  0.639927  ...              0              0   \n",
              "\n",
              "       clusters_g_32  clusters_g_33  clusters_g_34  clusters_c_0  \\\n",
              "0                  0              0              0             1   \n",
              "1                  1              0              0             1   \n",
              "2                  0              0              0             0   \n",
              "3                  0              0              0             0   \n",
              "4                  0              0              0             0   \n",
              "...              ...            ...            ...           ...   \n",
              "25567              0              0              0             0   \n",
              "25568              0              0              0             1   \n",
              "25569              0              0              0             1   \n",
              "25570              0              0              0             0   \n",
              "25571              0              0              0             0   \n",
              "\n",
              "       clusters_c_1  clusters_c_2  clusters_c_3  clusters_c_4  \n",
              "0                 0             0             0             0  \n",
              "1                 0             0             0             0  \n",
              "2                 1             0             0             0  \n",
              "3                 0             0             1             0  \n",
              "4                 0             0             0             1  \n",
              "...             ...           ...           ...           ...  \n",
              "25567             1             0             0             0  \n",
              "25568             0             0             0             0  \n",
              "25569             0             0             0             0  \n",
              "25570             0             0             0             1  \n",
              "25571             0             0             0             1  \n",
              "\n",
              "[25572 rows x 979 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iQeImT1CKmF1",
        "papermill": {
          "duration": 0.081017,
          "end_time": "2020-11-30T00:04:13.374194",
          "exception": false,
          "start_time": "2020-11-30T00:04:13.293177",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Basic stats"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:13.547978Z",
          "iopub.status.busy": "2020-11-30T00:04:13.546617Z",
          "iopub.status.idle": "2020-11-30T00:04:19.054739Z",
          "shell.execute_reply": "2020-11-30T00:04:19.053807Z"
        },
        "id": "Q-6Zi3IiKoFF",
        "papermill": {
          "duration": 5.599528,
          "end_time": "2020-11-30T00:04:19.054860",
          "exception": false,
          "start_time": "2020-11-30T00:04:13.455332",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "for stats in [\"sum\", \"mean\", \"std\", \"kurt\", \"skew\"]:\n",
        "    train[\"g_\" + stats] = getattr(train[features_g], stats)(axis=1)\n",
        "    train[\"c_\" + stats] = getattr(train[features_c], stats)(axis=1)\n",
        "    train[\"gc_\" + stats] = getattr(train[features_g + features_c], stats)(axis=1)\n",
        "\n",
        "    test[\"g_\" + stats] = getattr(test[features_g], stats)(axis=1)\n",
        "    test[\"c_\" + stats] = getattr(test[features_c], stats)(axis=1)\n",
        "    test[\"gc_\" + stats] = getattr(test[features_g + features_c], stats)(axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:19.232899Z",
          "iopub.status.busy": "2020-11-30T00:04:19.231710Z",
          "iopub.status.idle": "2020-11-30T00:04:19.267266Z",
          "shell.execute_reply": "2020-11-30T00:04:19.267998Z"
        },
        "id": "xi0-UnkaKqaw",
        "papermill": {
          "duration": 0.128371,
          "end_time": "2020-11-30T00:04:19.268154",
          "exception": false,
          "start_time": "2020-11-30T00:04:19.139783",
          "status": "completed"
        },
        "tags": [],
        "outputId": "3cc5f513-4439-4c74-f7c8-98ea0acbb949"
      },
      "source": [
        "train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cp_time</th>\n",
              "      <th>cp_dose</th>\n",
              "      <th>g-0</th>\n",
              "      <th>g-1</th>\n",
              "      <th>g-2</th>\n",
              "      <th>g-3</th>\n",
              "      <th>g-4</th>\n",
              "      <th>g-5</th>\n",
              "      <th>g-6</th>\n",
              "      <th>g-7</th>\n",
              "      <th>...</th>\n",
              "      <th>gc_mean</th>\n",
              "      <th>g_std</th>\n",
              "      <th>c_std</th>\n",
              "      <th>gc_std</th>\n",
              "      <th>g_kurt</th>\n",
              "      <th>c_kurt</th>\n",
              "      <th>gc_kurt</th>\n",
              "      <th>g_skew</th>\n",
              "      <th>c_skew</th>\n",
              "      <th>gc_skew</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.124260</td>\n",
              "      <td>0.896698</td>\n",
              "      <td>-0.436214</td>\n",
              "      <td>-0.965311</td>\n",
              "      <td>-0.287443</td>\n",
              "      <td>-1.016437</td>\n",
              "      <td>-1.360774</td>\n",
              "      <td>-0.045876</td>\n",
              "      <td>...</td>\n",
              "      <td>0.050884</td>\n",
              "      <td>0.868307</td>\n",
              "      <td>0.731294</td>\n",
              "      <td>0.869209</td>\n",
              "      <td>-0.270006</td>\n",
              "      <td>-0.321285</td>\n",
              "      <td>-0.270608</td>\n",
              "      <td>0.019115</td>\n",
              "      <td>0.073814</td>\n",
              "      <td>-0.015508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0.117451</td>\n",
              "      <td>0.667759</td>\n",
              "      <td>0.260124</td>\n",
              "      <td>0.097531</td>\n",
              "      <td>1.204172</td>\n",
              "      <td>0.692876</td>\n",
              "      <td>0.356691</td>\n",
              "      <td>0.559630</td>\n",
              "      <td>...</td>\n",
              "      <td>0.062270</td>\n",
              "      <td>0.850889</td>\n",
              "      <td>0.608372</td>\n",
              "      <td>0.842821</td>\n",
              "      <td>-0.217545</td>\n",
              "      <td>0.088938</td>\n",
              "      <td>-0.233240</td>\n",
              "      <td>0.045890</td>\n",
              "      <td>-0.163448</td>\n",
              "      <td>-0.041249</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.777229</td>\n",
              "      <td>0.935347</td>\n",
              "      <td>1.414044</td>\n",
              "      <td>-0.113563</td>\n",
              "      <td>-0.025489</td>\n",
              "      <td>1.494313</td>\n",
              "      <td>0.277364</td>\n",
              "      <td>0.357917</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.038900</td>\n",
              "      <td>0.941310</td>\n",
              "      <td>0.665178</td>\n",
              "      <td>0.914129</td>\n",
              "      <td>-0.356922</td>\n",
              "      <td>-0.182024</td>\n",
              "      <td>-0.286903</td>\n",
              "      <td>-0.044156</td>\n",
              "      <td>0.385872</td>\n",
              "      <td>-0.008376</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.749489</td>\n",
              "      <td>-0.299404</td>\n",
              "      <td>-0.459100</td>\n",
              "      <td>0.774708</td>\n",
              "      <td>2.344556</td>\n",
              "      <td>-0.856449</td>\n",
              "      <td>-2.323390</td>\n",
              "      <td>0.298781</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.136704</td>\n",
              "      <td>1.080671</td>\n",
              "      <td>0.576449</td>\n",
              "      <td>1.088267</td>\n",
              "      <td>-0.918764</td>\n",
              "      <td>3.952398</td>\n",
              "      <td>-0.959980</td>\n",
              "      <td>0.086528</td>\n",
              "      <td>1.953350</td>\n",
              "      <td>0.245358</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.460555</td>\n",
              "      <td>-0.508226</td>\n",
              "      <td>0.959313</td>\n",
              "      <td>0.984009</td>\n",
              "      <td>1.451890</td>\n",
              "      <td>-0.867329</td>\n",
              "      <td>-0.342599</td>\n",
              "      <td>-0.234770</td>\n",
              "      <td>...</td>\n",
              "      <td>0.020415</td>\n",
              "      <td>1.103348</td>\n",
              "      <td>0.677183</td>\n",
              "      <td>1.070526</td>\n",
              "      <td>-0.214614</td>\n",
              "      <td>-0.723722</td>\n",
              "      <td>-0.102022</td>\n",
              "      <td>-0.187344</td>\n",
              "      <td>0.076016</td>\n",
              "      <td>-0.251234</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25567</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.599819</td>\n",
              "      <td>-0.723658</td>\n",
              "      <td>2.333805</td>\n",
              "      <td>-0.964385</td>\n",
              "      <td>1.075072</td>\n",
              "      <td>1.786462</td>\n",
              "      <td>0.131113</td>\n",
              "      <td>-0.812649</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.105162</td>\n",
              "      <td>0.778264</td>\n",
              "      <td>0.472869</td>\n",
              "      <td>0.783517</td>\n",
              "      <td>-0.299977</td>\n",
              "      <td>0.946507</td>\n",
              "      <td>-0.386378</td>\n",
              "      <td>0.085385</td>\n",
              "      <td>0.903288</td>\n",
              "      <td>0.210787</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25568</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.865375</td>\n",
              "      <td>-0.307222</td>\n",
              "      <td>1.988666</td>\n",
              "      <td>0.523322</td>\n",
              "      <td>0.644383</td>\n",
              "      <td>0.437283</td>\n",
              "      <td>-0.989775</td>\n",
              "      <td>-0.015526</td>\n",
              "      <td>...</td>\n",
              "      <td>0.084645</td>\n",
              "      <td>0.737451</td>\n",
              "      <td>0.794342</td>\n",
              "      <td>0.776206</td>\n",
              "      <td>-0.242835</td>\n",
              "      <td>-0.665963</td>\n",
              "      <td>-0.291044</td>\n",
              "      <td>0.075617</td>\n",
              "      <td>-0.237591</td>\n",
              "      <td>0.122949</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25569</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.575003</td>\n",
              "      <td>-0.170529</td>\n",
              "      <td>0.221620</td>\n",
              "      <td>-1.053665</td>\n",
              "      <td>0.049099</td>\n",
              "      <td>0.641676</td>\n",
              "      <td>0.016354</td>\n",
              "      <td>1.208632</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.007073</td>\n",
              "      <td>0.817300</td>\n",
              "      <td>0.636256</td>\n",
              "      <td>0.822376</td>\n",
              "      <td>-0.345356</td>\n",
              "      <td>-0.251386</td>\n",
              "      <td>-0.375475</td>\n",
              "      <td>0.012237</td>\n",
              "      <td>-0.120099</td>\n",
              "      <td>-0.052690</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25570</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-1.635710</td>\n",
              "      <td>-1.976022</td>\n",
              "      <td>-0.636890</td>\n",
              "      <td>1.330359</td>\n",
              "      <td>-1.719197</td>\n",
              "      <td>-0.259467</td>\n",
              "      <td>-0.455103</td>\n",
              "      <td>-1.308468</td>\n",
              "      <td>...</td>\n",
              "      <td>0.139976</td>\n",
              "      <td>0.988002</td>\n",
              "      <td>0.631845</td>\n",
              "      <td>0.953971</td>\n",
              "      <td>-0.607105</td>\n",
              "      <td>-0.670234</td>\n",
              "      <td>-0.498840</td>\n",
              "      <td>-0.313186</td>\n",
              "      <td>0.111856</td>\n",
              "      <td>-0.319974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25571</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>-0.754997</td>\n",
              "      <td>0.494512</td>\n",
              "      <td>0.294161</td>\n",
              "      <td>-0.062373</td>\n",
              "      <td>-0.102483</td>\n",
              "      <td>1.097825</td>\n",
              "      <td>-1.164675</td>\n",
              "      <td>0.639927</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.055344</td>\n",
              "      <td>0.850755</td>\n",
              "      <td>0.721807</td>\n",
              "      <td>0.848050</td>\n",
              "      <td>-0.385637</td>\n",
              "      <td>-0.437851</td>\n",
              "      <td>-0.396671</td>\n",
              "      <td>0.122631</td>\n",
              "      <td>0.085609</td>\n",
              "      <td>0.080899</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25572 rows × 994 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       cp_time  cp_dose       g-0       g-1       g-2       g-3       g-4  \\\n",
              "0            0        0  1.124260  0.896698 -0.436214 -0.965311 -0.287443   \n",
              "1            2        0  0.117451  0.667759  0.260124  0.097531  1.204172   \n",
              "2            1        0  0.777229  0.935347  1.414044 -0.113563 -0.025489   \n",
              "3            1        0 -0.749489 -0.299404 -0.459100  0.774708  2.344556   \n",
              "4            2        1 -0.460555 -0.508226  0.959313  0.984009  1.451890   \n",
              "...        ...      ...       ...       ...       ...       ...       ...   \n",
              "25567        0        0  0.599819 -0.723658  2.333805 -0.964385  1.075072   \n",
              "25568        0        0 -0.865375 -0.307222  1.988666  0.523322  0.644383   \n",
              "25569        2        0 -0.575003 -0.170529  0.221620 -1.053665  0.049099   \n",
              "25570        1        1 -1.635710 -1.976022 -0.636890  1.330359 -1.719197   \n",
              "25571        2        0 -0.754997  0.494512  0.294161 -0.062373 -0.102483   \n",
              "\n",
              "            g-5       g-6       g-7  ...   gc_mean     g_std     c_std  \\\n",
              "0     -1.016437 -1.360774 -0.045876  ...  0.050884  0.868307  0.731294   \n",
              "1      0.692876  0.356691  0.559630  ...  0.062270  0.850889  0.608372   \n",
              "2      1.494313  0.277364  0.357917  ... -0.038900  0.941310  0.665178   \n",
              "3     -0.856449 -2.323390  0.298781  ... -0.136704  1.080671  0.576449   \n",
              "4     -0.867329 -0.342599 -0.234770  ...  0.020415  1.103348  0.677183   \n",
              "...         ...       ...       ...  ...       ...       ...       ...   \n",
              "25567  1.786462  0.131113 -0.812649  ... -0.105162  0.778264  0.472869   \n",
              "25568  0.437283 -0.989775 -0.015526  ...  0.084645  0.737451  0.794342   \n",
              "25569  0.641676  0.016354  1.208632  ... -0.007073  0.817300  0.636256   \n",
              "25570 -0.259467 -0.455103 -1.308468  ...  0.139976  0.988002  0.631845   \n",
              "25571  1.097825 -1.164675  0.639927  ... -0.055344  0.850755  0.721807   \n",
              "\n",
              "         gc_std    g_kurt    c_kurt   gc_kurt    g_skew    c_skew   gc_skew  \n",
              "0      0.869209 -0.270006 -0.321285 -0.270608  0.019115  0.073814 -0.015508  \n",
              "1      0.842821 -0.217545  0.088938 -0.233240  0.045890 -0.163448 -0.041249  \n",
              "2      0.914129 -0.356922 -0.182024 -0.286903 -0.044156  0.385872 -0.008376  \n",
              "3      1.088267 -0.918764  3.952398 -0.959980  0.086528  1.953350  0.245358  \n",
              "4      1.070526 -0.214614 -0.723722 -0.102022 -0.187344  0.076016 -0.251234  \n",
              "...         ...       ...       ...       ...       ...       ...       ...  \n",
              "25567  0.783517 -0.299977  0.946507 -0.386378  0.085385  0.903288  0.210787  \n",
              "25568  0.776206 -0.242835 -0.665963 -0.291044  0.075617 -0.237591  0.122949  \n",
              "25569  0.822376 -0.345356 -0.251386 -0.375475  0.012237 -0.120099 -0.052690  \n",
              "25570  0.953971 -0.607105 -0.670234 -0.498840 -0.313186  0.111856 -0.319974  \n",
              "25571  0.848050 -0.385637 -0.437851 -0.396671  0.122631  0.085609  0.080899  \n",
              "\n",
              "[25572 rows x 994 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nEU0TZF1vTgF",
        "papermill": {
          "duration": 0.095714,
          "end_time": "2020-11-30T00:04:19.462332",
          "exception": false,
          "start_time": "2020-11-30T00:04:19.366618",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Pick up categorical features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:19.656100Z",
          "iopub.status.busy": "2020-11-30T00:04:19.655064Z",
          "iopub.status.idle": "2020-11-30T00:04:19.658402Z",
          "shell.execute_reply": "2020-11-30T00:04:19.657847Z"
        },
        "id": "z0XGL1xtvXQp",
        "papermill": {
          "duration": 0.10344,
          "end_time": "2020-11-30T00:04:19.658540",
          "exception": false,
          "start_time": "2020-11-30T00:04:19.555100",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if False:\n",
        "    features_cat = [col for col in train.columns if col.startswith(\"cp_\") or col.startswith(\"clusters_\")]\n",
        "    features_cat_index = [train.columns.get_loc(col) for col in features_cat]\n",
        "\n",
        "    train_cat = train[features_cat].values.astype(\"int8\")\n",
        "    test_cat = test[features_cat].values.astype(\"int8\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:19.842311Z",
          "iopub.status.busy": "2020-11-30T00:04:19.841391Z",
          "iopub.status.idle": "2020-11-30T00:04:19.844672Z",
          "shell.execute_reply": "2020-11-30T00:04:19.844133Z"
        },
        "id": "HnDew1q2w5rb",
        "papermill": {
          "duration": 0.090862,
          "end_time": "2020-11-30T00:04:19.844789",
          "exception": false,
          "start_time": "2020-11-30T00:04:19.753927",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "#features_cat"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DM7P7qCtCqal",
        "papermill": {
          "duration": 0.084263,
          "end_time": "2020-11-30T00:04:20.014363",
          "exception": false,
          "start_time": "2020-11-30T00:04:19.930100",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Reduce mem"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:20.225329Z",
          "iopub.status.busy": "2020-11-30T00:04:20.215209Z",
          "iopub.status.idle": "2020-11-30T00:04:57.198592Z",
          "shell.execute_reply": "2020-11-30T00:04:57.199051Z"
        },
        "id": "NwUNWDd6DH9x",
        "papermill": {
          "duration": 37.099932,
          "end_time": "2020-11-30T00:04:57.199190",
          "exception": false,
          "start_time": "2020-11-30T00:04:20.099258",
          "status": "completed"
        },
        "tags": [],
        "outputId": "ee096835-85c6-429d-9b68-acac5fbcdc8c"
      },
      "source": [
        "train = reduce_mem_usage(train)\n",
        "test = reduce_mem_usage(test)\n",
        "\n",
        "train_pca = reduce_mem_usage(train_pca)\n",
        "test_pca = reduce_mem_usage(test_pca)\n",
        "\n",
        "target = reduce_mem_usage(target)\n",
        "non_target = reduce_mem_usage(non_target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Memory usage of dataframe is 187.10 MB\n",
            "Memory usage after optimization is: 48.43 MB\n",
            "Decreased by 74.1%\n",
            "Memory usage of dataframe is 29.13 MB\n",
            "Memory usage after optimization is: 7.54 MB\n",
            "Decreased by 74.1%\n",
            "Memory usage of dataframe is 13.07 MB\n",
            "Memory usage after optimization is: 3.22 MB\n",
            "Decreased by 75.4%\n",
            "Memory usage of dataframe is 2.04 MB\n",
            "Memory usage after optimization is: 0.50 MB\n",
            "Decreased by 75.4%\n",
            "Memory usage of dataframe is 40.19 MB\n",
            "Memory usage after optimization is: 10.05 MB\n",
            "Decreased by 75.0%\n",
            "Memory usage of dataframe is 67.32 MB\n",
            "Memory usage after optimization is: 8.41 MB\n",
            "Decreased by 87.5%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HEaEZxMKbLUT",
        "papermill": {
          "duration": 0.085775,
          "end_time": "2020-11-30T00:04:57.373747",
          "exception": false,
          "start_time": "2020-11-30T00:04:57.287972",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nUBSHNuL7OUI",
        "papermill": {
          "duration": 0.084422,
          "end_time": "2020-11-30T00:04:57.542615",
          "exception": false,
          "start_time": "2020-11-30T00:04:57.458193",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Multi input ResNet\n",
        "\n",
        "[MoA: Multi Input ResNet Model](https://www.kaggle.com/rahulsd91/moa-multi-input-resnet-model)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:57.740508Z",
          "iopub.status.busy": "2020-11-30T00:04:57.726230Z",
          "iopub.status.idle": "2020-11-30T00:04:57.744171Z",
          "shell.execute_reply": "2020-11-30T00:04:57.743662Z"
        },
        "id": "w4ySRylL7OUI",
        "papermill": {
          "duration": 0.113935,
          "end_time": "2020-11-30T00:04:57.744288",
          "exception": false,
          "start_time": "2020-11-30T00:04:57.630353",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def create_model_resnet(n_features, n_features_2, n_labels):\n",
        "    input_1 = L.Input(shape=(n_features,), name=\"Input1\")\n",
        "    input_2 = L.Input(shape=(n_features_2,), name=\"Input2\")\n",
        "\n",
        "    block_1 = tf.keras.Sequential(\n",
        "        [\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(128, activation=\"elu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(512, activation=\"selu\")),\n",
        "        ],\n",
        "        name=\"Block1\",\n",
        "    )\n",
        "\n",
        "    output_1 = block_1(input_1)\n",
        "    connection_1 = L.Concatenate(name=\"Connection1\")([input_2, output_1])\n",
        "\n",
        "    block_2 = tf.keras.Sequential(\n",
        "        [\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(256, activation=\"swish\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(256, activation=\"elu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(256, activation=\"relu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(1024, activation=\"relu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(512, activation=\"relu\")),\n",
        "        ],\n",
        "        name=\"Block2\",\n",
        "    )\n",
        "\n",
        "    output_2 = block_2(connection_1)\n",
        "    connection_2 = L.Average(name=\"Connection2\")([output_1, output_2])\n",
        "\n",
        "    block_3 = tf.keras.Sequential(\n",
        "        [\n",
        "            L.BatchNormalization(),\n",
        "            tfa.layers.WeightNormalization(L.Dense(128, activation=\"selu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(1024, activation=\"elu\")),\n",
        "            L.BatchNormalization(),\n",
        "        ],\n",
        "        name=\"Block3\",\n",
        "    )\n",
        "\n",
        "    output_3 = block_3(connection_2)\n",
        "\n",
        "    # output = L.Dense(n_labels, activation=\"sigmoid\", name=\"Output\")(output_3)\n",
        "    output = L.Dense(n_labels, name=\"Output\")(output_3)\n",
        "\n",
        "    model = tf.keras.models.Model(inputs=[input_1, input_2], outputs=output)\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:57.927093Z",
          "iopub.status.busy": "2020-11-30T00:04:57.926120Z",
          "iopub.status.idle": "2020-11-30T00:04:57.928834Z",
          "shell.execute_reply": "2020-11-30T00:04:57.929430Z"
        },
        "id": "V8ouPy9-qMSz",
        "papermill": {
          "duration": 0.097657,
          "end_time": "2020-11-30T00:04:57.929567",
          "exception": false,
          "start_time": "2020-11-30T00:04:57.831910",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if IN_COLAB:\n",
        "    model_test = create_model_resnet(len(train.columns), len(train_pca.columns), len(target.columns))\n",
        "    model_test.summary()\n",
        "    display_svg(SVG(model_to_dot(model_test, show_shapes=True, dpi=72).create(prog=\"dot\", format=\"svg\")))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:58.119698Z",
          "iopub.status.busy": "2020-11-30T00:04:58.118825Z",
          "iopub.status.idle": "2020-11-30T00:04:58.121298Z",
          "shell.execute_reply": "2020-11-30T00:04:58.121830Z"
        },
        "id": "mCGqI-8fyv7S",
        "papermill": {
          "duration": 0.107315,
          "end_time": "2020-11-30T00:04:58.121946",
          "exception": false,
          "start_time": "2020-11-30T00:04:58.014631",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def create_model_resnet_tuning(n_features, n_features_2, n_labels, params):\n",
        "    n_hidden_layers = params[\"n_layers\"]\n",
        "    units = params[\"units\"]\n",
        "    activations = params[\"activations\"]\n",
        "\n",
        "    input_1 = L.Input(shape=(n_features,), name=\"Input1\")\n",
        "    input_2 = L.Input(shape=(n_features_2,), name=\"Input2\")\n",
        "\n",
        "    block_1 = tf.keras.Sequential(\n",
        "        [\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(units[-3], activation=activations[-4])),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(units[n_hidden_layers - 1], activation=activations[-3])),\n",
        "        ],\n",
        "        name=\"Block1\",\n",
        "    )\n",
        "\n",
        "    output_1 = block_1(input_1)\n",
        "    connection_1 = L.Concatenate(name=\"Connection1\")([input_2, output_1])\n",
        "\n",
        "    layers_2 = []\n",
        "    for i in range(n_hidden_layers):\n",
        "        layers_2 += [\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(units[i], activation=activations[i])),\n",
        "        ]\n",
        "    block_2 = tf.keras.Sequential(layers_2, name=\"Block2\")\n",
        "\n",
        "    output_2 = block_2(connection_1)\n",
        "    connection_2 = L.Average(name=\"Connection2\")([output_1, output_2])\n",
        "\n",
        "    block_3 = tf.keras.Sequential(\n",
        "        [\n",
        "            L.BatchNormalization(),\n",
        "            tfa.layers.WeightNormalization(L.Dense(units[-2], activation=activations[-2])),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(units[-1], activation=activations[-1])),\n",
        "            L.BatchNormalization(),\n",
        "        ],\n",
        "        name=\"Block3\",\n",
        "    )\n",
        "\n",
        "    output_3 = block_3(connection_2)\n",
        "\n",
        "    # output = L.Dense(n_labels, activation=\"sigmoid\", name=\"Output\")(output_3)\n",
        "    output = L.Dense(n_labels, name=\"Output\")(output_3)\n",
        "\n",
        "    model = tf.keras.models.Model(inputs=[input_1, input_2], outputs=output)\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fiC-Q8MQ7OUK",
        "papermill": {
          "duration": 0.088592,
          "end_time": "2020-11-30T00:04:58.297415",
          "exception": false,
          "start_time": "2020-11-30T00:04:58.208823",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## TabNet\n",
        "\n",
        "[TabNet : Attentive Interpretable Tabular Learning](https://github.com/dreamquark-ai/tabnet)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:58.484641Z",
          "iopub.status.busy": "2020-11-30T00:04:58.483671Z",
          "iopub.status.idle": "2020-11-30T00:04:58.487017Z",
          "shell.execute_reply": "2020-11-30T00:04:58.486389Z"
        },
        "id": "o3z6PRzw7OUK",
        "papermill": {
          "duration": 0.101143,
          "end_time": "2020-11-30T00:04:58.487126",
          "exception": false,
          "start_time": "2020-11-30T00:04:58.385983",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def create_model_tabnet(seed, pre_train=False):\n",
        "    tabnet_params = dict(\n",
        "        n_d=32,\n",
        "        n_a=32,\n",
        "        n_steps=1,\n",
        "        n_independent=1,  # 2 is better CV than 1, but need more time\n",
        "        n_shared=1,  # same above\n",
        "        gamma=1.3,\n",
        "        lambda_sparse=0,\n",
        "        #cat_dims=[len(np.unique(train_cat[:, i])) for i in range(train_cat.shape[1])],\n",
        "        #cat_emb_dim=[1] * train_cat.shape[1],\n",
        "        #cat_idxs=features_cat_index,\n",
        "        optimizer_fn=optim.Adam,\n",
        "        optimizer_params=dict(lr=2e-2, weight_decay=1e-5),\n",
        "        mask_type=\"entmax\",\n",
        "        scheduler_params=dict(mode=\"min\", patience=5, min_lr=1e-5, threshold=1e-5, factor=0.1),\n",
        "        scheduler_fn=torch_ReduceLROnPlateau,\n",
        "        seed=seed,\n",
        "        verbose=0,\n",
        "    )\n",
        "\n",
        "    if pre_train:\n",
        "        model = TabNetPretrainer(**tabnet_params)\n",
        "    else:\n",
        "        model = TabNetRegressor(**tabnet_params)\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:58.673920Z",
          "iopub.status.busy": "2020-11-30T00:04:58.672125Z",
          "iopub.status.idle": "2020-11-30T00:04:58.674591Z",
          "shell.execute_reply": "2020-11-30T00:04:58.675060Z"
        },
        "id": "SJr0rKKtTOBR",
        "papermill": {
          "duration": 0.100261,
          "end_time": "2020-11-30T00:04:58.675177",
          "exception": false,
          "start_time": "2020-11-30T00:04:58.574916",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def create_model_tabnet_tuning(seed, params=None):\n",
        "    tabnet_params = dict(\n",
        "        n_d=params[\"n_d\"],\n",
        "        n_a=params[\"n_a\"],\n",
        "        n_steps=1,\n",
        "        n_independent=1,\n",
        "        n_shared=1,\n",
        "        gamma=1.3,\n",
        "        lambda_sparse=0,\n",
        "        optimizer_fn=optim.Adam,\n",
        "        optimizer_params=dict(lr=2e-2, weight_decay=1e-5),\n",
        "        mask_type=\"entmax\",\n",
        "        scheduler_params=dict(mode=\"min\", patience=5, min_lr=1e-5, threshold=1e-5, factor=0.1),\n",
        "        scheduler_fn=torch_ReduceLROnPlateau,\n",
        "        seed=seed,\n",
        "        verbose=0,\n",
        "    )\n",
        "\n",
        "    model = TabNetRegressor(**tabnet_params)\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P5y4WDfuaoXb",
        "papermill": {
          "duration": 0.087896,
          "end_time": "2020-11-30T00:04:58.848533",
          "exception": false,
          "start_time": "2020-11-30T00:04:58.760637",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## NODE\n",
        "\n",
        "Neural Oblivious Decision Ensembles\n",
        "\n",
        "[MoA Neural Oblivious Decision Ensembles (TF Keras)](https://www.kaggle.com/gogo827jz/moa-neural-oblivious-decision-ensembles-tf-keras)\n",
        "\n",
        "[NODE for Tensorflow](https://www.kaggle.com/marcusgawronsky/differentiable-catboost-node-in-tensorflow-2-0)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:59.023923Z",
          "iopub.status.busy": "2020-11-30T00:04:59.023027Z",
          "iopub.status.idle": "2020-11-30T00:04:59.026258Z",
          "shell.execute_reply": "2020-11-30T00:04:59.025732Z"
        },
        "id": "k-f2F6ikatMR",
        "papermill": {
          "duration": 0.093416,
          "end_time": "2020-11-30T00:04:59.026365",
          "exception": false,
          "start_time": "2020-11-30T00:04:58.932949",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "@tf.function\n",
        "def sparsemoid(inputs: tf.Tensor):\n",
        "    return tf.clip_by_value(0.5 * inputs + 0.5, 0.0, 1.0)\n",
        "\n",
        "@tf.function\n",
        "def identity(x: tf.Tensor):\n",
        "    return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:59.247261Z",
          "iopub.status.busy": "2020-11-30T00:04:59.223588Z",
          "iopub.status.idle": "2020-11-30T00:04:59.250459Z",
          "shell.execute_reply": "2020-11-30T00:04:59.249805Z"
        },
        "id": "gbbSGarVazGa",
        "papermill": {
          "duration": 0.12698,
          "end_time": "2020-11-30T00:04:59.250574",
          "exception": false,
          "start_time": "2020-11-30T00:04:59.123594",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "class ODST(L.Layer):\n",
        "    def __init__(self, n_trees: int = 3, depth: int = 4, units: int = 1, threshold_init_beta: float = 1.0):\n",
        "        super(ODST, self).__init__()\n",
        "        self.initialized = False\n",
        "        self.n_trees = n_trees\n",
        "        self.depth = depth\n",
        "        self.units = units\n",
        "        self.threshold_init_beta = threshold_init_beta\n",
        "\n",
        "    def build(self, input_shape: tf.TensorShape):\n",
        "        feature_selection_logits_init = tf.zeros_initializer()\n",
        "        self.feature_selection_logits = tf.Variable(\n",
        "            initial_value=feature_selection_logits_init(\n",
        "                shape=(input_shape[-1], self.n_trees, self.depth), dtype=\"float32\"\n",
        "            ),\n",
        "            trainable=True,\n",
        "            name=\"feature_selection_logits\",\n",
        "        )\n",
        "\n",
        "        feature_thresholds_init = tf.zeros_initializer()\n",
        "        self.feature_thresholds = tf.Variable(\n",
        "            initial_value=feature_thresholds_init(shape=(self.n_trees, self.depth), dtype=\"float32\"),\n",
        "            trainable=True,\n",
        "            name=\"feature_thresholds\",\n",
        "        )\n",
        "\n",
        "        log_temperatures_init = tf.ones_initializer()\n",
        "        self.log_temperatures = tf.Variable(\n",
        "            initial_value=log_temperatures_init(shape=(self.n_trees, self.depth), dtype=\"float32\"),\n",
        "            trainable=True,\n",
        "            name=\"log_temperatures\",\n",
        "        )\n",
        "\n",
        "        indices = K.arange(0, 2 ** self.depth, 1)\n",
        "        offsets = 2 ** K.arange(0, self.depth, 1)\n",
        "        bin_codes = tf.reshape(indices, (1, -1)) // tf.reshape(offsets, (-1, 1)) % 2\n",
        "        bin_codes_1hot = tf.stack([bin_codes, 1 - bin_codes], axis=-1)\n",
        "        self.bin_codes_1hot = tf.Variable(\n",
        "            initial_value=tf.cast(bin_codes_1hot, \"float32\"), trainable=False, name=\"bin_codes_1hot\"\n",
        "        )\n",
        "\n",
        "        response_init = tf.ones_initializer()\n",
        "        self.response = tf.Variable(\n",
        "            initial_value=response_init(shape=(self.n_trees, self.units, 2 ** self.depth), dtype=\"float32\"),\n",
        "            trainable=True,\n",
        "            name=\"response\",\n",
        "        )\n",
        "\n",
        "    def initialize(self, inputs):\n",
        "        feature_values = self.feature_values(inputs)\n",
        "\n",
        "        # intialize feature_thresholds\n",
        "        percentiles_q = 100 * tfp_distributions.Beta(self.threshold_init_beta, self.threshold_init_beta).sample(\n",
        "            [self.n_trees * self.depth]\n",
        "        )\n",
        "        flattened_feature_values = tf.map_fn(K.flatten, feature_values)\n",
        "        init_feature_thresholds = tf.linalg.diag_part(\n",
        "            tfp_stats.percentile(flattened_feature_values, percentiles_q, axis=0)\n",
        "        )\n",
        "\n",
        "        self.feature_thresholds.assign(tf.reshape(init_feature_thresholds, self.feature_thresholds.shape))\n",
        "\n",
        "        # intialize log_temperatures\n",
        "        self.log_temperatures.assign(\n",
        "            tfp_stats.percentile(tf.math.abs(feature_values - self.feature_thresholds), 50, axis=0)\n",
        "        )\n",
        "\n",
        "    def feature_values(self, inputs: tf.Tensor, training: bool = None):\n",
        "        feature_selectors = tfa.activations.sparsemax(self.feature_selection_logits)\n",
        "        # ^--[in_features, n_trees, depth]\n",
        "\n",
        "        feature_values = tf.einsum(\"bi,ind->bnd\", inputs, feature_selectors)\n",
        "        # ^--[batch_size, n_trees, depth]\n",
        "\n",
        "        return feature_values\n",
        "\n",
        "    def call(self, inputs: tf.Tensor, training: bool = None):\n",
        "        if not self.initialized:\n",
        "            self.initialize(inputs)\n",
        "            self.initialized = True\n",
        "\n",
        "        feature_values = self.feature_values(inputs)\n",
        "\n",
        "        threshold_logits_a = (feature_values - self.feature_thresholds) * tf.math.exp(-self.log_temperatures)\n",
        "\n",
        "        threshold_logits_b = tf.stack([-threshold_logits_a, threshold_logits_a], axis=-1)\n",
        "        # ^--[batch_size, n_trees, depth, 2]\n",
        "\n",
        "        bins = sparsemoid(threshold_logits_b)\n",
        "        # ^--[batch_size, n_trees, depth, 2], approximately binary\n",
        "\n",
        "        bin_matches = tf.einsum(\"btds,dcs->btdc\", bins, self.bin_codes_1hot)\n",
        "        # ^--[batch_size, n_trees, depth, 2 ** depth]\n",
        "\n",
        "        response_weights = tf.math.reduce_prod(bin_matches, axis=-2)\n",
        "        # ^-- [batch_size, n_trees, 2 ** depth]\n",
        "\n",
        "        response = tf.einsum(\"bnd,ncd->bnc\", response_weights, self.response)\n",
        "        # ^-- [batch_size, n_trees, units]\n",
        "\n",
        "        return tf.reduce_sum(response, axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:59.441347Z",
          "iopub.status.busy": "2020-11-30T00:04:59.434004Z",
          "iopub.status.idle": "2020-11-30T00:04:59.444341Z",
          "shell.execute_reply": "2020-11-30T00:04:59.443817Z"
        },
        "id": "WYGeBIoAa0ha",
        "papermill": {
          "duration": 0.109306,
          "end_time": "2020-11-30T00:04:59.444487",
          "exception": false,
          "start_time": "2020-11-30T00:04:59.335181",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "class NODE(tf.keras.Model):\n",
        "    def __init__(\n",
        "        self,\n",
        "        units: int = 1,\n",
        "        n_layers: int = 1,\n",
        "        output_dim=1,\n",
        "        dropout_rate=0.1,\n",
        "        link: tf.function = tf.identity,\n",
        "        n_trees: int = 3,\n",
        "        depth: int = 4,\n",
        "        threshold_init_beta: float = 1.0,\n",
        "        feature_column: Optional[L.DenseFeatures] = None,\n",
        "    ):\n",
        "        super(NODE, self).__init__()\n",
        "        self.units = units\n",
        "        self.n_layers = n_layers\n",
        "        self.n_trees = n_trees\n",
        "        self.depth = depth\n",
        "        self.units = units\n",
        "        self.threshold_init_beta = threshold_init_beta\n",
        "        self.feature_column = feature_column\n",
        "        self.dropout_rate = dropout_rate\n",
        "        self.output_dim = output_dim\n",
        "\n",
        "        if feature_column is None:\n",
        "            self.feature = L.Lambda(identity)\n",
        "        else:\n",
        "            self.feature = feature_column\n",
        "\n",
        "        self.bn = [L.BatchNormalization() for _ in range(n_layers + 1)]\n",
        "        self.dropout = [L.Dropout(self.dropout_rate) for _ in range(n_layers + 1)]\n",
        "        self.ensemble = [\n",
        "            ODST(n_trees=n_trees, depth=depth, units=units, threshold_init_beta=threshold_init_beta)\n",
        "            for _ in range(n_layers)\n",
        "        ]\n",
        "\n",
        "        self.last_layer = L.Dense(self.output_dim)\n",
        "\n",
        "        self.link = link\n",
        "\n",
        "    def call(self, inputs, training=None):\n",
        "        X_a = self.feature(inputs)\n",
        "        X_b = self.bn[0](X_a, training=training)\n",
        "        X_c = self.dropout[0](X_b, training=training)\n",
        "\n",
        "        X = defaultdict(dict)\n",
        "        X[0][0] = X_c\n",
        "        for i, tree in enumerate(self.ensemble):\n",
        "            X[i][1] = tf.concat([X[i][0], tree(X[i][0])], axis=1)\n",
        "            X[i][2] = self.bn[i + 1](X[i][1], training=training)\n",
        "            X[i + 1][0] = self.dropout[i + 1](X[i][2], training=training)\n",
        "\n",
        "        return self.link(self.last_layer(X[i + 1][0]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:59.625992Z",
          "iopub.status.busy": "2020-11-30T00:04:59.622973Z",
          "iopub.status.idle": "2020-11-30T00:04:59.626896Z",
          "shell.execute_reply": "2020-11-30T00:04:59.627471Z"
        },
        "id": "LwAET2WYa4b9",
        "papermill": {
          "duration": 0.097527,
          "end_time": "2020-11-30T00:04:59.627605",
          "exception": false,
          "start_time": "2020-11-30T00:04:59.530078",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def create_model_node(input_dim, output_dim):\n",
        "    model = tf.keras.Sequential(\n",
        "        [\n",
        "            L.Input(shape=(input_dim,)),\n",
        "            L.BatchNormalization(),\n",
        "            NODE(\n",
        "                n_layers=2,\n",
        "                units=256,\n",
        "                output_dim=1024,\n",
        "                dropout_rate=0.2,\n",
        "                depth=2,\n",
        "                n_trees=3,\n",
        "            ),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(512, activation=\"elu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(256, activation=\"swish\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dense(output_dim),  # from_logits=True\n",
        "        ]\n",
        "    )\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:59.802608Z",
          "iopub.status.busy": "2020-11-30T00:04:59.801608Z",
          "iopub.status.idle": "2020-11-30T00:04:59.805017Z",
          "shell.execute_reply": "2020-11-30T00:04:59.804459Z"
        },
        "id": "dQ3Gbm_IjFcS",
        "papermill": {
          "duration": 0.092454,
          "end_time": "2020-11-30T00:04:59.805115",
          "exception": false,
          "start_time": "2020-11-30T00:04:59.712661",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if IN_COLAB:\n",
        "    model_test = create_model_node(len(train.columns), len(target.columns))\n",
        "    model_test.summary()\n",
        "    # display_svg(SVG(model_to_dot(model_test, show_shapes=True, dpi=72).create(prog=\"dot\", format=\"svg\")))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:04:59.984263Z",
          "iopub.status.busy": "2020-11-30T00:04:59.983505Z",
          "iopub.status.idle": "2020-11-30T00:04:59.986190Z",
          "shell.execute_reply": "2020-11-30T00:04:59.986847Z"
        },
        "id": "yx6NoLrktUVg",
        "papermill": {
          "duration": 0.097785,
          "end_time": "2020-11-30T00:04:59.986973",
          "exception": false,
          "start_time": "2020-11-30T00:04:59.889188",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def create_model_node_tuning(input_dim, output_dim, params=None):\n",
        "    model = tf.keras.Sequential(\n",
        "        [\n",
        "            L.Input(shape=(input_dim,)),\n",
        "            L.BatchNormalization(),\n",
        "            NODE(\n",
        "                n_layers=params[\"n_layers\"],\n",
        "                units=params[\"units\"],\n",
        "                output_dim=params[\"output_dim\"],\n",
        "                dropout_rate=0.2,\n",
        "                depth=params[\"depth\"],\n",
        "                n_trees=params[\"n_trees\"],\n",
        "            ),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(params[\"d_1\"], activation=\"elu\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dropout(0.2),\n",
        "            tfa.layers.WeightNormalization(L.Dense(params[\"d_2\"], activation=\"swish\")),\n",
        "            L.BatchNormalization(),\n",
        "            L.Dense(output_dim),  # from_logits=True\n",
        "        ]\n",
        "    )\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zzOOGJtq7OUL",
        "papermill": {
          "duration": 0.083746,
          "end_time": "2020-11-30T00:05:00.155176",
          "exception": false,
          "start_time": "2020-11-30T00:05:00.071430",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:00.331697Z",
          "iopub.status.busy": "2020-11-30T00:05:00.330645Z",
          "iopub.status.idle": "2020-11-30T00:05:00.411774Z",
          "shell.execute_reply": "2020-11-30T00:05:00.411216Z"
        },
        "id": "QUyOmuMg7OUU",
        "papermill": {
          "duration": 0.170506,
          "end_time": "2020-11-30T00:05:00.411891",
          "exception": false,
          "start_time": "2020-11-30T00:05:00.241385",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def learning(\n",
        "    train_,\n",
        "    train_pca_,\n",
        "    target_,\n",
        "    drug_,\n",
        "    test_,\n",
        "    test_pca_,\n",
        "    N_STARTS=6,\n",
        "    N_SPLITS=5,\n",
        "    train_flags=[\"normal\"],\n",
        "    transfer_learning_base=None,\n",
        "    params=None,\n",
        "):\n",
        "    oof = {}\n",
        "    predictions = {}\n",
        "\n",
        "    for seed in range(N_STARTS):\n",
        "        model_name = models[seed % len(models)][\"model_name\"]\n",
        "        cv = models[seed % len(models)][\"cv\"]\n",
        "        fit = models[seed % len(models)][\"fit\"]\n",
        "\n",
        "        if (\"pre_train\" in train_flags and model_name not in pre_train_models):\n",
        "            continue\n",
        "            \n",
        "        seed_result = pd.DataFrame(np.zeros(target_.shape))\n",
        "        prediction = pd.DataFrame(np.zeros(ss.shape))\n",
        "\n",
        "        kfold_seed = random_seed + seed\n",
        "        if \"pre_train\" in train_flags:\n",
        "            kfold_seed += random_seed\n",
        "\n",
        "        fix_seed(kfold_seed)\n",
        "\n",
        "        if \"fold\" in drug_.columns:\n",
        "            drug_.drop([\"fold\"], axis=1, inplace=True)\n",
        "\n",
        "        # LOCATE DRUGS\n",
        "        vc = drug_.drug_id.value_counts()\n",
        "        if DRUG_KFOLD == \"soft\":\n",
        "            vc1 = vc.loc[(vc == 6) | (vc == 12) | (vc == 18)].index.sort_values()\n",
        "            vc2 = vc.loc[(vc != 6) & (vc != 12) & (vc != 18)].index.sort_values()\n",
        "        else:\n",
        "            vc1 = vc.loc[vc <= 19].index.sort_values()\n",
        "            vc2 = vc.loc[vc > 19].index.sort_values()\n",
        "        \n",
        "        dct1 = {}\n",
        "        dct2 = {}\n",
        "\n",
        "        # STRATIFY DRUGS 18X OR LESS\n",
        "        skf = MultilabelStratifiedKFold(n_splits=N_SPLITS, random_state=kfold_seed, shuffle=True)\n",
        "        tmp = pd.concat([drug_, target_], axis=1).groupby(\"drug_id\").mean().loc[vc1]\n",
        "        for fold, (idxT, idxV) in enumerate(skf.split(tmp, tmp)):\n",
        "            dd = {k: fold for k in tmp.index[idxV].values}\n",
        "            dct1.update(dd)\n",
        "\n",
        "        # STRATIFY DRUGS MORE THAN 18X\n",
        "        skf = MultilabelStratifiedKFold(n_splits=N_SPLITS, random_state=kfold_seed, shuffle=True)\n",
        "        tmp = drug_.loc[drug_.drug_id.isin(vc2)].reset_index(drop=True)\n",
        "        for fold, (idxT, idxV) in enumerate(skf.split(tmp, tmp)):\n",
        "            dd = {k: fold for k in tmp.sig_id[idxV].values}\n",
        "            dct2.update(dd)\n",
        "\n",
        "        # ASSIGN FOLDS\n",
        "        drug_[\"fold\"] = drug_.drug_id.map(dct1)\n",
        "        drug_.loc[drug_.fold.isna(), \"fold\"] = drug_.loc[drug_.fold.isna(), \"sig_id\"].map(dct2)\n",
        "        drug_.fold = drug_.fold.astype(\"int8\")\n",
        "\n",
        "        for n, (tr, te) in enumerate(\n",
        "            MultilabelStratifiedKFold(n_splits=N_SPLITS, random_state=kfold_seed, shuffle=True).split(target_, target_)\n",
        "        ):\n",
        "            if cv == \"with_drug_id\":\n",
        "                tr = drug_[drug_[\"fold\"] != n].index\n",
        "                te = drug_[drug_[\"fold\"] == n].index\n",
        "\n",
        "            start_time = time()\n",
        "\n",
        "            # Build Data Sets\n",
        "            if model_name == \"ResNet\":\n",
        "                x_tr = [\n",
        "                    add_swap_noise(tr, train_.values[tr], train_.values[tr]),\n",
        "                    # train_.values[tr],\n",
        "                    train_pca_.values[tr],\n",
        "                ]\n",
        "                x_val = [\n",
        "                    train_.values[te],\n",
        "                    train_pca_.values[te],\n",
        "                ]\n",
        "                y_tr, y_val = target_.astype(float).values[tr], target_.astype(float).values[te]\n",
        "                x_tt = [test_.values, test_pca_.values]\n",
        "\n",
        "            else:\n",
        "                x_tr, x_val = add_swap_noise(tr, train_.values[tr], train_.values[tr]), train_.values[te]\n",
        "                # x_tr, x_val = train_.values[tr], train_.values[te]\n",
        "                y_tr, y_val = target_.astype(float).values[tr], target_.astype(float).values[te]\n",
        "                x_tt = test_.values\n",
        "\n",
        "            # Build Model\n",
        "            if model_name == \"ResNet\":\n",
        "                if \"hyperparameter_tuning\" in train_flags:\n",
        "                    model = create_model_resnet_tuning(\n",
        "                        len(train_.columns), len(train_pca_.columns), len(target_.columns), params[model_name]\n",
        "                    )\n",
        "                    \n",
        "                    if \"fine_tuning\" in train_flags:\n",
        "                        model_base = create_model_resnet_tuning(\n",
        "                            len(train_.columns),\n",
        "                            len(train_pca_.columns),\n",
        "                            len(transfer_learning_base.columns),\n",
        "                            params[model_name],\n",
        "                        )\n",
        "                    \n",
        "                else:\n",
        "                    model = create_model_resnet(len(train_.columns), len(train_pca_.columns), len(target_.columns))\n",
        "\n",
        "                    if \"fine_tuning\" in train_flags:\n",
        "                        if not fit:\n",
        "                            model_base = create_model_resnet(\n",
        "                                len(train_.columns), len(train_pca_.columns), len(target_.columns)\n",
        "                            )\n",
        "                        else:\n",
        "                            model_base = create_model_resnet(\n",
        "                                len(train_.columns), len(train_pca_.columns), len(transfer_learning_base.columns)\n",
        "                            )\n",
        "\n",
        "            elif model_name == \"TabNet\":\n",
        "                if \"hyperparameter_tuning\" in train_flags:\n",
        "                    model = create_model_tabnet_tuning(kfold_seed, params[model_name])\n",
        "                    \n",
        "                elif \"pre_train\" in train_flags:\n",
        "                    #model = create_model_tabnet(kfold_seed, pre_train=True)\n",
        "                    model = create_model_tabnet(kfold_seed, pre_train=False)\n",
        "\n",
        "                else:\n",
        "                    model = create_model_tabnet(kfold_seed)\n",
        "                    \n",
        "                    if \"fine_tuning\" in train_flags:\n",
        "                        model_base = create_model_tabnet(kfold_seed, pre_train=True)\n",
        "                        \n",
        "            elif model_name == \"NODE\":\n",
        "                if \"hyperparameter_tuning\" in train_flags:\n",
        "                    model = create_model_node_tuning(len(train_.columns), len(target_.columns), params[model_name])\n",
        "                    \n",
        "                    if \"fine_tuning\" in train_flags:\n",
        "                        model_base = create_model_node_tuning(\n",
        "                            len(train_.columns), len(transfer_learning_base.columns), params[model_name]\n",
        "                        )\n",
        "\n",
        "                else:\n",
        "                    model = create_model_node(len(train_.columns), len(target_.columns))\n",
        "                    \n",
        "                    if \"fine_tuning\" in train_flags:\n",
        "                        model_base = create_model_node(len(train_.columns), len(transfer_learning_base.columns))\n",
        "                        \n",
        "            else:\n",
        "                raise \"Model name is invalid.\"\n",
        "\n",
        "            if model_name == \"TabNet\":\n",
        "                checkpoint_path = f\"{model_name}_repeat:{seed // len(models)}_fold:{n}\"\n",
        "                \n",
        "                if fit:\n",
        "                    MODEL_DIR = PRE_TRAIN_MODEL_DIR_TABNET\n",
        "                else:\n",
        "                    MODEL_DIR = PRE_TRAIN_MODEL_DIR_NO_FIT\n",
        "\n",
        "                if PRE_TRAIN_MODEL == \"load-others\":\n",
        "                    checkpoint_path = os.path.join(\n",
        "                        MODEL_DIR,\n",
        "                        checkpoint_path\n",
        "                    )\n",
        "                    \n",
        "                if \"fine_tuning\" in train_flags and os.path.exists(checkpoint_path) and fit:\n",
        "                    model_base.load_model(checkpoint_path + \".zip\")\n",
        "\n",
        "                    model.fit(\n",
        "                        X_train=x_tr,\n",
        "                        y_train=y_tr,\n",
        "                        eval_set=[(x_val, y_val)],\n",
        "                        eval_name=[\"val\"],\n",
        "                        eval_metric=[\"logits_ll\"],\n",
        "                        max_epochs=200,\n",
        "                        patience=10,\n",
        "                        batch_size=1024,\n",
        "                        virtual_batch_size=32,\n",
        "                        num_workers=1,\n",
        "                        drop_last=True,\n",
        "                        # loss_fn=F.binary_cross_entropy_with_logits,\n",
        "                        loss_fn=SmoothBCEwLogits(smoothing=1e-6),\n",
        "                        from_unsupervised=model_base,\n",
        "                    )\n",
        "                    \n",
        "                elif fit:\n",
        "                    model.fit(\n",
        "                        X_train=x_tr,\n",
        "                        y_train=y_tr,\n",
        "                        eval_set=[(x_val, y_val)],\n",
        "                        eval_name=[\"val\"],\n",
        "                        eval_metric=[\"logits_ll\"],\n",
        "                        max_epochs=200,\n",
        "                        patience=10,\n",
        "                        batch_size=1024,\n",
        "                        virtual_batch_size=32,\n",
        "                        num_workers=1,\n",
        "                        drop_last=True,\n",
        "                        # loss_fn=F.binary_cross_entropy_with_logits,\n",
        "                        loss_fn=SmoothBCEwLogits(smoothing=1e-6),\n",
        "                    )\n",
        "\n",
        "                else:\n",
        "                    model.load_model(checkpoint_path + \".zip\")\n",
        "                    \n",
        "                if \"pre_train\" in train_flags or SAVE_WEIGHT:\n",
        "                    try:\n",
        "                        os.remove(os.path.basename(checkpoint_path))\n",
        "                    except OSError:\n",
        "                        pass\n",
        "                    model.save_model(os.path.basename(checkpoint_path))\n",
        "\n",
        "            else:\n",
        "                model.compile(\n",
        "                    optimizer=tfa.optimizers.AdamW(lr=1e-3, weight_decay=1e-5),\n",
        "                    loss=tf.keras.losses.BinaryCrossentropy(from_logits=True, label_smoothing=1e-6),\n",
        "                    metrics=logloss,\n",
        "                )\n",
        "\n",
        "                checkpoint_path = f\"{model_name}_repeat:{seed // len(models)}_fold:{n}.hdf5\"\n",
        "\n",
        "                if fit:\n",
        "                    if model_name == \"ResNet\":\n",
        "                        MODEL_DIR = PRE_TRAIN_MODEL_DIR_RESNET\n",
        "                    else:\n",
        "                        MODEL_DIR = PRE_TRAIN_MODEL_DIR_NODE\n",
        "                else:\n",
        "                    MODEL_DIR = PRE_TRAIN_MODEL_DIR_NO_FIT\n",
        "\n",
        "                if PRE_TRAIN_MODEL == \"load-others\":\n",
        "                    checkpoint_path = os.path.join(\n",
        "                        MODEL_DIR,\n",
        "                        checkpoint_path\n",
        "                    )\n",
        "\n",
        "                if \"fine_tuning\" in train_flags and os.path.exists(checkpoint_path):\n",
        "                    model_base.load_weights(checkpoint_path)\n",
        "                    for layer in range(len(model_base.layers[:-1])):\n",
        "                        model.layers[layer].set_weights(model_base.layers[layer].get_weights())\n",
        "                    if not fit:\n",
        "                        model.layers[-1].set_weights(model_base.layers[-1].get_weights())\n",
        "\n",
        "                if \"pre_train\" in train_flags or SAVE_WEIGHT:\n",
        "                    cb_checkpt = ModelCheckpoint(\n",
        "                        os.path.basename(checkpoint_path),\n",
        "                        monitor=\"val_loss\",\n",
        "                        verbose=0,\n",
        "                        save_best_only=True,\n",
        "                        save_weights_only=True,\n",
        "                        mode=\"min\",\n",
        "                    )\n",
        "                reduce_lr_loss = ReduceLROnPlateau(\n",
        "                    monitor=\"val_loss\", factor=0.1, patience=5, verbose=0, min_delta=1e-5, min_lr=1e-5, mode=\"min\"\n",
        "                )\n",
        "                early_stopping = EarlyStopping(\n",
        "                    monitor=\"val_loss\",\n",
        "                    patience=10,\n",
        "                    mode=\"min\",\n",
        "                    verbose=0,\n",
        "                    min_delta=1e-5,\n",
        "                    restore_best_weights=True,\n",
        "                )\n",
        "                if \"pre_train\" in train_flags or SAVE_WEIGHT:\n",
        "                    callbacks = [cb_checkpt, reduce_lr_loss, early_stopping]\n",
        "                else:\n",
        "                    callbacks = [reduce_lr_loss, early_stopping]\n",
        "\n",
        "                if fit:\n",
        "                    model.fit(\n",
        "                        x_tr,\n",
        "                        y_tr,\n",
        "                        validation_data=(x_val, y_val),\n",
        "                        epochs=200,\n",
        "                        batch_size=128,\n",
        "                        callbacks=callbacks,\n",
        "                        verbose=0,\n",
        "                    )\n",
        "\n",
        "            val_predict = model.predict(x_val)\n",
        "            val_predict = 1 / (1 + np.exp(-val_predict))\n",
        "            seed_result.loc[te, :] += val_predict\n",
        "            fold_score = metric(target_.loc[te].values, val_predict)\n",
        "\n",
        "            if any(flag in train_flags for flag in [\"normal\", \"fine_tuning\"]):\n",
        "                test_predict = model.predict(x_tt)\n",
        "                test_predict = 1 / (1 + np.exp(-test_predict))\n",
        "                prediction += test_predict / N_SPLITS\n",
        "\n",
        "            print(\n",
        "                f\"[{str(datetime.timedelta(seconds = time() - start_time))[2:7]}] {model_name} {cv}: Seed {seed}, Fold {n}:\",\n",
        "                fold_score,\n",
        "            )\n",
        "\n",
        "            K.clear_session()\n",
        "            del model\n",
        "            if \"model_base\" in globals():\n",
        "                del model_base\n",
        "            gc.collect()\n",
        "\n",
        "        oof[f\"{model_name}_{cv}_fit({str(fit)})_{seed}\"] = seed_result\n",
        "        predictions[f\"{model_name}_{cv}_fit({str(fit)})_{seed}\"] = prediction\n",
        "\n",
        "    return oof, predictions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xa-XZKZr0OcL",
        "papermill": {
          "duration": 0.085041,
          "end_time": "2020-11-30T00:05:00.583298",
          "exception": false,
          "start_time": "2020-11-30T00:05:00.498257",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Hyper parameter tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:00.782704Z",
          "iopub.status.busy": "2020-11-30T00:05:00.780757Z",
          "iopub.status.idle": "2020-11-30T00:05:00.783396Z",
          "shell.execute_reply": "2020-11-30T00:05:00.783921Z"
        },
        "id": "jDdNTfxi0Q2Y",
        "papermill": {
          "duration": 0.113524,
          "end_time": "2020-11-30T00:05:00.784052",
          "exception": false,
          "start_time": "2020-11-30T00:05:00.670528",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "def objective(trial):\n",
        "    params = {model[\"model_name\"]: None for model in models}\n",
        "\n",
        "    if TUNING_RESNET:\n",
        "        n_layers = trial.suggest_int(\"n_layers\", 2, 5)\n",
        "        # n_layers = 5\n",
        "\n",
        "        units = []\n",
        "        for i in range(n_layers + 3):\n",
        "            u = trial.suggest_categorical(f\"units_{i}\", [128, 256, 512, 1024])\n",
        "            units.append(u)\n",
        "\n",
        "        activations = []\n",
        "        for i in range(n_layers + 4):\n",
        "            a = trial.suggest_categorical(f\"activations_{i}\", [\"relu\", \"elu\", \"selu\", \"swish\"])\n",
        "            activations.append(a)\n",
        "\n",
        "        params[\"ResNet\"] = {\n",
        "            \"n_layers\": n_layers,\n",
        "            \"units\": units,\n",
        "            \"activations\": activations,\n",
        "        }\n",
        "\n",
        "    if TUNING_TABNET:\n",
        "        n_d = trial.suggest_categorical(\"n_d\", [16, 24, 32, 48, 64, 96, 128, 160])\n",
        "        n_a = trial.suggest_categorical(\"n_a\", [16, 24, 32, 48, 64, 96, 128, 160])\n",
        "\n",
        "        params[\"TabNet\"] = {\n",
        "            \"n_d\": n_d,\n",
        "            \"n_a\": n_a,\n",
        "        }\n",
        "\n",
        "    if TUNING_NODE:\n",
        "        n_layers = 2  # trial.suggest_int(\"n_layers\", 2, 3)\n",
        "        depth = 2  # trial.suggest_int(\"depth\", 2, 3)\n",
        "        n_trees = 3  # trial.suggest_int(\"n_trees\", 2, 3)\n",
        "        units = trial.suggest_categorical(\"units\", [128, 256, 512, 1024])\n",
        "        output_dim = trial.suggest_categorical(\"output_dim\", [128, 256, 512, 1024])\n",
        "        d_1 = trial.suggest_categorical(\"d_1\", [128, 256, 512, 1024])\n",
        "        d_2 = trial.suggest_categorical(\"d_2\", [128, 256, 512, 1024])\n",
        "\n",
        "        params[\"NODE\"] = {\n",
        "            \"n_layers\": n_layers,\n",
        "            \"depth\": depth,\n",
        "            \"n_trees\": n_trees,\n",
        "            \"units\": units,\n",
        "            \"output_dim\": output_dim,\n",
        "            \"d_1\": d_1,\n",
        "            \"d_2\": d_2,\n",
        "        }\n",
        "\n",
        "    # Training\n",
        "    if PRE_TRAIN_MODEL == \"in-notebook\":\n",
        "        _, _ = learning(\n",
        "            train[: non_target.shape[0]],\n",
        "            train_pca[: non_target.shape[0]],\n",
        "            pd.concat([non_target, target[: non_target.shape[0]]], axis=1),\n",
        "            non_target_drug,\n",
        "            test,\n",
        "            test_pca,\n",
        "            N_STARTS,\n",
        "            N_SPLITS,\n",
        "            train_flags=[\"pre_train\", \"hyperparameter_tuning\"],\n",
        "            params=params,\n",
        "        )\n",
        "\n",
        "    oof, predictions = learning(\n",
        "        train,\n",
        "        train_pca,\n",
        "        target,\n",
        "        target_drug,\n",
        "        test,\n",
        "        test_pca,\n",
        "        N_STARTS,\n",
        "        N_SPLITS,\n",
        "        train_flags=[\"fine_tuning\", \"hyperparameter_tuning\"],  # \"normal\", \"fine_tuning\"\n",
        "        transfer_learning_base=pd.concat([non_target, target[: non_target.shape[0]]], axis=1),\n",
        "        params=params,\n",
        "    )\n",
        "\n",
        "    initial_weights = [1.0 / N_STARTS for _ in range(N_STARTS)]\n",
        "    y_true = target.values[: non_target.shape[0]]\n",
        "\n",
        "    cv, auc, _ = cross_validation(y_true.shape, initial_weights, y_true, oof)\n",
        "\n",
        "    return cv * 1000 / auc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:00.966543Z",
          "iopub.status.busy": "2020-11-30T00:05:00.965712Z",
          "iopub.status.idle": "2020-11-30T00:05:00.970992Z",
          "shell.execute_reply": "2020-11-30T00:05:00.970047Z"
        },
        "id": "Uzqitb2E07g9",
        "papermill": {
          "duration": 0.097945,
          "end_time": "2020-11-30T00:05:00.971101",
          "exception": false,
          "start_time": "2020-11-30T00:05:00.873156",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if HYPER_PARAMETER_TUNING:\n",
        "    study = optuna.create_study(direction='minimize')\n",
        "    study.optimize(objective, n_trials=20, gc_after_trial=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:01.155044Z",
          "iopub.status.busy": "2020-11-30T00:05:01.154220Z",
          "iopub.status.idle": "2020-11-30T00:05:01.157501Z",
          "shell.execute_reply": "2020-11-30T00:05:01.156940Z"
        },
        "id": "opMxHb9N6o7Y",
        "papermill": {
          "duration": 0.101242,
          "end_time": "2020-11-30T00:05:01.157607",
          "exception": false,
          "start_time": "2020-11-30T00:05:01.056365",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if HYPER_PARAMETER_TUNING:\n",
        "    print(\"Best trial:\")\n",
        "    trial = study.best_trial\n",
        "\n",
        "    print(\"  CV:  {}\".format(trial.value))\n",
        "    print(\"  Params: \")\n",
        "    for key, value in trial.params.items():\n",
        "        print(\"    {}: {}\".format(key, value))\n",
        "\n",
        "    print(optuna.importance.get_param_importances(study))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:01.339902Z",
          "iopub.status.busy": "2020-11-30T00:05:01.338918Z",
          "iopub.status.idle": "2020-11-30T00:05:01.342095Z",
          "shell.execute_reply": "2020-11-30T00:05:01.341511Z"
        },
        "id": "tsSV0mNk6xHe",
        "papermill": {
          "duration": 0.096332,
          "end_time": "2020-11-30T00:05:01.342201",
          "exception": false,
          "start_time": "2020-11-30T00:05:01.245869",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if HYPER_PARAMETER_TUNING:\n",
        "    raise \"Finished parameter tuning.\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oMBw29HH0TGT",
        "papermill": {
          "duration": 0.084272,
          "end_time": "2020-11-30T00:05:01.515296",
          "exception": false,
          "start_time": "2020-11-30T00:05:01.431024",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Normal training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:01.692578Z",
          "iopub.status.busy": "2020-11-30T00:05:01.691776Z",
          "iopub.status.idle": "2020-11-30T00:05:01.695915Z",
          "shell.execute_reply": "2020-11-30T00:05:01.696423Z"
        },
        "id": "QEPrZIyP7OUV",
        "papermill": {
          "duration": 0.097078,
          "end_time": "2020-11-30T00:05:01.696566",
          "exception": false,
          "start_time": "2020-11-30T00:05:01.599488",
          "status": "completed"
        },
        "tags": [],
        "outputId": "95ee426a-0f14-494b-b6d5-4940584113a7"
      },
      "source": [
        "%%time\n",
        "\n",
        "if PRE_TRAIN_MODEL == \"in-notebook\":\n",
        "    _, _ = learning(\n",
        "        train[: non_target.shape[0]],\n",
        "        train_pca[: non_target.shape[0]],\n",
        "        pd.concat([non_target, target[: non_target.shape[0]]], axis=1),\n",
        "        non_target_drug,\n",
        "        test,\n",
        "        test_pca,\n",
        "        N_STARTS,\n",
        "        N_SPLITS,\n",
        "        train_flags=[\"pre_train\"],\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3 µs, sys: 0 ns, total: 3 µs\n",
            "Wall time: 6.91 µs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T00:05:01.884341Z",
          "iopub.status.busy": "2020-11-30T00:05:01.883497Z",
          "iopub.status.idle": "2020-11-30T01:40:21.239207Z",
          "shell.execute_reply": "2020-11-30T01:40:21.239723Z"
        },
        "id": "PA4xk_Tn7OUX",
        "papermill": {
          "duration": 5719.45886,
          "end_time": "2020-11-30T01:40:21.239891",
          "exception": false,
          "start_time": "2020-11-30T00:05:01.781031",
          "status": "completed"
        },
        "tags": [],
        "outputId": "030f7aa2-17ad-416a-e8f7-ab204f1d0324"
      },
      "source": [
        "%%time\n",
        "\n",
        "oof, predictions = learning(\n",
        "    train,\n",
        "    train_pca,\n",
        "    target,\n",
        "    target_drug,\n",
        "    test,\n",
        "    test_pca,\n",
        "    N_STARTS,\n",
        "    N_SPLITS,\n",
        "    train_flags=[\"fine_tuning\"],  # \"normal\", \"fine_tuning\"\n",
        "    transfer_learning_base=pd.concat([non_target, target[: non_target.shape[0]]], axis=1),\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[02:05] ResNet without_drug_id: Seed 0, Fold 0: 0.015988617666400244\n",
            "[02:00] ResNet without_drug_id: Seed 0, Fold 1: 0.016561587230930812\n",
            "[02:02] ResNet without_drug_id: Seed 0, Fold 2: 0.016460160299281146\n",
            "[02:02] ResNet without_drug_id: Seed 0, Fold 3: 0.01748774524775867\n",
            "[02:07] ResNet without_drug_id: Seed 0, Fold 4: 0.017378904586047548\n",
            "[01:50] ResNet without_drug_id: Seed 0, Fold 5: 0.016428686192403865\n",
            "[01:58] ResNet without_drug_id: Seed 0, Fold 6: 0.016393450192781617\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 49 with best_epoch = 39 and best_val_logits_ll = 0.01771\n",
            "Best weights from best epoch are automatically used!\n",
            "[01:42] TabNet without_drug_id: Seed 1, Fold 0: 0.017711500060807878\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 57 with best_epoch = 47 and best_val_logits_ll = 0.01683\n",
            "Best weights from best epoch are automatically used!\n",
            "[01:57] TabNet without_drug_id: Seed 1, Fold 1: 0.016834965008491812\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 39 with best_epoch = 29 and best_val_logits_ll = 0.01732\n",
            "Best weights from best epoch are automatically used!\n",
            "[01:21] TabNet without_drug_id: Seed 1, Fold 2: 0.01731542871363045\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 55 with best_epoch = 45 and best_val_logits_ll = 0.01708\n",
            "Best weights from best epoch are automatically used!\n",
            "[01:53] TabNet without_drug_id: Seed 1, Fold 3: 0.017078980362128682\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 69 with best_epoch = 59 and best_val_logits_ll = 0.01739\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:28] TabNet without_drug_id: Seed 1, Fold 4: 0.017389852135415072\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 54 with best_epoch = 44 and best_val_logits_ll = 0.01692\n",
            "Best weights from best epoch are automatically used!\n",
            "[01:55] TabNet without_drug_id: Seed 1, Fold 5: 0.01691527451610066\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 60 with best_epoch = 50 and best_val_logits_ll = 0.01691\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:08] TabNet without_drug_id: Seed 1, Fold 6: 0.016909387983829325\n",
            "[04:17] NODE without_drug_id: Seed 2, Fold 0: 0.016013801600829273\n",
            "[01:39] NODE without_drug_id: Seed 2, Fold 1: 0.016315341232009957\n",
            "[03:51] NODE without_drug_id: Seed 2, Fold 2: 0.016322717391569347\n",
            "[01:53] NODE without_drug_id: Seed 2, Fold 3: 0.016419345370010083\n",
            "[01:30] NODE without_drug_id: Seed 2, Fold 4: 0.01657922343617685\n",
            "[02:04] NODE without_drug_id: Seed 2, Fold 5: 0.018624402648199676\n",
            "[03:25] NODE without_drug_id: Seed 2, Fold 6: 0.016244861721716713\n",
            "[02:00] ResNet without_drug_id: Seed 3, Fold 0: 0.016367053494955982\n",
            "[01:52] ResNet without_drug_id: Seed 3, Fold 1: 0.0163580665564915\n",
            "[01:47] ResNet without_drug_id: Seed 3, Fold 2: 0.017007073965624373\n",
            "[01:55] ResNet without_drug_id: Seed 3, Fold 3: 0.01674500532398561\n",
            "[02:10] ResNet without_drug_id: Seed 3, Fold 4: 0.01727699403293901\n",
            "[01:55] ResNet without_drug_id: Seed 3, Fold 5: 0.016469437772525672\n",
            "[01:47] ResNet without_drug_id: Seed 3, Fold 6: 0.01656587245745259\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 64 with best_epoch = 54 and best_val_logits_ll = 0.01672\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:29] TabNet without_drug_id: Seed 4, Fold 0: 0.016716205664725016\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 70 with best_epoch = 60 and best_val_logits_ll = 0.01753\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:41] TabNet without_drug_id: Seed 4, Fold 1: 0.017529822422024225\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 57 with best_epoch = 47 and best_val_logits_ll = 0.01703\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:11] TabNet without_drug_id: Seed 4, Fold 2: 0.01703226136425691\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 83 with best_epoch = 73 and best_val_logits_ll = 0.01682\n",
            "Best weights from best epoch are automatically used!\n",
            "[03:13] TabNet without_drug_id: Seed 4, Fold 3: 0.016822018519889348\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 38 with best_epoch = 28 and best_val_logits_ll = 0.01737\n",
            "Best weights from best epoch are automatically used!\n",
            "[01:30] TabNet without_drug_id: Seed 4, Fold 4: 0.017370473500793276\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 56 with best_epoch = 46 and best_val_logits_ll = 0.01693\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:10] TabNet without_drug_id: Seed 4, Fold 5: 0.016934218417482397\n",
            "Device used : cuda\n",
            "Device used : cuda\n",
            "\n",
            "Early stopping occured at epoch 69 with best_epoch = 59 and best_val_logits_ll = 0.01761\n",
            "Best weights from best epoch are automatically used!\n",
            "[02:37] TabNet without_drug_id: Seed 4, Fold 6: 0.017609686032389698\n",
            "[03:49] NODE without_drug_id: Seed 5, Fold 0: 0.016469544768378645\n",
            "[01:34] NODE without_drug_id: Seed 5, Fold 1: 0.01620307447585431\n",
            "[03:04] NODE without_drug_id: Seed 5, Fold 2: 0.016985238183321413\n",
            "[03:16] NODE without_drug_id: Seed 5, Fold 3: 0.01637563731880786\n",
            "[02:47] NODE without_drug_id: Seed 5, Fold 4: 0.016200138698445037\n",
            "[01:36] NODE without_drug_id: Seed 5, Fold 5: 0.016900782784768063\n",
            "[01:33] NODE without_drug_id: Seed 5, Fold 6: 0.01668365728234402\n",
            "CPU times: user 1h 33min 24s, sys: 10min 45s, total: 1h 44min 9s\n",
            "Wall time: 1h 35min 19s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iy2jGG1j7OUY",
        "papermill": {
          "duration": 0.111018,
          "end_time": "2020-11-30T01:40:21.459012",
          "exception": false,
          "start_time": "2020-11-30T01:40:21.347994",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "## Cross Validation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T01:40:21.692662Z",
          "iopub.status.busy": "2020-11-30T01:40:21.691694Z",
          "iopub.status.idle": "2020-11-30T01:40:35.582724Z",
          "shell.execute_reply": "2020-11-30T01:40:35.583611Z"
        },
        "id": "lx66CzK_7OUa",
        "papermill": {
          "duration": 14.015105,
          "end_time": "2020-11-30T01:40:35.583763",
          "exception": false,
          "start_time": "2020-11-30T01:40:21.568658",
          "status": "completed"
        },
        "tags": [],
        "outputId": "5379a9e3-c84c-4d4a-fe16-4030cffde8af"
      },
      "source": [
        "oof_weights = [1.0 / N_STARTS for _ in range(N_STARTS)]\n",
        "model_weights = [1.0 / len(models) for _ in range(len(models))]\n",
        "seed_weights = [1.0 / N_SEED for _ in range(N_SEED)]\n",
        "\n",
        "y_true = target.values[: non_target.shape[0]]\n",
        "\n",
        "print(f\"===== OOF - CV =====\")\n",
        "for key, val in oof.items():\n",
        "    if CTRL_VEHICLE in (\"use\", \"keep\"):\n",
        "        oof[key].loc[train_pub_test[\"cp_type\"] == \"ctl_vehicle\", :] = 0\n",
        "\n",
        "    print(f\"OOF Key: {key}, CV: {metric(y_true, val.values[:y_true.shape[0]])}\")\n",
        "\n",
        "oof_by_model = {\n",
        "    f\"{model['model_name']}_{model['cv']}_fit({str(model['fit'])})\": {\n",
        "        k: v\n",
        "        for k, v in oof.items()\n",
        "        if k.startswith(f\"{model['model_name']}_{model['cv']}_fit({str(model['fit'])})\")\n",
        "    }\n",
        "    for model in models\n",
        "}\n",
        "\n",
        "blend_by_model = {}\n",
        "for model, oof_ in oof_by_model.items():\n",
        "    print(f\"\\n===== Model {model} - CV =====\")\n",
        "    _, _, blend_by_model[model] = cross_validation(y_true.shape, seed_weights, y_true, oof_)\n",
        "\n",
        "print(f\"\\n===== Overall - CV =====\")\n",
        "_ = cross_validation(y_true.shape, model_weights, y_true, blend_by_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "===== OOF - CV =====\n",
            "OOF Key: ResNet_without_drug_id_fit(True)_0, CV: 0.015549920786218762\n",
            "OOF Key: TabNet_without_drug_id_fit(True)_1, CV: 0.016379545413917004\n",
            "OOF Key: NODE_without_drug_id_fit(True)_2, CV: 0.015617632925199512\n",
            "OOF Key: ResNet_without_drug_id_fit(True)_3, CV: 0.015571681574280447\n",
            "OOF Key: TabNet_without_drug_id_fit(True)_4, CV: 0.016357813363721805\n",
            "OOF Key: NODE_without_drug_id_fit(True)_5, CV: 0.015517192821919377\n",
            "\n",
            "===== Model ResNet_without_drug_id_fit(True) - CV =====\n",
            "Blended CV: 0.015187612866172193, AUC : 0.8413028495809948\n",
            "\n",
            "===== Model TabNet_without_drug_id_fit(True) - CV =====\n",
            "Blended CV: 0.01615445874266707, AUC : 0.7883251866030232\n",
            "\n",
            "===== Model NODE_without_drug_id_fit(True) - CV =====\n",
            "Blended CV: 0.015304891187367803, AUC : 0.8276664422474292\n",
            "\n",
            "===== Overall - CV =====\n",
            "Blended CV: 0.01512241439772212, AUC : 0.8415474216651967\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T01:40:35.825363Z",
          "iopub.status.busy": "2020-11-30T01:40:35.824396Z",
          "iopub.status.idle": "2020-11-30T01:40:35.827618Z",
          "shell.execute_reply": "2020-11-30T01:40:35.827126Z"
        },
        "id": "CWP8oBRmrKn7",
        "papermill": {
          "duration": 0.128706,
          "end_time": "2020-11-30T01:40:35.827724",
          "exception": false,
          "start_time": "2020-11-30T01:40:35.699018",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "if optimize == \"fixed\":\n",
        "    model_weights = fixed_weight\n",
        "    print(f\"Fixed weights: {model_weights}\")\n",
        "\n",
        "    cross_validation(y_true.shape, model_weights, y_true, blend_by_model)\n",
        "\n",
        "elif optimize == \"lagrange\":\n",
        "    # https://www.kaggle.com/gogo827jz/optimise-blending-weights-with-bonus-0#Bonus-(Lagrange-Multiplier)\n",
        "\n",
        "    def lagrange_func(params):\n",
        "        # weights, _lambda = params\n",
        "        blend_ = blend(y_true.shape, params[:-1], blend_by_model)\n",
        "        return metric(y_true, blend_) - params[-1] * (sum(params[:-1]) - 1)\n",
        "\n",
        "    grad_l = grad(lagrange_func)\n",
        "\n",
        "    def lagrange_obj(params):\n",
        "        # weights, _lambda = params\n",
        "        d = grad_l(params).tolist()\n",
        "        return d[:-1] + [sum(params[:-1]) - 1]\n",
        "\n",
        "    optimized_weights = fsolve(lagrange_obj, model_weights + [1.0])\n",
        "\n",
        "    print(f\"Optimized weights: {optimized_weights[:-1]}\")\n",
        "    print(f\"Check the sum of all weights: {sum(optimized_weights[:-1])}\")\n",
        "\n",
        "    cross_validation(y_true.shape, optimized_weights[:-1], y_true, blend_by_model)\n",
        "\n",
        "    model_weights = optimized_weights[:-1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T01:40:36.069719Z",
          "iopub.status.busy": "2020-11-30T01:40:36.068591Z",
          "iopub.status.idle": "2020-11-30T01:40:36.471024Z",
          "shell.execute_reply": "2020-11-30T01:40:36.470296Z"
        },
        "id": "fkapdEQ8pDfg",
        "papermill": {
          "duration": 0.529852,
          "end_time": "2020-11-30T01:40:36.471143",
          "exception": false,
          "start_time": "2020-11-30T01:40:35.941291",
          "status": "completed"
        },
        "tags": [],
        "outputId": "91538e7d-fa4d-4a25-f707-830eb19aff4a"
      },
      "source": [
        "predictions_by_model = {\n",
        "    f\"{model['model_name']}_{model['cv']}_fit({str(model['fit'])})\": {\n",
        "        k: v\n",
        "        for k, v in predictions.items()\n",
        "        if k.startswith(f\"{model['model_name']}_{model['cv']}_fit({str(model['fit'])})\")\n",
        "    }\n",
        "    for model in models\n",
        "}\n",
        "\n",
        "blend_by_model = {\n",
        "    f\"{model['model_name']}_{model['cv']}_fit({str(model['fit'])})\": pd.DataFrame(blend(\n",
        "        ss.shape,\n",
        "        seed_weights,\n",
        "        predictions_by_model[f\"{model['model_name']}_{model['cv']}_fit({str(model['fit'])})\"]\n",
        "    ))\n",
        "    for model in models\n",
        "}\n",
        "\n",
        "for a, b in itertools.combinations(blend_by_model.keys(), 2):\n",
        "    corr = blend_by_model[a].corrwith(blend_by_model[b], axis=1)\n",
        "    print(f\"Prediction correlation between {a} and {b}: {corr.mean()}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Prediction correlation between ResNet_without_drug_id_fit(True) and TabNet_without_drug_id_fit(True): 0.8500191513483923\n",
            "Prediction correlation between ResNet_without_drug_id_fit(True) and NODE_without_drug_id_fit(True): 0.9447938261897578\n",
            "Prediction correlation between TabNet_without_drug_id_fit(True) and NODE_without_drug_id_fit(True): 0.8546747993252847\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zghrQJ2L7OUl",
        "papermill": {
          "duration": 0.116541,
          "end_time": "2020-11-30T01:40:36.702194",
          "exception": false,
          "start_time": "2020-11-30T01:40:36.585653",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Postprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T01:40:36.938584Z",
          "iopub.status.busy": "2020-11-30T01:40:36.937377Z",
          "iopub.status.idle": "2020-11-30T01:40:37.271532Z",
          "shell.execute_reply": "2020-11-30T01:40:37.270673Z"
        },
        "id": "6R71_Ar-7OUl",
        "papermill": {
          "duration": 0.455715,
          "end_time": "2020-11-30T01:40:37.271655",
          "exception": false,
          "start_time": "2020-11-30T01:40:36.815940",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Weighted blend\n",
        "submit_df.loc[:, target.columns] = blend(ss.shape, model_weights, blend_by_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T01:40:37.521895Z",
          "iopub.status.busy": "2020-11-30T01:40:37.520863Z",
          "iopub.status.idle": "2020-11-30T01:40:37.559981Z",
          "shell.execute_reply": "2020-11-30T01:40:37.559490Z"
        },
        "id": "JHpr_90W7OUp",
        "papermill": {
          "duration": 0.171411,
          "end_time": "2020-11-30T01:40:37.560099",
          "exception": false,
          "start_time": "2020-11-30T01:40:37.388688",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "submit_df.loc[test_df[\"cp_type\"] == \"ctl_vehicle\", target.columns] = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7mHXnoKn7OUr",
        "papermill": {
          "duration": 0.118146,
          "end_time": "2020-11-30T01:40:37.799370",
          "exception": false,
          "start_time": "2020-11-30T01:40:37.681224",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "# Output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "execution": {
          "iopub.execute_input": "2020-11-30T01:40:38.116646Z",
          "iopub.status.busy": "2020-11-30T01:40:38.115645Z",
          "iopub.status.idle": "2020-11-30T01:40:40.486262Z",
          "shell.execute_reply": "2020-11-30T01:40:40.484714Z"
        },
        "id": "7pdd1k1E7OUr",
        "papermill": {
          "duration": 2.55584,
          "end_time": "2020-11-30T01:40:40.486387",
          "exception": false,
          "start_time": "2020-11-30T01:40:37.930547",
          "status": "completed"
        },
        "tags": []
      },
      "source": [
        "submit_df.to_csv(\"submission.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}