final_cnn_v7.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "final-cnn-v7.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [
        "VizHiNgIepz3",
        "pxQjuU2CYx5f"
      ],
      "toc_visible": true,
      "mount_file_id": "1xcIId9jP3RPLAeyWC55VfEuSnxEBtl8S",
      "authorship_tag": "ABX9TyOMOWuO+hjO2AmUpDMElsHf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/puaqieshang/automatic-labeling-heart-vessels/blob/master/final_cnn_v7.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0q3bQztjs6pW",
        "colab_type": "text"
      },
      "source": [
        "# Update from Previous Version\n",
        "\n",
        "* Part 1 - Overfitting done\n",
        "* Part 2 - Regularization done?? Val Accuracy > Train Accuracy tho \n",
        "* Part 3 - Tuned hyperparameters; rationale is in book\n",
        "\n",
        "Results:\n",
        "\n",
        "*   Test accuracy is promising\n",
        "    *  Tested on 2 datasets \n",
        "    *  Accuracy is around 88%\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ogFQbGjAd5_L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import dependencies\n",
        "\n",
        "import os\n",
        "import glob\n",
        "import time\n",
        "import helper\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.utils import shuffle\n",
        "from collections import defaultdict\n",
        "\n",
        "import torch\n",
        "from torch import tensor, nn, optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda:0\")  # you can continue going on here, like cuda:1 cuda:2....etc. \n",
        "    print(\"Running on the GPU\")\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "    print(\"Running on the CPU\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BsXDGkEGj5wt",
        "colab_type": "text"
      },
      "source": [
        "# Data Retrieval\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4EE5l4kdHw8r",
        "colab_type": "text"
      },
      "source": [
        "<table>\n",
        "  <tr>\n",
        "    <th>Label</th>\n",
        "    <th>Class (Vessels)</th>\n",
        "  </tr>\n",
        "  <tr>\n",
        "    <td>0</td>\n",
        "    <td>LAD</td>\n",
        "  </tr>\n",
        "  <tr>\n",
        "    <td>1</td>\n",
        "    <td>Diagonals</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>2</td>\n",
        "    <td>Septals</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>3</td>\n",
        "    <td>LCX</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>4</td>\n",
        "    <td>Obtuse Marginal</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>5</td>\n",
        "    <td>Atrials</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>6</td>\n",
        "    <td>LCIM</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>7</td>\n",
        "    <td>Acutes</td>\n",
        "  </tr>\n",
        "    <tr>\n",
        "    <td>8</td>\n",
        "    <td>Crux</td>\n",
        "  </tr>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W8qlfLlwx8MO",
        "colab_type": "text"
      },
      "source": [
        "## Import from Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4B4g74MZc0dL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load the Drive helper and mount\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# !unzip -uq \"/content/drive/My Drive/ldai-load.zip\"\n",
        "!unzip -uq \"/content/drive/My Drive/processed_v2.zip\"\n",
        "!ls \"/content/\" # Lists all the files "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nLr3PkaLAK7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def storeData(dest):\n",
        "\n",
        "    X_all = []\n",
        "    y_all = []\n",
        "    folders = os.listdir(dest)\n",
        "    idCount = 0\n",
        "\n",
        "    for folder in folders:\n",
        "\n",
        "        if (folder == \".DS_Store\"):\n",
        "            continue\n",
        "        \n",
        "        file_des = f\"{dest}/{folder}\"\n",
        "        all_files = os.listdir(file_des)\n",
        "        # print(f\"Folder {folder} containes {all_files}\")\n",
        "\n",
        "        os.chdir(file_des)\n",
        "        for f in glob.glob(\"*.txt\"):\n",
        "            \n",
        "            geometry = np.genfromtxt(f)\n",
        "            geometry = geometry.tolist()\n",
        "            # print(f\"The file {f} has {geometry[0]}\")              \n",
        "            # print(geometry.shape)\n",
        "            X_all.append(geometry)\n",
        "            y_all.append(folder)\n",
        "\n",
        "            idCount = idCount +1\n",
        "\n",
        "    X_all, y_all = shuffle(np.array(X_all), np.array(y_all), random_state=0)\n",
        "    \n",
        "    return X_all, y_all\n",
        "\n",
        "dest = '/content/processed_v2/'\n",
        "X_all, y_all = storeData(dest)\n",
        "\n",
        "\n",
        "print(X_all.shape)\n",
        "print(y_all[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vorrIKyZwZvO",
        "colab_type": "text"
      },
      "source": [
        "## Develop Custom Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-now3mxWhlcL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class HeartVesselsDataset(Dataset):\n",
        "\n",
        "    # Initialize your data, download, etc.\n",
        "    def __init__(self, attributes_data, labels):\n",
        "\n",
        "        self.x_data = attributes_data # Access dict\n",
        "        self.y_data = labels # Access dict\n",
        "        self.len = labels.shape[0]\n",
        "        \n",
        "    # USE DICTIONARY\n",
        "    def __getitem__(self, index):\n",
        "        # print(index)\n",
        "        geometry = self.x_data[index]\n",
        "        label = self.y_data[index] # Its a number\n",
        "        # print(label)\n",
        "        \n",
        "        return geometry, tensor(int(label))\n",
        "\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.x_data)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4KoNEnafwq8Y",
        "colab_type": "text"
      },
      "source": [
        "## Create Train and Validate Datasets\n",
        "\n",
        "Cross validation folds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HpQ6U3ySIs8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import KFold # import KFold\n",
        "X = X_all \n",
        "print(X.shape)# create an array\n",
        "y = y_all # Create another array\n",
        "kf = KFold(n_splits=5,random_state=None, shuffle=False) # Define the split - into 5 folds \n",
        "# kf.get_n_splits(X) # returns the number of splitting iterations in the cross-validator\n",
        "print(kf) \n",
        "# kf = KFold(n_splits=5, random_state=None, shuffle=False)\n",
        "\n",
        "for train_index, val_index in kf.split(X):\n",
        "    \n",
        "    X_train, X_val = X[train_index], X[val_index]\n",
        "    y_train, y_val = y[train_index], y[val_index]\n",
        "    # print(X_val[0][0])\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6trhsNY-wp-j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "train_dataset = HeartVesselsDataset(X_train, y_train)\n",
        "trainloader = DataLoader(dataset=train_dataset, batch_size=16,shuffle=True,num_workers=0)\n",
        "\n",
        "\n",
        "val_dataset = HeartVesselsDataset(X_val, y_val)\n",
        "valloader = DataLoader(dataset=val_dataset, batch_size=16,shuffle=True,num_workers=0)\n",
        "\n",
        "print(f\"Train has {len(trainloader)} batches and {train_dataset.len} datasets\")\n",
        "print(f\"Validation has {len(valloader)} batches and {val_dataset.len} datasets\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "APGAYL6FxfHE",
        "colab_type": "text"
      },
      "source": [
        "## Inspecting Input - A Single Branch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jDOQJS0Wm29G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "geometries, label = next(iter(trainloader))\n",
        "\n",
        "# print(f\"Geo is of type {geometries.type}\")\n",
        "# print(f\"Label is of type {label.type}\")\n",
        "print(\"A Random Coronary Branch\")\n",
        "print(f\"Dimension of a branch/matrix is {geometries[0].shape}\")\n",
        "\n",
        "plt.imshow(geometries[0].view(20, 20))\n",
        "plt.show()\n",
        "print(f\"The label is {label[0]}\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xv9OG9k3kAjA",
        "colab_type": "text"
      },
      "source": [
        "# Model Definition\n",
        "\n",
        "So that different model architectures can work interchangeable\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V7dBvPPD73SN",
        "colab_type": "text"
      },
      "source": [
        "## Linear Layers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b9wlmug-LeLy",
        "colab_type": "text"
      },
      "source": [
        "### Model 1 - nn.Linear in Sequential Form\n",
        "\n",
        "A simple Multi-Layer Perceptron, but achieves accuracy of 0.78 - 0.8 after 350 epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RQy8VcNALcrc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# model = nn.Sequential(nn.Linear(400,128), nn.ReLU(), nn.Linear(128,64), \n",
        "#                       nn.ReLU(), nn.Linear(64,9), nn.LogSoftmax(dim=1))\n",
        "\n",
        "# model = model.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rZ8LPLeb1dHQ",
        "colab_type": "text"
      },
      "source": [
        "### Model 2 - nn.Linear in Class form \n",
        "Basically same as previous model, just that its in nn.fucntional() form. This allows user to tune the parameters more, which is better. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSpsZlv2B9be",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # Start Timer\n",
        "# t0 = time.time()\n",
        "\n",
        "\n",
        "# class ConvNN(nn.Module):\n",
        "#     def __init__(self):\n",
        "#         super().__init__()\n",
        "#         # Defining the layers, 128, 64, 9 units each\n",
        "#         self.fc1 = nn.Linear(400, 128)\n",
        "#         self.fc2 = nn.Linear(128, 64)\n",
        "#         # Output layer, 9 units - one for each digit\n",
        "#         self.fc3 = nn.Linear(64, 9)\n",
        "        \n",
        "#     def forward(self, x):\n",
        "#         ''' Forward pass through the network, returns the output logits '''\n",
        "        \n",
        "#         x = self.fc1(x)\n",
        "#         x = F.relu(x)\n",
        "#         x = self.fc2(x)\n",
        "#         x = F.relu(x)\n",
        "#         x = self.fc3(x)\n",
        "#         x = F.log_softmax(x, dim=1)\n",
        "        \n",
        "#         return x\n",
        "\n",
        "\n",
        "# model = ConvNN()\n",
        "# model = model.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "voy06Xl8797G",
        "colab_type": "text"
      },
      "source": [
        "## Convolutional Layers\n",
        "\n",
        "Suggestions to Improve:\n",
        "* Use He Initialization for ReLU layers instead of random or zero initiliazation. Refer to roadmap\n",
        "* Use more regularization techniques (if necessary)\n",
        "\n",
        "Things to note:\n",
        "* Increasing number of layers by more than 2 do not improve the accuracy - could be due to my model being simple?\n",
        "* No need to have a lot of nodes in a hidden layer; the input is not a 2D image anyways.\n",
        "* [Dropout Layers FAQ ](https://https://stats.stackexchange.com/questions/240305/where-should-i-place-dropout-layers-in-a-neural-network )\n",
        "* Dropout layers may result in [Test Accuracy > Train Accuracy](https://www.quora.com/How-can-I-explain-the-fact-that-test-accuracy-is-much-higher-than-train-accuracy)  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aSq31moBhyNi",
        "colab_type": "text"
      },
      "source": [
        "### Model 3 - Conv1D with 3 Hidden Layers\n",
        "\n",
        "Comment out the Regularisation if not needed\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGcAZPPMhxWD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ConvNN(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        \n",
        "        # Input Tensor [batch, in_channels, in_width]\n",
        "        x = torch.randn(100, 4).view(-1, 4, 100)\n",
        "        # print(x.shape)\n",
        "\n",
        "        # Defining the layers, 4, 8, 16 and 32 units each\n",
        "\n",
        "        self.conv1 = nn.Conv1d(in_channels=4, out_channels=16, kernel_size=3)\n",
        "        self.conv2 = nn.Conv1d(in_channels=16, out_channels=64, kernel_size=3)\n",
        "        self.conv3 = nn.Conv1d(in_channels=64, out_channels=256, kernel_size=3)\n",
        "\n",
        "        # Activated during eval mode\n",
        "        self.conv1_bn = nn.BatchNorm1d(16)\n",
        "        self.conv2_bn = nn.BatchNorm1d(64)\n",
        "        self.conv3_bn = nn.BatchNorm1d(256)\n",
        "        self.fc1_bn = nn.BatchNorm1d(128)\n",
        "        self.fc2_bn = nn.BatchNorm1d(32)\n",
        "    \n",
        "        self.fc_dropout = nn.Dropout(p=0.5)\n",
        "        self.conv_dropout = nn.Dropout(p=0.1)\n",
        "        \n",
        "        self._to_linear = None\n",
        "\n",
        "        self.convs(x)\n",
        "\n",
        "        # self.fc1 = nn.Linear(self._to_linear, 64) #flattening.\n",
        "        # self.fc2 = nn.Linear(64, 9) # 64 in, 9out bc we're doing 9 classes \n",
        "\n",
        "        self.fc1 = nn.Linear(self._to_linear, 128) #flattening.\n",
        "        self.fc2 = nn.Linear(128, 32)\n",
        "        self.fc3 = nn.Linear(32, 9)\n",
        "\n",
        "    def convs(self, x):\n",
        "        # max pooling over 2\n",
        "        \n",
        "        x = self.conv1(x)\n",
        "        x = self.conv1_bn(x)  \n",
        "        x = F.relu(x)\n",
        "        x = self.conv_dropout(x)    \n",
        "        x = F.max_pool1d(x, 2) #or Elu\n",
        "\n",
        "        x = self.conv2(x)\n",
        "        x = self.conv2_bn(x)   \n",
        "        x = F.relu(x)\n",
        "        x = self.conv_dropout(x)\n",
        "        x = F.max_pool1d(x, 2) #or Elu\n",
        "\n",
        "        x = self.conv3(x)\n",
        "        x = self.conv3_bn(x)   \n",
        "        x = F.relu(x)\n",
        "        x = self.conv_dropout(x)\n",
        "        x = F.max_pool1d(x, 2) #or Elu\n",
        "        \n",
        "        if self._to_linear is None:\n",
        "            self._to_linear = x[0].shape[0]*x[0].shape[1]\n",
        "            \n",
        "        return x\n",
        "        \n",
        "    def forward(self, x):\n",
        "        ''' Forward pass through the network, returns the output logits '''\n",
        "        \n",
        "        x = self.convs(x)\n",
        "        x = x.view(-1, self._to_linear)  # .view is reshape ... this flattens X before \n",
        "\n",
        "        x = self.fc1(x)\n",
        "        x = self.fc1_bn(x)\n",
        "        x = F.relu(x)\n",
        "        x = self.fc_dropout(x) \n",
        "\n",
        "        x = self.fc2(x) # bc this is our output layer. No activation here.\n",
        "        x = self.fc2_bn(x)\n",
        "        x = F.relu(x)\n",
        "        x = self.fc_dropout(x)   \n",
        "\n",
        "        x = self.fc3(x)\n",
        "\n",
        "        return F.log_softmax(x, dim=1)\n",
        "        \n",
        "        return x\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VizHiNgIepz3",
        "colab_type": "text"
      },
      "source": [
        "### Model 4 - Conv1D with 2 Layers\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_7UrOf4aBgpe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# class ConvNN(nn.Module):\n",
        "#     def __init__(self):\n",
        "#         super().__init__()\n",
        "        \n",
        "#         # Input Tensor [batch, in_channels, in_width]\n",
        "#         x = torch.randn(100, 4).view(-1, 4, 100)\n",
        "#         print(x.shape)\n",
        "    \n",
        "#         '''\n",
        "#         The node layers and numbers are based on multiple websites\n",
        "#         https://stats.stackexchange.com/questions/181/how-to-choose-the-number-of-hidden-layers-and-nodes-in-a-feedforward-neural-netw\n",
        "#         https://www.researchgate.net/post/In_neural_networks_model_which_number_of_hidden_units_to_select\n",
        "#         '''\n",
        "        \n",
        "#         self.conv1 = nn.Conv1d(in_channels=4, out_channels=7, kernel_size=3) \n",
        "#         self.conv2 = nn.Conv1d(in_channels=7, out_channels=5, kernel_size=3)\n",
        "#         # self.conv1_bn = nn.BatchNorm1d(7)\n",
        "#         # self.conv2_bn = nn.BatchNorm1d(5)\n",
        "#         self.fc_dropout = nn.Dropout(p=0.5)\n",
        "#         self.conv_dropout = nn.Dropout(p=0.1)\n",
        "        \n",
        "#         self._to_linear = None\n",
        "\n",
        "#         self.convs(x)\n",
        "\n",
        "#         self.fc1 = nn.Linear(self._to_linear, 64) #flattening.\n",
        "#         self.fc2 = nn.Linear(64, 9) # 64 in, 9out bc we're doing 9 classes \n",
        "\n",
        "#     def convs(self, x):\n",
        "#         # max pooling over 2\n",
        "#         x = self.conv1(x)\n",
        "#         # x = self.conv1_bn(x)       \n",
        "#         x = F.max_pool1d(self.conv_dropout(F.relu(x)), 2) #or Elu\n",
        "\n",
        "#         x = self.conv2(x)\n",
        "#         # x = self.conv2_bn(x)       \n",
        "#         x = F.max_pool1d(self.conv_dropout(F.relu(x)), 2) #or Elu\n",
        "\n",
        "#         # x = self.conv3(x)\n",
        "#         # # x = self.conv3_bn(x)       \n",
        "#         # x = F.max_pool1d(F.relu(x), 2) #or Elu\n",
        "        \n",
        "#         if self._to_linear is None:\n",
        "#             self._to_linear = x[0].shape[0]*x[0].shape[1]\n",
        "            \n",
        "#         return x\n",
        "        \n",
        "#     def forward(self, x):\n",
        "#         ''' Forward pass through the network, returns the output logits '''\n",
        "        \n",
        "#         x = self.convs(x)\n",
        "#         x = x.view(-1, self._to_linear)  # .view is reshape ... this flattens X before \n",
        "\n",
        "#         x = self.fc1(x)\n",
        "#         # x = self.fc1_bn(x)\n",
        "#         x = F.relu(x)\n",
        "#         x = self.fc_dropout(x) \n",
        "\n",
        "#         x = self.fc2(x) # bc this is our output layer. No activation here.\n",
        "#         return F.log_softmax(x, dim=1)\n",
        "        \n",
        "#         return x\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5c8rxsQup9Ts",
        "colab_type": "text"
      },
      "source": [
        "# Train and Validate Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y_c7jC0r8Lzv",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "```\n",
        "model.train(), model.eval()\n",
        "```\n",
        "\n",
        "Both are the same. By default all the modules are initialized to train mode (self.training = True). \n",
        "\n",
        "In case you want to validate your data, call model.eval() before feeding the data, as this will change the behavior of the BatchNorm (or Dropout) layer to use the running estimates instead of calculating them for the current batch.\n",
        "If you want to train your model and can’t use a bigger batch size, you could switch e.g. to InstanceNorm."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlPEe_rknZYi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(loader): # for train cases\n",
        "    running_loss = 0\n",
        "    sumCorrect = 0\n",
        "    sumTotal = 0      \n",
        "\n",
        "    model.train() # for train cases\n",
        "        \n",
        "    for X, y in loader:\n",
        "\n",
        "        X = X.view(-1, 4, 100)\n",
        "        # print(X.shape)\n",
        "        X, y = X.to(device), y.to(device)\n",
        "        \n",
        "        optimiser.zero_grad()\n",
        "        \n",
        "        # train_X = train_X.view(-1, 4, 100)\n",
        "        output = model(X.float())\n",
        "        _, predicted = torch.max(output.data, 1)\n",
        "        loss = criterion(output,y)\n",
        "\n",
        "        loss.backward()\n",
        "        optimiser.step()\n",
        "        \n",
        "        running_loss += loss.item()\n",
        "        \n",
        "        # Accuracy\n",
        "        sumCorrect += (predicted == y).sum().item()\n",
        "        sumTotal += y.size(0)\n",
        "\n",
        "    l = running_loss/len(loader) #average loss for whole loader dataset\n",
        "    acc = sumCorrect/sumTotal\n",
        "\n",
        "    return acc, l\n",
        "\n",
        "def validate(loader):\n",
        "    running_loss = 0\n",
        "    sumCorrect = 0\n",
        "    sumTotal = 0      \n",
        "\n",
        "    model.eval() # for validation cases\n",
        "    with torch.no_grad():   \n",
        "        for X, y in loader:\n",
        "\n",
        "            X = X.view(-1, 4, 100)\n",
        "            # print(X.shape)\n",
        "            X, y = X.to(device), y.to(device)\n",
        "            \n",
        "            \n",
        "            # train_X = train_X.view(-1, 4, 100)\n",
        "            output = model(X.float())\n",
        "            _, predicted = torch.max(output.data, 1)\n",
        "            loss = criterion(output,y)\n",
        "\n",
        "            running_loss += loss.item()         \n",
        "\n",
        "            # Accuracy\n",
        "            sumCorrect += (predicted == y).sum().item()\n",
        "            sumTotal += y.size(0)\n",
        "            # print(sumTotal)   \n",
        "\n",
        "    l = running_loss/len(loader) #average loss for whole loader dataset\n",
        "    acc = sumCorrect/sumTotal\n",
        "\n",
        "    return acc, l\n",
        "\n",
        "def train_and_validate(trainloader, valloader):\n",
        "    \n",
        "    for e in tqdm(range(EPOCHS), position=0, leave=True):\n",
        "\n",
        "        # train\n",
        "        train_acc, train_loss = train(trainloader)\n",
        "        train_acc_list.append(train_acc)\n",
        "        train_loss_list.append(train_loss)\n",
        "        \n",
        "        # Validate\n",
        "    \n",
        "        val_acc, val_loss = validate(valloader)\n",
        "        val_acc_list.append(val_acc)\n",
        "        val_loss_list.append(val_loss)\n",
        "\n",
        "        # print(f\"Epoch {e} has test accuracy of {round(test_acc, 3)} and loss of {round(test_loss,3)}\")\n",
        "    \n",
        "    return sum(val_acc_list)/len(val_acc_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UfCPrZ1UkJhM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = ConvNN()\n",
        "model = model.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hAF_19BC52Hy",
        "colab_type": "text"
      },
      "source": [
        "## Calculate Accuracy and Loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nYYBoUuVB7Ek",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Start Timer\n",
        "t0 = time.time()\n",
        "\n",
        "'''\n",
        "https://medium.com/octavian-ai/which-optimizer-and-learning-rate-should-i-use-for-deep-learning-5acb418f9b2\n",
        "'''\n",
        "criterion = nn.NLLLoss()\n",
        "# optimiser = optim.Adam(model.parameters(), lr=0.0001)\n",
        "optimiser = optim.Adam(model.parameters(), lr=0.0001, weight_decay=1e-4)\n",
        "\n",
        "# Refer to Deep Learning Roadmap\n",
        "\n",
        "train_acc_list, train_loss_list, val_acc_list, val_loss_list = [],[],[],[]\n",
        "EPOCHS = 300\n",
        "\n",
        "avg_val_accuracy = train_and_validate(trainloader, valloader)\n",
        "\n",
        "t1 = time.time()\n",
        "duration = (t1-t0)/60\n",
        "print(\"\\nTime taken to train is \" + \"{:.2f}\".format(duration) + \" minutes.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9l8qw83t58OQ",
        "colab_type": "text"
      },
      "source": [
        "## Plot Accuracies and Loss vs Epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wZCaUppuB2Aj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from matplotlib import style\n",
        "import pandas as pd\n",
        "\n",
        "style.use(\"ggplot\")\n",
        "\n",
        "def create_acc_loss_graph():\n",
        "    epoch = list(range(0, EPOCHS, 1))\n",
        "\n",
        "    d = {'epochs': epoch,\n",
        "        'train_acc': train_acc_list,\n",
        "        'train_loss': train_loss_list,\n",
        "        'val_acc': val_acc_list, \n",
        "        'val_loss': val_loss_list}\n",
        "\n",
        "    df = pd.DataFrame(d)\n",
        "\n",
        "    df['train_acc_avg'] = df['train_acc'].ewm(alpha=.02).mean()  # exponential weighted moving average\n",
        "    df['val_acc_avg'] = df['val_acc'].ewm(alpha=.02).mean()\n",
        "    df['train_loss_avg'] = df['train_loss'].ewm(alpha=.02).mean()\n",
        "    df['val_loss_avg'] = df['val_loss'].ewm(alpha=.02).mean()\n",
        "\n",
        "    # Then plot using pandas:\n",
        "    df.plot(x='epochs', y=['train_acc_avg', 'val_acc_avg'], figsize=(8,4))\n",
        "    plt.ylabel(\"Accuracy\")\n",
        "    df.plot(x='epochs', y=['train_loss_avg', 'val_loss_avg'], figsize=(8,4))\n",
        "    plt.ylabel(\"Loss\")\n",
        "\n",
        "    plt.show()\n",
        "\n",
        "create_acc_loss_graph()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F09_QQ7r_qyw",
        "colab_type": "text"
      },
      "source": [
        "## Displays Model Summary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-AEnfmgy3qVL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torchvision import models\n",
        "model = models.vgg16()\n",
        "print(model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TvErnUHdHhYn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Prints the model's state for each layer\n",
        "print(\"Model's state_dict:\")\n",
        "for param_tensor in model.state_dict():\n",
        "    print(param_tensor, \"\\t\", model.state_dict()[param_tensor].size())\n",
        "\n",
        "# Print the weights of the model - A LOT OF MATRICES\n",
        "# print(\"Optimizer's state_dict:\")\n",
        "# for var_name in optimiser.state_dict():\n",
        "#     print(var_name, \"\\t\", optimiser.state_dict()[var_name])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZHLzE8CxsFpk",
        "colab_type": "text"
      },
      "source": [
        "# Visualization\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TNJoiAuHI_xn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "%config InlineBackend.figure_format = 'retina'\n",
        "\n",
        "dataiter = iter(valloader)\n",
        "geometries, labels = dataiter.next()\n",
        "geom = geometries[0]\n",
        "# Convert 2D matrix/image to 1D vector\n",
        "\n",
        "geom = geom.resize_(1, 400)\n",
        "\n",
        "\n",
        "# TODO: Calculate the class probabilities (softmax) for img\n",
        "with torch.no_grad():\n",
        "    \n",
        "    logps = model(geom.cuda().view(-1, 4, 100).float()) # convert to cuda tensor to run faster\n",
        "    \n",
        "ps = torch.exp(logps)\n",
        "\n",
        "# Plot the image and probabilities\n",
        "\n",
        "def view_classify(geom, ps):\n",
        "    ''' Function for viewing an image and it's predicted classes.\n",
        "    '''\n",
        "    ps = ps.data.cpu().numpy().squeeze()\n",
        "\n",
        "    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n",
        "    ax1.imshow(geom.resize_(20, 20).numpy().squeeze()) #First figure on left\n",
        "    \n",
        "    ax1.axis('off')\n",
        "    ax2.barh(np.arange(9), ps) #Make a horizontal bar plot\n",
        "    ax2.set_aspect(0.1)\n",
        "    ax2.set_yticks(np.arange(10))\n",
        "    ax2.set_yticklabels(vessels_names, size='medium');\n",
        "    ax2.set_title('Class Probability')\n",
        "    ax2.set_xlim(0, 1.1)\n",
        "\n",
        "    plt.tight_layout()\n",
        "\n",
        "\n",
        "geom_numpy = geom.cpu().numpy()\n",
        "view_classify(geom.reshape(20, 20), ps) # convert back to numpy\n",
        "\n",
        "print(f\"The actual vessel is {vessels_names[labels[0]]}\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pxQjuU2CYx5f",
        "colab_type": "text"
      },
      "source": [
        "# Analysis "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iY5iEU8f69aZ",
        "colab_type": "text"
      },
      "source": [
        "## Confusion Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cqnJMlKtUlfs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn import metrics\n",
        "\n",
        "# True values\n",
        "y_true = true_class_array\n",
        "# Predicted values\n",
        "y_pred = predicted_class_array\n",
        "\n",
        "# Print the confusion matrix\n",
        "print(metrics.confusion_matrix(y_true, y_pred))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dEAMgvzs_eCH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Print the precision and recall, among other metrics\n",
        "print(metrics.classification_report(y_true, y_pred, digits=3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_zvNSlZY16Q7",
        "colab_type": "text"
      },
      "source": [
        "# Main Pipeline\n",
        "\n",
        "Done **last** after finalising code.\n",
        "\n",
        "Basically change every module or header section into a def so that its easier to call. \n",
        "\n",
        "All the user does is to run this code.\n",
        "\n",
        " Outputs the average accuracy of the model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NftXXQowxWWl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "accuracies = []\n",
        "import statistics as stats\n",
        "\n",
        "\n",
        "X = X_all \n",
        "y = y_all # Create another array\n",
        "kf = KFold(n_splits=5,random_state=None, shuffle=False) # Define the split - into 5 folds \n",
        "# kf.get_n_splits(X) # returns the number of splitting iterations in the cross-validator\n",
        "# print(kf) \n",
        "# kf = KFold(n_splits=5, random_state=None, shuffle=False)\n",
        "\n",
        "for train_index, val_index in kf.split(X):\n",
        "    # print(\"TRAIN:\", train_index, \"TEST:\", val_index)\n",
        "    # print(train)\n",
        "    X_train, X_val = X[train_index], X[val_index]\n",
        "    y_train, y_val = y[train_index], y[val_index]\n",
        "\n",
        "    train_dataset = HeartVesselsDataset(X_train, y_train)\n",
        "    trainloader = DataLoader(dataset=train_dataset, batch_size=16,shuffle=True,num_workers=0)\n",
        "\n",
        "    val_dataset = HeartVesselsDataset(X_val, y_val)\n",
        "    valloader = DataLoader(dataset=val_dataset, batch_size=16,shuffle=True,num_workers=0)\n",
        "\n",
        "    model = ConvNN()\n",
        "    model = model.to(device)\n",
        "\n",
        "    t0 = time.time()\n",
        "    criterion = nn.NLLLoss()\n",
        "    optimiser = optim.Adam(model.parameters(), lr=0.0001, weight_decay=1e-4)\n",
        "\n",
        "    # Refer to Deep Learning Roadmap\n",
        "    train_acc_list, train_loss_list, val_acc_list, val_loss_list = [],[],[],[]\n",
        "    EPOCHS = 275\n",
        "\n",
        "    acc = train_and_validate(trainloader, valloader)\n",
        "    accuracies.append(acc)\n",
        "\n",
        "    t1 = time.time()\n",
        "    duration = (t1-t0)/60\n",
        "    print(\"\\nTime taken to train for epoch is \" + \"{:.2f}\".format(duration) + \" minutes.\")\n",
        "\n",
        "\n",
        "\n",
        "print(f\"\\nThis model has an accuracy of {round(stats.mean(accuracies), 3)} +/- {round(stats.stdev(accuracies), 3)}\")      "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "roCsWtjWr3Qb",
        "colab_type": "text"
      },
      "source": [
        "# Test Model on Unseen Data \n",
        "Well technically, `model.eval() `is not learning any new weights so its testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wQXsop42Nztw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -uq \"/content/drive/My Drive/processed-unseen.zip\" -d '/content' # sends to the specific directory\n",
        "\n",
        "dest = '/content/processed-unseen/'\n",
        "X_test, y_test = storeData(dest)\n",
        "\n",
        "test_dataset = HeartVesselsDataset(X_test, y_test)\n",
        "testloader = DataLoader(dataset=test_dataset, batch_size=16,shuffle=True,num_workers=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8cAsvvzwOvKq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vessels_names = ['ldai', 'diags', 'septals', 'lcxi', 'obtmar', 'atrials', 'lcim', 'acutes', 'crux']\n",
        "\n",
        "def test(model):\n",
        "\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    vessels_count = np.zeros((9,), dtype=int)\n",
        "    vessels_correct = np.zeros((9,), dtype=int)\n",
        "    mainBifurcations_count = np.zeros((3,), dtype=int) \n",
        "    mainBifurcations_correct = np.zeros((3,), dtype=int) \n",
        "\n",
        "    predicted_class_array = []\n",
        "    true_class_array = []\n",
        "\n",
        "    with torch.no_grad():\n",
        "\n",
        "        model.eval()\n",
        "\n",
        "        for test_X, test_y in testloader:\n",
        "\n",
        "            test_y_list = test_y.tolist()\n",
        "            # true_class_array.append(test_y)\n",
        "            true_class_array.extend(test_y_list)\n",
        "            test_X, test_y = test_X.to(device), test_y.to(device)\n",
        "\n",
        "            for i in tqdm(range(len(test_X)), position=0, leave=True):\n",
        "                real_class = test_y[i].item()\n",
        "                # print(f\"real class is {real_class}\")\n",
        "\n",
        "                vessels_count[real_class] += 1\n",
        "                # ps = net(test_X[i].view(-1, 1, 20, 20))[0] \n",
        "                geom = test_X[i]\n",
        "                geom = geom.view(-1, 4, 100)\n",
        "                    \n",
        "                logps = model(geom.float())\n",
        "                ps = torch.exp(logps)  #probabilities\n",
        "                predicted_class = torch.argmax(ps).item()\n",
        "                predicted_class_array.append(predicted_class)\n",
        "\n",
        "                if predicted_class is real_class:\n",
        "                    vessels_correct[real_class] += 1\n",
        "                    correct += 1\n",
        "                \n",
        "                else: \n",
        "                    # These numbers follow the sequence in the \"vessels_name\" list\n",
        "                    # print(predicted_class)\n",
        "                    if predicted_class in {0, 1, 2}: \n",
        "                        \n",
        "                        if real_class in {0, 1, 2}:\n",
        "                            # print(f\"predicted {predicted_class}, real {real_class}\")\n",
        "                            mainBifurcations_correct[0] += 1\n",
        "\n",
        "                    elif predicted_class in {3, 4}:\n",
        "                        if real_class in {3, 4}:\n",
        "                            # print(f\"predicted {predicted_class}, real {real_class}\")\n",
        "                            mainBifurcations_correct[1] += 1\n",
        "                    \n",
        "                    elif predicted_class in {7, 8}:\n",
        "                        if real_class in {7, 8}:\n",
        "                            mainBifurcations_correct[2] += 1\n",
        "                    else: \n",
        "                        continue\n",
        "\n",
        "                total += 1\n",
        "\n",
        "    print(\"\\n\\nOverall Accuracy is:\", round(correct/total,3))\n",
        "    print(\"------------------------\")\n",
        "\n",
        "    for i in range(len(vessels_count)):\n",
        "        acc = round(vessels_correct[i]/vessels_count[i],3)\n",
        "        string = f\"Accuracy of {vessels_names[i]} is:\"\n",
        "        print(\"{:<30} {:<5}\".format(string, acc))\n",
        "\n",
        "    print(\"------------------------\")\n",
        "    print(f\"No of samples for each vessel in a batch is {vessels_count}\")\n",
        "\n",
        "    mainBifurcations_count[0] = vessels_count[0] + vessels_count[1] + vessels_count[2] #LCA\n",
        "    mainBifurcations_count[1] = vessels_count[3] + vessels_count[4] #LCX\n",
        "    mainBifurcations_count[2] = vessels_count[7] + vessels_count[8] #RCA \n",
        "\n",
        "    mainBifurcations_correct[0] += vessels_correct[0] + vessels_correct[1] + vessels_correct[2]  #LCA\n",
        "    mainBifurcations_correct[1] += vessels_correct[3] + vessels_correct[4]  #LCX\n",
        "    mainBifurcations_correct[2] += vessels_correct[7] + vessels_correct[8]  #RCA\n",
        "\n",
        "    print(\"------------------------\")\n",
        "    print(f\"Bifurcation LCA has accuracy of {round(mainBifurcations_correct[0]/mainBifurcations_count[0], 3)}\")\n",
        "    print(f\"Bifurcation LCX has accuracy of {round(mainBifurcations_correct[1]/mainBifurcations_count[1], 3)}\")\n",
        "    print(f\"Bifurcation RCA has accuracy of {round(mainBifurcations_correct[2]/mainBifurcations_count[2], 3)}\")\n",
        "\n",
        "    # print(test_y)\n",
        "    return true_class_array, predicted_class_array\n",
        "\n",
        "\n",
        "true_class_array, predicted_class_array = test(model)\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}