Vanishing_Gradient_Train_layers_separately.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.14",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "none",
      "dataSources": [],
      "dockerImageVersionId": 30787,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": false
    },
    "colab": {
      "name": "Vanishing_Gradient_Train_layers_separately",
      "provenance": []
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the read-only \"../input/\" directory\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
        "\n",
        "import os\n",
        "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirname, filename))\n",
        "\n",
        "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n",
        "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
      ],
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:24.158356Z",
          "iopub.execute_input": "2024-12-01T13:02:24.159138Z",
          "iopub.status.idle": "2024-12-01T13:02:24.487872Z",
          "shell.execute_reply.started": "2024-12-01T13:02:24.159102Z",
          "shell.execute_reply": "2024-12-01T13:02:24.487145Z"
        },
        "id": "WVp1Xxpop32c"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torch import nn\n",
        "from torch.utils.data import Dataset, DataLoader, random_split\n",
        "from torchvision.datasets import FashionMNIST\n",
        "import os\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:24.489448Z",
          "iopub.execute_input": "2024-12-01T13:02:24.489799Z",
          "iopub.status.idle": "2024-12-01T13:02:28.682458Z",
          "shell.execute_reply.started": "2024-12-01T13:02:24.489773Z",
          "shell.execute_reply": "2024-12-01T13:02:28.681773Z"
        },
        "id": "NBWuDrjWp32d"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "def set_seed(seed):\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.cuda.manual_seed_all(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "\n",
        "SEED = 42\n",
        "set_seed(SEED)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:28.683581Z",
          "iopub.execute_input": "2024-12-01T13:02:28.68408Z",
          "iopub.status.idle": "2024-12-01T13:02:28.695641Z",
          "shell.execute_reply.started": "2024-12-01T13:02:28.684039Z",
          "shell.execute_reply": "2024-12-01T13:02:28.694981Z"
        },
        "id": "iXhl5_Kzp32d"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "root_dir = '/kaggle/working/'\n",
        "train_dataset = FashionMNIST(os.path.join(root_dir, './data'), train= True, download = True, transform = transforms.ToTensor())\n",
        "test_dataset = FashionMNIST(os.path.join(root_dir, './data'), train= False, download = True, transform = transforms.ToTensor())"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:28.697243Z",
          "iopub.execute_input": "2024-12-01T13:02:28.697484Z",
          "iopub.status.idle": "2024-12-01T13:02:33.474734Z",
          "shell.execute_reply.started": "2024-12-01T13:02:28.697462Z",
          "shell.execute_reply": "2024-12-01T13:02:33.473847Z"
        },
        "id": "pJqQlrL2p32d"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 512\n",
        "train_ratio = 0.9\n",
        "\n",
        "train_size = int(len(train_dataset) * 0.9)\n",
        "val_size = len(train_dataset) - train_size\n",
        "\n",
        "train_subset, val_subset = random_split(train_dataset,[train_size, val_size])\n",
        "\n",
        "train_loader = DataLoader(train_subset, batch_size = batch_size, shuffle=True)\n",
        "val_loader = DataLoader(val_subset, batch_size = batch_size, shuffle=False)\n",
        "test_loader = DataLoader(test_dataset, batch_size = batch_size, shuffle=False)\n",
        "\n",
        "print(f\"Train size: {len(train_subset)}\")\n",
        "print(f\"Validation size: {len(val_subset)}\")\n",
        "print(f\"Test size: {len(test_dataset)}\")"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:33.475738Z",
          "iopub.execute_input": "2024-12-01T13:02:33.476Z",
          "iopub.status.idle": "2024-12-01T13:02:33.499242Z",
          "shell.execute_reply.started": "2024-12-01T13:02:33.475974Z",
          "shell.execute_reply": "2024-12-01T13:02:33.49835Z"
        },
        "id": "DJQndXQap32e"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "class MLP_1layer(nn.Module):\n",
        "    def __init__(self, input_dims, hidden_dims):\n",
        "        super(MLP_1layer, self).__init__()\n",
        "        self.layer1 = nn.Linear(input_dims, hidden_dims)\n",
        "\n",
        "        for module in self.modules():\n",
        "            if isinstance(module, nn.Linear):\n",
        "                nn.init.normal_(module.weight, mean=0.0, std =0.05)\n",
        "                nn.init.constant_(module.bias, 0)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = nn.Flatten()(x)\n",
        "        x = self.layer1(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "class MLP_2layer(nn.Module):\n",
        "    def __init__(self, input_dims, hidden_dims):\n",
        "        super(MLP_2layer, self).__init__()\n",
        "        self.layer1 = nn.Linear(input_dims, hidden_dims)\n",
        "        self.layer2 = nn.Linear(hidden_dims, hidden_dims)\n",
        "\n",
        "        for module in self.modules():\n",
        "            if isinstance(module, nn.Linear):\n",
        "                nn.init.normal_(module.weight, mean=0.0, std =0.05)\n",
        "                nn.init.constant_(module.bias, 0)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = nn.Flatten()(x)\n",
        "        x = self.layer1(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer2(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "        return x\n",
        "\n",
        "def compute_accuracy(preds, labels):\n",
        "    acc = (torch.argmax(preds, dim = 1) == labels).sum()\n",
        "    return acc"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:33.500308Z",
          "iopub.execute_input": "2024-12-01T13:02:33.500563Z",
          "iopub.status.idle": "2024-12-01T13:02:33.508724Z",
          "shell.execute_reply.started": "2024-12-01T13:02:33.500539Z",
          "shell.execute_reply": "2024-12-01T13:02:33.507953Z"
        },
        "id": "Ll3oo5m8p32e"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "input_dims = 784\n",
        "hidden_dims = 128\n",
        "output_dims = 10\n",
        "lr = 1e-2\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "sub_model_1 = MLP_2layer(input_dims, hidden_dims)\n",
        "sub_model_2 = MLP_2layer(hidden_dims, hidden_dims)\n",
        "sub_model_3 = MLP_2layer(hidden_dims, hidden_dims)\n",
        "sub_model_4 = MLP_1layer(hidden_dims, hidden_dims)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:33.510131Z",
          "iopub.execute_input": "2024-12-01T13:02:33.510532Z",
          "iopub.status.idle": "2024-12-01T13:02:33.537293Z",
          "shell.execute_reply.started": "2024-12-01T13:02:33.510481Z",
          "shell.execute_reply": "2024-12-01T13:02:33.536638Z"
        },
        "id": "YTONpv2Tp32e"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 100\n",
        "train_loss_lst = []\n",
        "train_acc_lst = []\n",
        "val_loss_lst = []\n",
        "val_acc_lst = []\n",
        "\n",
        "# 1st Train\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "\n",
        "# 2nd Train\n",
        "for param in sub_model_1.parameters():\n",
        "    param.require_grad = False\n",
        "\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    sub_model_2,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "# 3rd Train\n",
        "for param in sub_model_1.parameters():\n",
        "    param.require_grad = True\n",
        "\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    sub_model_2,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "\n",
        "# 4th Train\n",
        "for param in sub_model_1.parameters():\n",
        "    param.require_grad = False\n",
        "for param in sub_model_2.parameters():\n",
        "    param.require_grad = False\n",
        "\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    sub_model_2,\n",
        "    sub_model_3,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "# 5th Train\n",
        "for param in sub_model_1.parameters():\n",
        "    param.require_grad = True\n",
        "for param in sub_model_2.parameters():\n",
        "    param.require_grad = True\n",
        "\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    sub_model_2,\n",
        "    sub_model_3,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "# 6th Train\n",
        "for param in sub_model_1.parameters():\n",
        "    param.require_grad = False\n",
        "for param in sub_model_2.parameters():\n",
        "    param.require_grad = False\n",
        "for param in sub_model_3.parameters():\n",
        "    param.require_grad = False\n",
        "\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    sub_model_2,\n",
        "    sub_model_3,\n",
        "    sub_model_4,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "# 7th Train\n",
        "for param in sub_model_1.parameters():\n",
        "    param.require_grad = True\n",
        "for param in sub_model_2.parameters():\n",
        "    param.require_grad = True\n",
        "for param in sub_model_3.parameters():\n",
        "    param.require_grad = True\n",
        "\n",
        "model = nn.Sequential(\n",
        "    sub_model_1,\n",
        "    sub_model_2,\n",
        "    sub_model_3,\n",
        "    sub_model_4,\n",
        "    nn.Linear(hidden_dims, output_dims)\n",
        ").to(device)\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        train_loss += loss\n",
        "        train_acc += compute_accuracy(preds, y_train)\n",
        "        count += len(y_train)\n",
        "\n",
        "    train_loss /= len(train_loader)\n",
        "    train_loss_lst.append(train_loss.detach().cpu().numpy())\n",
        "    train_acc /= count\n",
        "    train_acc_lst.append(train_acc.detach().cpu().numpy())\n",
        "\n",
        "\n",
        "\n",
        "    val_loss = 0.0\n",
        "    val_acc = 0.0\n",
        "    count = 0\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for X_val, y_val in val_loader:\n",
        "            X_val, y_val = X_val.to(device), y_val.to(device)\n",
        "            preds = model.forward(X_val)\n",
        "            loss = criterion(preds, y_val)\n",
        "            val_loss += loss\n",
        "            val_acc += compute_accuracy(preds, y_val)\n",
        "            count += len(y_val)\n",
        "\n",
        "    val_loss /= len(test_loader)\n",
        "    val_loss_lst.append(val_loss.detach().cpu().numpy())\n",
        "    val_acc /= count\n",
        "    val_acc_lst.append(val_acc.detach().cpu().numpy())\n",
        "\n",
        "    print(f\"EPOCH {epoch + 1}/{epochs}, Train_Loss: {train_loss:.4f}, Train_Acc: {train_acc:.4f}, Validation Loss: {val_loss:.4f}, Val_Acc: {val_acc:.4f}\")"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-01T13:02:33.538378Z",
          "iopub.execute_input": "2024-12-01T13:02:33.538652Z"
        },
        "id": "OM-kO7Nop32f"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(2, 2, figsize=(12, 10))\n",
        "ax[0, 0].plot(train_loss_lst, color='green')\n",
        "ax[0, 0].set(xlabel='Epoch', ylabel='Loss')\n",
        "ax[0, 0].set_title('Training Loss')\n",
        "\n",
        "ax[0, 1].plot(val_loss_lst, color='orange')\n",
        "ax[0, 1].set(xlabel='Epoch', ylabel='Loss')\n",
        "ax[0, 1].set_title('Validation Loss')\n",
        "\n",
        "ax[1, 0].plot(train_acc_lst, color='green')\n",
        "ax[1, 0].set(xlabel='Epoch', ylabel='Accuracy')\n",
        "ax[1, 0].set_title('Training Accuracy')\n",
        "\n",
        "ax[1, 1].plot(val_acc_lst, color='orange')\n",
        "ax[1, 1].set(xlabel='Epoch', ylabel='Accuracy')\n",
        "ax[1, 1].set_title('Validation Accuracy')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "trusted": true,
        "id": "iFRxXEomp32f"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "val_target = []\n",
        "val_predict = []\n",
        "\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "    for X_val, y_val in val_loader:\n",
        "        X_val = X_val.to(device)\n",
        "        y_val = y_val.to(device)\n",
        "        outputs = model(X_val)\n",
        "\n",
        "        val_predict.append(outputs.cpu())\n",
        "        val_target.append(y_val.cpu())\n",
        "\n",
        "    val_predict = torch.cat(val_predict)\n",
        "    val_target = torch.cat(val_target)\n",
        "    val_acc = (torch.argmax(val_predict, 1) == val_target).sum().item() / len(val_target)\n",
        "\n",
        "    print('Evaluation on val set:')\n",
        "    print(f'Accuracy: {val_acc}')"
      ],
      "metadata": {
        "trusted": true,
        "id": "r0XRAuqep32f"
      },
      "outputs": [],
      "execution_count": null
    }
  ]
}