09.3 - About Relu Activation with MNIST Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Lab 9.3: About Relu Activation with MNIST Classifier\r\n",
    "\r\n",
    "Edited By Steve Ive\r\n",
    "\r\n",
    "Here, we are going to train our neural network with new activation function 'relu' including multiple layers.\r\n",
    "You can learn more about the activation function in \"09.2 About Activations\".\r\n",
    "\r\n",
    "Reference from\r\n",
    "\r\n",
    "https://github.com/deeplearningzerotoall/PyTorch/blob/master/lab-09_2_mnist_nn.ipynb"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "## Imports"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "import torch\r\n",
    "import torch.nn as nn\r\n",
    "import torch.nn.functional as F\r\n",
    "import torch.optim as optim\r\n",
    "import torchvision.datasets as datasets\r\n",
    "import torchvision.transforms as transforms\r\n",
    "import random"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\r\n",
    "random.seed(1)\r\n",
    "torch.manual_seed(1)\r\n",
    "\r\n",
    "if device == 'cuda':\r\n",
    "    torch.cuda.manual_seed_all(1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Set Hyperparameters"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "learning_rate = 0.001\r\n",
    "training_epochs = 15\r\n",
    "batch_size = 100"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Load MNIST Dataset"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "mnist_train = datasets.MNIST(root='MNIST_data/',\r\n",
    "                             transform=transforms.ToTensor(),\r\n",
    "                             download=True,\r\n",
    "                             train=True)\r\n",
    "mnist_test = datasets.MNIST(root='MNIST_data/',\r\n",
    "                            transform=transforms.ToTensor(),\r\n",
    "                            download=True,\r\n",
    "                            train=False)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\buddhalight\\envs\\buddhalight\\lib\\site-packages\\torchvision\\datasets\\mnist.py:498: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  ..\\torch\\csrc\\utils\\tensor_numpy.cpp:180.)\n",
      "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "data_loader = torch.utils.data.DataLoader(dataset=mnist_train, shuffle=True, drop_last=True, batch_size=batch_size)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Define Model"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "source": [
    "class Relu_MNIST_Classifier(nn.Module):\r\n",
    "    def __init__(self):\r\n",
    "        super().__init__()\r\n",
    "        self.sq = nn.Sequential(\r\n",
    "            nn.Linear(784, 256),\r\n",
    "            nn.ReLU(),\r\n",
    "            nn.Linear(256, 256),\r\n",
    "            nn.ReLU(),\r\n",
    "            nn.Linear(256, 10),\r\n",
    "        )\r\n",
    "        self.weighter()\r\n",
    "\r\n",
    "    def forward(self, x):\r\n",
    "        return self.sq(x)\r\n",
    "\r\n",
    "    def weighter(self):\r\n",
    "        for index, layer in enumerate(self.sq):\r\n",
    "            if index != 1 and index != 3:\r\n",
    "                nn.init.normal_(layer.weight)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "source": [
    "model = Relu_MNIST_Classifier().to(device)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Linear(in_features=784, out_features=256, bias=True)\n",
      "Linear(in_features=256, out_features=256, bias=True)\n",
      "Linear(in_features=256, out_features=10, bias=True)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Sometimes optimizer cannot find paramter error.\r\n",
    "Just rewriting the model class's init part, solved.\r\n",
    "seems to be problem of initializing parameters."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Train Model"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "total_batch = len(data_loader)\r\n",
    "\r\n",
    "for epoch in range(training_epochs):\r\n",
    "\r\n",
    "    avg_cost = 0\r\n",
    "\r\n",
    "    for X, Y in data_loader:\r\n",
    "\r\n",
    "        X = X.view(-1, 28 * 28).to(device)\r\n",
    "        Y = Y.to(device)\r\n",
    "\r\n",
    "        #prediction\r\n",
    "        pred = model(X)\r\n",
    "        \r\n",
    "        #cost\r\n",
    "        cost = F.cross_entropy(pred, Y)\r\n",
    "\r\n",
    "        #Reduce cost\r\n",
    "        optimizer.zero_grad()\r\n",
    "        cost.backward()\r\n",
    "        optimizer.step()\r\n",
    "\r\n",
    "        avg_cost += cost\r\n",
    "        \r\n",
    "    avg_cost = avg_cost / total_batch\r\n",
    "    print('Epoch: {:d} / 15, Cost: {:.6f}'.format(epoch + 1, cost.item()))\r\n",
    "\r\n",
    "print('Learning Finished')"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch: 1 / 15, Cost: 0.134539\n",
      "Epoch: 2 / 15, Cost: 0.162091\n",
      "Epoch: 3 / 15, Cost: 0.041155\n",
      "Epoch: 4 / 15, Cost: 0.092741\n",
      "Epoch: 5 / 15, Cost: 0.007326\n",
      "Epoch: 6 / 15, Cost: 0.108856\n",
      "Epoch: 7 / 15, Cost: 0.032949\n",
      "Epoch: 8 / 15, Cost: 0.004769\n",
      "Epoch: 9 / 15, Cost: 0.024743\n",
      "Epoch: 10 / 15, Cost: 0.010422\n",
      "Epoch: 11 / 15, Cost: 0.010646\n",
      "Epoch: 12 / 15, Cost: 0.009288\n",
      "Epoch: 13 / 15, Cost: 0.002405\n",
      "Epoch: 14 / 15, Cost: 0.001527\n",
      "Epoch: 15 / 15, Cost: 0.020818\n",
      "Learning Finished\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "source": [
    "#Test the model using test sets\r\n",
    "\r\n",
    "with torch.no_grad():\r\n",
    "    X_test = mnist_test.data.view(-1, 28 * 28).float().to(device)\r\n",
    "    Y_test = mnist_test.targets.to(device)\r\n",
    "\r\n",
    "    #prediction\r\n",
    "    pred = model(X_test)\r\n",
    "\r\n",
    "    correct_prediction = torch.argmax(pred, 1) == Y_test\r\n",
    "    accuracy = correct_prediction.float().mean()\r\n",
    "    print('Accuracy: {:.9f}'.format(accuracy.item()))\r\n",
    "\r\n",
    "    r = random.randint(0, len(mnist_test) - 1)\r\n",
    "\r\n",
    "    X_single_test = X_test[r]\r\n",
    "    Y_single_test = Y_test[r]\r\n",
    "\r\n",
    "    print('Label: {}'.format(Y_single_test))\r\n",
    "    print('Prediction: {}'.format(torch.argmax(model(X_single_test))))\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Accuracy: 0.979900002\n",
      "Label: 0\n",
      "Prediction: 0\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('buddhalight': conda)"
  },
  "interpreter": {
   "hash": "38ed4d61829b01de31b0fe0651719916120d9f7e023a62cbbfea93b7d24a50a0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}