06.1 - Fancy Softmax Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Lab 6.1: Fancy Softmax Classification\r\n",
    "\r\n",
    "Edited By Steve Ive.\r\n",
    "\r\n",
    "Reference from Seungjae Lee\r\n",
    "\r\n",
    "https://github.com/deeplearningzerotoall/PyTorch/blob/master/lab-06_2_fancy_softmax_classification.ipynb"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Imports"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "source": [
    "import numpy as np\r\n",
    "import torch\r\n",
    "import torch.nn as nn\r\n",
    "import torch.nn.functional as F\r\n",
    "import torch.optim as optim"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "source": [
    "torch.manual_seed(1)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x1cacac79490>"
      ]
     },
     "metadata": {},
     "execution_count": 102
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Cross Entropy Loss with ```torch.nn.functional```\r\n",
    "PyTorch has ```F.log_softmax()```"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "source": [
    "z = torch.rand(3, 5, requires_grad = True)\r\n",
    "pred = F.softmax(z, dim =1)\r\n",
    "\r\n",
    "y = torch.randint(5, (3,)).long()\r\n",
    "\r\n",
    "y_one_hot = torch.zeros_like(pred)\r\n",
    "y_one_hot = y_one_hot.scatter_(1, y.unsqueeze(1), 1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "source": [
    "#Low Level\r\n",
    "torch.log(F.softmax(z, dim = 1))"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tensor([[-1.3301, -1.8084, -1.6846, -1.3530, -2.0584],\n",
       "        [-1.4147, -1.8174, -1.4602, -1.6450, -1.7758],\n",
       "        [-1.5025, -1.6165, -1.4586, -1.8360, -1.6776]], grad_fn=<LogBackward>)"
      ]
     },
     "metadata": {},
     "execution_count": 104
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "source": [
    "#High Level\r\n",
    "F.log_softmax(z, dim = 1)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tensor([[-1.3301, -1.8084, -1.6846, -1.3530, -2.0584],\n",
       "        [-1.4147, -1.8174, -1.4602, -1.6450, -1.7758],\n",
       "        [-1.5025, -1.6165, -1.4586, -1.8360, -1.6776]],\n",
       "       grad_fn=<LogSoftmaxBackward>)"
      ]
     },
     "metadata": {},
     "execution_count": 105
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "PyTorch also has ```F.nll_loss()``` function that computes the negative loss likelihood."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "source": [
    "F.cross_entropy(z, y)"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tensor(1.4689, grad_fn=<NllLossBackward>)"
      ]
     },
     "metadata": {},
     "execution_count": 106
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Data"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "source": [
    "xy = np.loadtxt('data-04-zoo.csv', delimiter=',', dtype =np.float32)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "source": [
    "x_train = torch.FloatTensor(xy[:, 0:-1])\r\n",
    "y_train = torch.LongTensor(xy[:, -1])"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "source": [
    "print(x_train.shape)\r\n",
    "print(y_train.shape)\r\n",
    "print(len(x_train))\r\n",
    "print(x_train[:5])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "torch.Size([101, 16])\n",
      "torch.Size([101])\n",
      "101\n",
      "tensor([[1., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 4., 0., 0., 1.],\n",
      "        [1., 0., 0., 1., 0., 0., 0., 1., 1., 1., 0., 0., 4., 1., 0., 1.],\n",
      "        [0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 1., 0., 1., 0., 0.],\n",
      "        [1., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 4., 0., 0., 1.],\n",
      "        [1., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 4., 1., 0., 1.]])\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "source": [
    "nb_classes = 7\r\n",
    "y_one_hot = torch.zeros((len(x_train), nb_classes))\r\n",
    "y_one_hot = y_one_hot.scatter_(1, y_train.unsqueeze(1), 1)\r\n",
    "# or F.one_hot(y_train)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Training with ```F.cross.entropy```"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "source": [
    "# Model initialize\r\n",
    "W = torch.zeros((16, 7), requires_grad = True)\r\n",
    "b = torch.zeros(1, requires_grad=True)\r\n",
    "\r\n",
    "#Set Optimizer\r\n",
    "optimizer = optim.SGD([W, b], lr=0.1)\r\n",
    "\r\n",
    "nb_epochs = 1000\r\n",
    "\r\n",
    "for epoch in range(nb_epochs):\r\n",
    "\r\n",
    "    #prediction\r\n",
    "    pred = x_train.matmul(W) + b\r\n",
    "\r\n",
    "    #cost\r\n",
    "    cost = F.cross_entropy(pred, y_train)\r\n",
    "\r\n",
    "    #Reduce cost\r\n",
    "    optimizer.zero_grad()\r\n",
    "    cost.backward()\r\n",
    "    optimizer.step()\r\n",
    "\r\n",
    "    if epoch % 100 == 0:\r\n",
    "        accuracy = (F.one_hot(torch.argmax(F.softmax(pred, dim=1), dim=1)) == F.one_hot(y_train)).float().mean()*100\r\n",
    "        print('Epoch {:2d}/{} accuracy: {} cost: {:.6f}'.format(epoch, nb_epochs, accuracy, cost))\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch  0/1000 accuracy: 14.285715103149414 cost: 1.945910\n",
      "Epoch 100/1000 accuracy: 96.3224868774414 cost: 0.471836\n",
      "Epoch 200/1000 accuracy: 98.01980590820312 cost: 0.326327\n",
      "Epoch 300/1000 accuracy: 98.58557891845703 cost: 0.257839\n",
      "Epoch 400/1000 accuracy: 98.58557891845703 cost: 0.215762\n",
      "Epoch 500/1000 accuracy: 98.58557891845703 cost: 0.186603\n",
      "Epoch 600/1000 accuracy: 99.1513442993164 cost: 0.164898\n",
      "Epoch 700/1000 accuracy: 99.4342269897461 cost: 0.147955\n",
      "Epoch 800/1000 accuracy: 100.0 cost: 0.134279\n",
      "Epoch 900/1000 accuracy: 100.0 cost: 0.122962\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## High-level Implementaion with ```nn.Module```"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "source": [
    "class SoftmaxClassifierModel(nn.Module):\r\n",
    "    def __init__(self):\r\n",
    "        super().__init__()\r\n",
    "        self.linear = nn.Linear(16, 7)\r\n",
    "\r\n",
    "    def forward(self, x):\r\n",
    "        return self.linear(x)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "source": [
    "model = SoftmaxClassifierModel()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\r\n",
    "\r\n",
    "nb_epochs = 1000\r\n",
    "\r\n",
    "for epoch in range(nb_epochs):\r\n",
    "\r\n",
    "    #prediction\r\n",
    "    pred = model(x_train)\r\n",
    "\r\n",
    "    #cost\r\n",
    "    cost = F.cross_entropy(pred, y_train)\r\n",
    "\r\n",
    "    #Reduce cost\r\n",
    "    optimizer.zero_grad()\r\n",
    "    cost.backward()\r\n",
    "    optimizer.step()\r\n",
    "\r\n",
    "    if epoch % 100 == 0:\r\n",
    "        accuracy = (F.one_hot(torch.argmax(F.softmax(pred, dim=1), dim =1)) == F.one_hot(y_train)).float().mean()*100\r\n",
    "        print('Epoch {:2d}/{} Accuracy: {:.3f} Cost: {:.6f}'.format(epoch, nb_epochs, accuracy, cost))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch  0/1000 Accuracy: 74.257 Cost: 1.919160\n",
      "Epoch 100/1000 Accuracy: 96.605 Cost: 0.468405\n",
      "Epoch 200/1000 Accuracy: 97.737 Cost: 0.320585\n",
      "Epoch 300/1000 Accuracy: 98.586 Cost: 0.248953\n",
      "Epoch 400/1000 Accuracy: 98.586 Cost: 0.204819\n",
      "Epoch 500/1000 Accuracy: 99.151 Cost: 0.174506\n",
      "Epoch 600/1000 Accuracy: 99.151 Cost: 0.152248\n",
      "Epoch 700/1000 Accuracy: 100.000 Cost: 0.135139\n",
      "Epoch 800/1000 Accuracy: 100.000 Cost: 0.121543\n",
      "Epoch 900/1000 Accuracy: 100.000 Cost: 0.110461\n"
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('buddhalight': conda)"
  },
  "interpreter": {
   "hash": "38ed4d61829b01de31b0fe0651719916120d9f7e023a62cbbfea93b7d24a50a0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}