linearPerceptron.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fwxWcO7fqb-L"
   },
   "source": [
    "# Excercise - Multi-class classification of MNIST using Perceptron"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vcygblmOmQDZ"
   },
   "source": [
    "In binary perceptron, where $\\mathbf{y} \\in \\{-1, +1\\}$, we used to update our weights only for wrongly classified examples.\n",
    "\n",
    "The multi-class perceptron is regarded as a generalization of binary perceptron. Learning through iteration is the same as the perceptron. Weighted inputs are passed through a multiclass signum activation function. If the predicted output label is the same as true label then weights are not updated. However, when predicted output label $\\neq$ true label, then the wrongly classified input example is added to the weights of the correct label and subtracted from the weights of the incorrect label. Effectively, this amounts to ’rewarding’ the correct weight vector, ’punishing’ the misleading, incorrect weight\n",
    "vector, and leaving alone an other weight vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#from utils import plot_decision_boundary, multi_class_signum, get_accuracy, get_prediction\n",
    "#from utils import plot_2D_input_datapoints, generate_gifs, normalize, signum\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import numpy as np\n",
    "import seaborn as sns; sns.set();\n",
    "import pandas as pd\n",
    "import math\n",
    "#import gif\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 223975,
     "status": "ok",
     "timestamp": 1596984132348,
     "user": {
      "displayName": "KARTIK RISHI BHARADWAJ 14BEE0070",
      "photoUrl": "",
      "userId": "12368401133146776355"
     },
     "user_tz": -330
    },
    "id": "gNkGLnbjTY-s"
   },
   "outputs": [],
   "source": [
    "# Setting the seed to ensure reproducibility of experiments\n",
    "np.random.seed(11)\n",
    "\n",
    "# One-hot encoding of target label, Y\n",
    "def one_hot(a):\n",
    "    b = -1 * np.ones((a.size, a.max()+1))\n",
    "    b[np.arange(a.size), a] = 1\n",
    "    return b\n",
    "\n",
    "# Loading digits datasets\n",
    "digits = datasets.load_digits()\n",
    "\n",
    "# One-hot encoding of target label, Y\n",
    "Y = digits.target\n",
    "Y = one_hot(Y)\n",
    "\n",
    "# Adding column of ones to absorb bias b of the hyperplane into X\n",
    "X = digits.data\n",
    "bias_ones = np.ones((len(X), 1))\n",
    "X = np.hstack((X, bias_ones))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 223957,
     "status": "ok",
     "timestamp": 1596984132353,
     "user": {
      "displayName": "KARTIK RISHI BHARADWAJ 14BEE0070",
      "photoUrl": "",
      "userId": "12368401133146776355"
     },
     "user_tz": -330
    },
    "id": "0BPvc5P8KvrM",
    "outputId": "233f09b1-7641-4c60-c21d-74a2264f8bc3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training dataset:  (1257, 65)\n",
      "Validation dataset:  (180, 65)\n",
      "Test dataset:  (360, 65)\n"
     ]
    }
   ],
   "source": [
    "# Train-val-test data\n",
    "X_train_val, X_test, Y_train_val, Y_test = train_test_split(X, Y, shuffle=True, test_size = 0.2)\n",
    "X_train, X_val, Y_train, Y_val = train_test_split(X_train_val, Y_train_val, test_size = 0.12517)\n",
    "\n",
    "print(\"Training dataset: \", X_train.shape)\n",
    "print(\"Validation dataset: \", X_val.shape)\n",
    "print(\"Test dataset: \", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 292
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 223939,
     "status": "ok",
     "timestamp": 1596984132358,
     "user": {
      "displayName": "KARTIK RISHI BHARADWAJ 14BEE0070",
      "photoUrl": "",
      "userId": "12368401133146776355"
     },
     "user_tz": -330
    },
    "id": "QPJZdeDtUfoy",
    "outputId": "66a50417-5c21-4158-f029-20ef755e50f4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAALwElEQVR4nO3d34tc9RnH8c/HNUFrYhaiFTViLJSACN0ECRVF2oRIrBK96EUCFVZa0otWDA2I9qbJPyDpRRFC1ASMEY0GirTWgFlEaLVJXGvMxmJCxAR1/UFI4kWD5unFnJR02XbPrud7Znae9wuGzM7OnOfZ3XzmnDNz5jyOCAHob5d0uwEA5RF0IAGCDiRA0IEECDqQAEEHEuiJoNtebft92x/YfrRwradsj9s+VLLORfVusL3P9mHb79l+uHC9y2y/Zfudqt7mkvWqmgO237b9culaVb3jtt+1PWp7f+Fag7Z32z5ie8z2bQVrLal+pguX07Y3NLLwiOjqRdKApKOSvidprqR3JN1csN6dkpZJOtTSz3etpGXV9fmS/ln457OkedX1OZLelPTDwj/jbyQ9K+nlln6nxyVd1VKtHZJ+UV2fK2mwpboDkj6RdGMTy+uFNfpySR9ExLGIOCfpOUn3lSoWEa9L+rLU8iep93FEHKyun5E0Jun6gvUiIs5WX86pLsWOirK9SNI9kraVqtEttheos2J4UpIi4lxEnGqp/EpJRyPiwyYW1gtBv17SRxd9fUIFg9BNthdLWqrOWrZknQHbo5LGJe2NiJL1tkh6RNL5gjUmCkmv2j5ge33BOjdJ+kzS09WuyTbbVxSsd7G1knY1tbBeCHoKtudJelHShog4XbJWRHwTEUOSFklabvuWEnVs3ytpPCIOlFj+/3FHRCyTdLekX9m+s1CdS9XZzXsiIpZK+kpS0deQJMn2XElrJL3Q1DJ7IegnJd1w0deLqtv6hu056oR8Z0S81FbdajNzn6TVhUrcLmmN7ePq7HKtsP1MoVr/EREnq3/HJe1RZ/evhBOSTly0RbRbneCXdrekgxHxaVML7IWg/13S923fVD2TrZX0xy731BjbVmcfbywiHm+h3tW2B6vrl0taJelIiVoR8VhELIqIxer83V6LiJ+VqHWB7Stsz79wXdJdkoq8gxIRn0j6yPaS6qaVkg6XqDXBOjW42S51Nk26KiK+tv1rSX9R55XGpyLivVL1bO+S9CNJV9k+Iel3EfFkqXrqrPUekPRutd8sSb+NiD8VqnetpB22B9R5In8+Ilp526sl10ja03n+1KWSno2IVwrWe0jSzmoldEzSgwVrXXjyWiXpl40ut3opH0Af64VNdwCFEXQgAYIOJEDQgQQIOpBATwW98OGMXatFPep1u15PBV1Sm7/MVv9w1KNeN+v1WtABFFDkgBnbfX0UzsDAwLQfc/78eV1yycyeV6+77rppP+bs2bOaN2/ejOotXLhw2o/54osvZvQ4STpz5sy0H3P69GldeeWVM6p39OjRGT1utogIT7yt64fAzkbz589vtd7GjRtbrTc8PNxqvZGRkVbr3X///a3W6wVsugMJEHQgAYIOJEDQgQQIOpAAQQcSIOhAAgQdSKBW0NscmQSgeVMGvTrJ4B/UOQXtzZLW2b65dGMAmlNnjd7qyCQAzasT9DQjk4B+1diHWqoPyrf9mV0ANdQJeq2RSRGxVdJWqf8/pgrMNnU23ft6ZBKQwZRr9LZHJgFoXq199GpOWKlZYQAK48g4IAGCDiRA0IEECDqQAEEHEiDoQAIEHUiAoAMJMKllBrZv395qvfvua/dTwZs3b261XtuTYdqu1/b/l8mwRgcSIOhAAgQdSICgAwkQdCABgg4kQNCBBAg6kABBBxIg6EACdUYyPWV73PahNhoC0Lw6a/TtklYX7gNAQVMGPSJel/RlC70AKIR9dCABZq8BCTQWdGavAb2LTXcggTpvr+2S9FdJS2yfsP3z8m0BaFKdIYvr2mgEQDlsugMJEHQgAYIOJEDQgQQIOpAAQQcSIOhAAgQdSKAvZq8tXry41Xptz0LbsWNHq/U2bdrUar3BwcFW6w0NDbVarxewRgcSIOhAAgQdSICgAwkQdCABgg4kQNCBBAg6kABBBxIg6EACdU4OeYPtfbYP237P9sNtNAagOXWOdf9a0saIOGh7vqQDtvdGxOHCvQFoSJ3Zax9HxMHq+hlJY5KuL90YgOZMax/d9mJJSyW9WaIZAGXU/piq7XmSXpS0ISJOT/J9Zq8BPapW0G3PUSfkOyPipcnuw+w1oHfVedXdkp6UNBYRj5dvCUDT6uyj3y7pAUkrbI9Wl58U7gtAg+rMXntDklvoBUAhHBkHJEDQgQQIOpAAQQcSIOhAAgQdSICgAwkQdCCBvpi9durUqW63UNT27du73UJR/f736wWs0YEECDqQAEEHEiDoQAIEHUiAoAMJEHQgAYIOJEDQgQQIOpBAnbPAXmb7LdvvVLPXNrfRGIDm1DnW/V+SVkTE2er87m/Y/nNE/K1wbwAaUucssCHpbPXlnOrCgAZgFqm1j257wPaopHFJeyOC2WvALFIr6BHxTUQMSVokabntWybex/Z62/tt72+6SQDfzrRedY+IU5L2SVo9yfe2RsStEXFrU80BaEadV92vtj1YXb9c0ipJR0o3BqA5dV51v1bSDtsD6jwxPB8RL5dtC0CT6rzq/g9JS1voBUAhHBkHJEDQgQQIOpAAQQcSIOhAAgQdSICgAwkQdCCBvpi9NjQ01O0WgJ7GGh1IgKADCRB0IAGCDiRA0IEECDqQAEEHEiDoQAIEHUiAoAMJ1A56NcThbducGBKYZaazRn9Y0lipRgCUU3ck0yJJ90jaVrYdACXUXaNvkfSIpPMFewFQSJ1JLfdKGo+IA1Pcj9lrQI+qs0a/XdIa28clPSdphe1nJt6J2WtA75oy6BHxWEQsiojFktZKei0ifla8MwCN4X10IIFpnUoqIkYkjRTpBEAxrNGBBAg6kABBBxIg6EACBB1IgKADCRB0IAGCDiTQF7PXRkdHu91CUQsWLGi13uDgYKv12p6dt2nTplbr9QLW6EACBB1IgKADCRB0IAGCDiRA0IEECDqQAEEHEiDoQAIEHUig1iGw1amez0j6RtLXnNIZmF2mc6z7jyPi82KdACiGTXcggbpBD0mv2j5ge33JhgA0r+6m+x0RcdL2dyXttX0kIl6/+A7VEwBPAkAPqrVGj4iT1b/jkvZIWj7JfZi9BvSoOtNUr7A9/8J1SXdJOlS6MQDNqbPpfo2kPbYv3P/ZiHilaFcAGjVl0CPimKQftNALgEJ4ew1IgKADCRB0IAGCDiRA0IEECDqQAEEHEiDoQAKOiOYXaje/0B4yMjLS7RaKOn78eLdbKGp4eLjbLRQVEZ54G2t0IAGCDiRA0IEECDqQAEEHEiDoQAIEHUiAoAMJEHQgAYIOJFAr6LYHbe+2fcT2mO3bSjcGoDl1Bzj8XtIrEfFT23MlfadgTwAaNmXQbS+QdKekYUmKiHOSzpVtC0CT6my63yTpM0lP237b9rZqkMN/sb3e9n7b+xvvEsC3Uifol0paJumJiFgq6StJj068EyOZgN5VJ+gnJJ2IiDerr3erE3wAs8SUQY+ITyR9ZHtJddNKSYeLdgWgUXVfdX9I0s7qFfdjkh4s1xKAptUKekSMSmLfG5ilODIOSICgAwkQdCABgg4kQNCBBAg6kABBBxIg6EACzF6bgcHBwVbrbdmypdV6Q0NDrdZrexba6Ohoq/Xaxuw1ICmCDiRA0IEECDqQAEEHEiDoQAIEHUiAoAMJEHQggSmDbnuJ7dGLLqdtb2ijOQDNmPKccRHxvqQhSbI9IOmkpD2F+wLQoOluuq+UdDQiPizRDIAyphv0tZJ2lWgEQDm1g16d032NpBf+x/eZvQb0qLoDHCTpbkkHI+LTyb4ZEVslbZX6/2OqwGwznU33dWKzHZiVagW9GpO8StJLZdsBUELdkUxfSVpYuBcAhXBkHJAAQQcSIOhAAgQdSICgAwkQdCABgg4kQNCBBAg6kECp2WufSZrJZ9avkvR5w+30Qi3qUa+tejdGxNUTbywS9JmyvT8ibu23WtSjXrfrsekOJEDQgQR6Lehb+7QW9ajX1Xo9tY8OoIxeW6MDKICgAwkQdCABgg4kQNCBBP4NCzV9vYiL0lkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.reset_orig();\n",
    "\n",
    "plt.gray()\n",
    "plt.matshow(digits.images[10])\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For digit 0 the number of misclassified samples after training is 0\n",
      "For digit 1 the number of misclassified samples after training is 16\n",
      "For digit 2 the number of misclassified samples after training is 0\n",
      "For digit 3 the number of misclassified samples after training is 5\n",
      "For digit 4 the number of misclassified samples after training is 0\n",
      "For digit 5 the number of misclassified samples after training is 0\n",
      "For digit 6 the number of misclassified samples after training is 0\n",
      "For digit 7 the number of misclassified samples after training is 0\n",
      "For digit 8 the number of misclassified samples after training is 77\n",
      "For digit 9 the number of misclassified samples after training is 49\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def core_algo(X_train,Y_train,epochs,lr):\n",
    "    \n",
    "    start = 1\n",
    "    W = np.zeros((X_train.shape[1],1))\n",
    "\n",
    "    for i in range(epochs):\n",
    "        error_count = 0\n",
    "        total_sum = 0\n",
    "        for x,y in zip(X_train,Y_train):\n",
    "\n",
    "            x = np.reshape(x,(x.shape[0],1))\n",
    "            y_pred = W.T@x\n",
    "           \n",
    "            if (y_pred*y)<=0:\n",
    "                \n",
    "                error_count+=1\n",
    "                total_sum+=((lr*y*x))\n",
    "\n",
    "        start+=1\n",
    "        W += total_sum\n",
    "    return W,error_count\n",
    "\n",
    "\n",
    "\n",
    "def train_algo(X_train,Y_train,epochs,lr):\n",
    "    \n",
    "    W_final = np.zeros((X_train.shape[1],Y_train.shape[1]))\n",
    "    for classie in range(Y_train.shape[1]):\n",
    "\n",
    "        yee = Y_train[:,classie]\n",
    "\n",
    "        W,error_count = core_algo(X_train,yee,epochs,lr)\n",
    "        print(\"For digit %d the number of misclassified samples after training is %d\"%(classie,error_count))\n",
    "        W_final[:,classie] = W[:,0]\n",
    "    return W_final\n",
    "        \n",
    "Weightsies = train_algo(X_train,Y_train,1000,0.1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy for train is:\n",
      "90.69212410501193\n",
      "The accuracy for validation is:\n",
      "87.22222222222223\n",
      "The accuracy for test is:\n",
      "86.38888888888889\n"
     ]
    }
   ],
   "source": [
    "def prediction(X,Y,W):\n",
    "    \n",
    "#     print(W.shape)\n",
    "    no_of_digits = Y.shape[1]\n",
    "    predicted_classes = np.zeros((X.shape[0],1))\n",
    "    \n",
    "    for i in range(X.shape[0]):\n",
    "        \n",
    "        for digit in range(no_of_digits):\n",
    "            \n",
    "            x = np.reshape(X[i,:],(X[i,:].shape[0],1))\n",
    "\n",
    "            W_slice = W[:,digit]\n",
    "            predict_digit = W_slice.T@x\n",
    "\n",
    "            if predict_digit>0:\n",
    "                \n",
    "                predicted_classes[i,0] = digit\n",
    "                break\n",
    "                \n",
    "    return predicted_classes\n",
    "\n",
    "predicted_val = prediction(X_val,Y_val,Weightsies)\n",
    "predicted_test = prediction(X_test,Y_test,Weightsies)\n",
    "predicted_train = prediction(X_train,Y_train,Weightsies)\n",
    "\n",
    "def accuracy_calculator(preds,y):\n",
    "    \n",
    "    error = 0\n",
    "    no_of_samples = y.shape[0]\n",
    "    \n",
    "    for i in range(no_of_samples):\n",
    "        \n",
    "        y_sample = y[i,:]\n",
    "        pred = preds[i]\n",
    "        \n",
    "        if y_sample[int(pred)]!=1:\n",
    "            \n",
    "            error+=1\n",
    "            \n",
    "    accuracy = 1 - (error/no_of_samples)\n",
    "    return accuracy\n",
    "\n",
    "accuracy_val = accuracy_calculator(predicted_val,Y_val)\n",
    "accuracy_test = accuracy_calculator(predicted_test,Y_test)\n",
    "accuracy_train = accuracy_calculator(predicted_train,Y_train)\n",
    "\n",
    "print(\"The accuracy for train is:\")\n",
    "print(accuracy_train*100)\n",
    "print(\"The accuracy for validation is:\")\n",
    "print(accuracy_val*100)\n",
    "print(\"The accuracy for test is:\")\n",
    "print(accuracy_test*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "ZQQfFFOrqST3"
   ],
   "name": "LinearPerceptron_draft4.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}