mnist dataset using functional api and conv net.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "958a130a",
   "metadata": {},
   "source": [
    "In this notebook I created a deep convolutional neural network using functional API of keras module. My submission to Digit \n",
    "\n",
    "Recognizer got a score of 99.025% which is improved as compared to my previous submission using deep neural network and \n",
    "\n",
    "sequential api. Lets see how can we create a convolutional layers which is building block of Computer Vision.\n",
    "\n",
    "PROBLEM STATEMENT:\n",
    "\n",
    "We have a training set consisting of gray scale images of pixels size (28,28,1).There are hand written images in the training \n",
    "\n",
    "data and we have to identify its label using our machine learning model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3dbe9608",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing required libraries\n",
    "import numpy as np \n",
    "import pandas as pd #For loading csv files\n",
    "from tensorflow import keras  # Keras module for creating our CNN\n",
    "from tensorflow.keras import layers #layers to add in our neural net\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt #Plotting library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "24083d83",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eceaeee2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n",
      "0           0       0       0       0       0       0       0       0       0   \n",
      "1           0       0       0       0       0       0       0       0       0   \n",
      "2           0       0       0       0       0       0       0       0       0   \n",
      "3           0       0       0       0       0       0       0       0       0   \n",
      "4           0       0       0       0       0       0       0       0       0   \n",
      "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
      "41995       0       0       0       0       0       0       0       0       0   \n",
      "41996       0       0       0       0       0       0       0       0       0   \n",
      "41997       0       0       0       0       0       0       0       0       0   \n",
      "41998       0       0       0       0       0       0       0       0       0   \n",
      "41999       0       0       0       0       0       0       0       0       0   \n",
      "\n",
      "       pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
      "0           0  ...         0         0         0         0         0   \n",
      "1           0  ...         0         0         0         0         0   \n",
      "2           0  ...         0         0         0         0         0   \n",
      "3           0  ...         0         0         0         0         0   \n",
      "4           0  ...         0         0         0         0         0   \n",
      "...       ...  ...       ...       ...       ...       ...       ...   \n",
      "41995       0  ...         0         0         0         0         0   \n",
      "41996       0  ...         0         0         0         0         0   \n",
      "41997       0  ...         0         0         0         0         0   \n",
      "41998       0  ...         0         0         0         0         0   \n",
      "41999       0  ...         0         0         0         0         0   \n",
      "\n",
      "       pixel779  pixel780  pixel781  pixel782  pixel783  \n",
      "0             0         0         0         0         0  \n",
      "1             0         0         0         0         0  \n",
      "2             0         0         0         0         0  \n",
      "3             0         0         0         0         0  \n",
      "4             0         0         0         0         0  \n",
      "...         ...       ...       ...       ...       ...  \n",
      "41995         0         0         0         0         0  \n",
      "41996         0         0         0         0         0  \n",
      "41997         0         0         0         0         0  \n",
      "41998         0         0         0         0         0  \n",
      "41999         0         0         0         0         0  \n",
      "\n",
      "[42000 rows x 784 columns]\n"
     ]
    }
   ],
   "source": [
    "y = train['label']\n",
    "train = train.drop(['label'],axis =1) # Dropping the label of training data so as to use this to create mapping\n",
    "print(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "07dfd3a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([42000   784], shape=(2,), dtype=int32)\n",
      "tf.Tensor([28000   784], shape=(2,), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "X = tf.convert_to_tensor(train)\n",
    "\n",
    "s = tf.shape(X)\n",
    "\n",
    "y = tf.convert_to_tensor(y)\n",
    "Xtest =tf.convert_to_tensor(test)\n",
    "w = tf.shape(Xtest)\n",
    "print(s)\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "92af674d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x203cbace220>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPLElEQVR4nO3df7BU5X3H8c/n4gUF1ICKIlKVhEm0GeOPW9BoWxymFk0ddCZ2wiSWtk5JE3U0Y9oYM6m242ScxGgTa0yuSsTGmHFijEzqtDKEibG11CtShRADtRiIBLRUxcTABb794x7tFe8+e9lzds/q837N3Nnd892z58sOn3v27nPOeRwRAvDO11N3AwA6g7ADmSDsQCYIO5AJwg5k4oBObmysx8WBmtDJTQJZ+Y1+pV2x0yPVSoXd9jxJX5E0RtIdEXFD6vkHaoJme26ZTQJIWBnLG9Za/hhve4ykWyWdK+lESQtsn9jq6wForzJ/s8+StCEino2IXZK+I2l+NW0BqFqZsE+TtGnY483Fsjexvcj2gO2BQe0ssTkAZZQJ+0hfArzl2NuI6I+Ivojo69W4EpsDUEaZsG+WNH3Y42MkPV+uHQDtUibsj0uaaft422MlfUTS0mraAlC1lofeImK37csk/YuGht4WR8TayjoDUKlS4+wR8ZCkhyrqBUAbcbgskAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSi1JTNtjdK2iFpj6TdEdFXRVMAqlcq7IWzI+LFCl4HQBvxMR7IRNmwh6SHbT9he9FIT7C9yPaA7YFB7Sy5OQCtKvsx/syIeN72FEnLbP80Ih4Z/oSI6JfUL0mHeHKU3B6AFpXas0fE88XtNkkPSJpVRVMAqtdy2G1PsH3w6/clnSNpTVWNAahWmY/xR0p6wPbrr/PtiPjnSrpCZXrGj0/Xjzyi1OtvunBasv7EVbeUev0yej2mYW3eTz+UXHfP305J1nt+9GRLPdWp5bBHxLOSPlBhLwDaiKE3IBOEHcgEYQcyQdiBTBB2IBNVnAiDmo05YWbD2vj+/02ue8+M75badk+T/cVe7S31+mUMJo7XfPC930+uu+LOicn6Vz90frK+55kNyXod2LMDmSDsQCYIO5AJwg5kgrADmSDsQCYIO5AJxtnfBnzabyfrG/6q8amcT8/4dtXtdMyK19Jj3X9z/Z8n65++pvG/ff6E9DVSzz7o1WT90k8cnqy/50rG2QHUhLADmSDsQCYIO5AJwg5kgrADmSDsQCYYZ+8CLy46I1m/9ep/SNZPGVffOePttGLHCcn64d//SbK++E/Oalib3+R89mbGvOZS69eBPTuQCcIOZIKwA5kg7EAmCDuQCcIOZIKwA5lgnL0D4oz0ZLf3fu7GZP34Aw5M1t+Zo+zS5Yc9mqzP+fynk/UL37WyynbeZM/037Tttdul6Z7d9mLb22yvGbZssu1lttcXt5Pa2yaAskbzMf4uSfP2WXa1pOURMVPS8uIxgC7WNOwR8Yik7fssni9pSXF/iaQLqm0LQNVa/YLuyIjYIknF7ZRGT7S9yPaA7YFB7WxxcwDKavu38RHRHxF9EdHXq3Ht3hyABloN+1bbUyWpuN1WXUsA2qHVsC+VtLC4v1DSg9W0A6Bdmo6z275X0hxJh9veLOlaSTdIus/2JZJ+LumidjbZ7XrGj0/W//COHyXrzcbRe934uvBSeh7ysv5jZ/q87U2DhyXr31yYmMf8359Krrv5sx9M1tdddkuynnrfBiO9n7v+xZOS9fd99oVkfXeyWo+mYY+IBQ1KcyvuBUAbcbgskAnCDmSCsAOZIOxAJgg7kAlOca1Az1ENjxaWJE3vXZOs721ykmqzobVm66fc8fKMZP2huenpondv+WWTLTQeXus56X3JNS+/OH34Rpn3bemv0idqPvKZ9LDf2E2PJ+vdiD07kAnCDmSCsAOZIOxAJgg7kAnCDmSCsAOZYJy9Aruf3ZisX9f/sWT9d6/4UrI+qSd9CmwZd9/wR8n6u7Y8lqw3O7335fMbnyo65+p/S677Z4duTNabOfvpxmdeH/rJ9Bj92GfffuPozbBnBzJB2IFMEHYgE4QdyARhBzJB2IFMEHYgE45o43WI93GIJ8dsc1Hatzg9fdniH9z/zWS9zPns63al1/3YNz6VrMfvvJysrzr9rv1t6Q337piWrH/xWx9O1qdfnx7HfydaGcv1Smwf8frf7NmBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgE4+xvA+vvPjVZXzf3Gx3q5K16muwvHtvZeNrkT9zxyeS6x/Y/k6zvefF/kvUclRpnt73Y9jbba4Ytu872L2yvLn7Oq7JhANUbzcf4uyTNG2H5zRFxcvHzULVtAaha07BHxCOStnegFwBtVOYLustsP1V8zG84cZbtRbYHbA8MameJzQEoo9Ww3ybp3ZJOlrRF0pcbPTEi+iOiLyL6ejWuxc0BKKulsEfE1ojYExF7Jd0uaVa1bQGoWkthtz112MMLJaXnJAZQu6bXjbd9r6Q5kg63vVnStZLm2D5ZUkjaKOnj7WsRJ1ybHk/umVvfsVG9bjyOLkl/uarxNfOP/fvVyXX3/PrXrbSEBpqGPSIWjLD4zjb0AqCNOFwWyARhBzJB2IFMEHYgE4QdyARTNneBOOMDyfr689PTIqcuJf3c7l3Jdcc7fYrzEWPSRz0ONjlD+uunfqth7Qvv/Wh65SfXpuvYL+zZgUwQdiAThB3IBGEHMkHYgUwQdiAThB3IBOPsFThg2tHJ+uZbD03Wl532tWR9Us+ByfpH/3uk64EO2f75Y5Prbj0t/drLr/hSst6st9njBhvWdsw8OLnuxCeTZewn9uxAJgg7kAnCDmSCsAOZIOxAJgg7kAnCDmSCcfYKbDsnPZb9tZNuTdYP7RmbrF+77ZT09r8wo2Ft3IrHk+sevSJZ1uwZn0rWfzb/tvQLJGw7dcSZhd8w8b6WXxojYM8OZIKwA5kg7EAmCDuQCcIOZIKwA5kg7EAmGGcfpdS13f/p725MrttsHP2aX85O1tfNTZ/3Pe6l9Fh6GWO3p6dkLmPKqiYXnUelmu7ZbU+3vcL2OttrbV9RLJ9se5nt9cXtpPa3C6BVo/kYv1vSVRFxgqTTJV1q+0RJV0taHhEzJS0vHgPoUk3DHhFbImJVcX+HpHWSpkmaL2lJ8bQlki5oU48AKrBfX9DZPk7SKZJWSjoyIrZIQ78QJE1psM4i2wO2Bwa1s2S7AFo16rDbnijpfklXRsQro10vIvojoi8i+nqVniQQQPuMKuy2ezUU9Hsi4nvF4q22pxb1qZK2tadFAFVoOvRm25LulLQuIm4aVloqaaGkG4rbB9vSYZfY8teNL4nc7HLKizbNSda3zkv/zt3z0svJejsdd8amZL3X6aG5ZlM6o3NGM85+pqSLJT1te3Wx7BoNhfw+25dI+rmki9rSIYBKNA17RDwqqdFVBuZW2w6AduFwWSAThB3IBGEHMkHYgUwQdiATnOJa8Lj00X1HHbKjYW2v9ibX/dcV70/Wj3/psWS9WW97Zp2YrKdsuDj9X+DHM29O1gfjoGS92XuDzmHPDmSCsAOZIOxAJgg7kAnCDmSCsAOZIOxAJhhnL3hM+rzsQ8e+1vJrf/Wixcn61z84J1k/pMm2b/+t/v1taT+Uu7rQc7t3Nawd9ELjGqrHnh3IBGEHMkHYgUwQdiAThB3IBGEHMkHYgUwwzl7w2N5k/Yn1xzWsrZg6Mbnu2Qe9mq6/5wfJek+T38l1njF+2k2XJ+tH/7DxNe/HPLmq6naQwJ4dyARhBzJB2IFMEHYgE4QdyARhBzJB2IFMOCI9gbbt6ZLulnSUhoZ0+yPiK7avk/QXkl4onnpNRDyUeq1DPDlm+5038eve3z8lWd+wID2G/8Nzb0rWjzkgfW32x3Y2Phd/4cOLkus2c8It6bnh96x9ptTro1orY7leie0jzro8moNqdku6KiJW2T5Y0hO2lxW1myPixqoaBdA+o5mffYukLcX9HbbXSZrW7sYAVGu//ma3fZykUyStLBZdZvsp24ttT2qwziLbA7YHBrWzXLcAWjbqsNueKOl+SVdGxCuSbpP0bkkna2jP/+WR1ouI/ojoi4i+3pLXMwPQulGF3XavhoJ+T0R8T5IiYmtE7ImIvZJulzSrfW0CKKtp2G1b0p2S1kXETcOWTx32tAslram+PQBVGc3Q21mSfizpaf3/2ZTXSFqgoY/wIWmjpI8XX+Y19E4degO6Ramht4h4VNJIKyfH1AF0F46gAzJB2IFMEHYgE4QdyARhBzJB2IFMEHYgE4QdyARhBzJB2IFMEHYgE4QdyARhBzJB2IFMND2fvdKN2S9Iem7YosMlvdixBvZPt/bWrX1J9NaqKns7NiKOGKnQ0bC/ZeP2QET01dZAQrf21q19SfTWqk71xsd4IBOEHchE3WHvr3n7Kd3aW7f2JdFbqzrSW61/swPonLr37AA6hLADmagl7Lbn2X7G9gbbV9fRQyO2N9p+2vZq2wM197LY9jbba4Ytm2x7me31xe2Ic+zV1Nt1tn9RvHerbZ9XU2/Tba+wvc72WttXFMtrfe8SfXXkfev43+y2x0j6maQ/kLRZ0uOSFkTETzraSAO2N0rqi4jaD8Cw/XuSXpV0d0S8v1j2RUnbI+KG4hflpIj4TJf0dp2kV+uexruYrWjq8GnGJV0g6U9V43uX6OuP1YH3rY49+yxJGyLi2YjYJek7kubX0EfXi4hHJG3fZ/F8SUuK+0s09J+l4xr01hUiYktErCru75D0+jTjtb53ib46oo6wT5O0adjjzequ+d5D0sO2n7C9qO5mRnDk69NsFbdTau5nX02n8e6kfaYZ75r3rpXpz8uqI+wjTSXVTeN/Z0bEqZLOlXRp8XEVozOqabw7ZYRpxrtCq9Ofl1VH2DdLmj7s8TGSnq+hjxFFxPPF7TZJD6j7pqLe+voMusXttpr7eUM3TeM90jTj6oL3rs7pz+sI++OSZto+3vZYSR+RtLSGPt7C9oTiixPZniDpHHXfVNRLJS0s7i+U9GCNvbxJt0zj3WiacdX83tU+/XlEdPxH0nka+kb+vyR9ro4eGvQ1Q9J/Fj9r6+5N0r0a+lg3qKFPRJdIOkzScknri9vJXdTbP2poau+nNBSsqTX1dpaG/jR8StLq4ue8ut+7RF8ded84XBbIBEfQAZkg7EAmCDuQCcIOZIKwA5kg7EAmCDuQif8Dzu1293RTBYAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(next(X).numpy().astype(\"uint8\"))#Visualize images in train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d52d5170",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.reshape(X,(s[0],28,28,1))\n",
    "Xtest = tf.reshape(Xtest,(w[0],28,28,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e8a6c1f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0 4132]\n",
      " [   1 4684]\n",
      " [   2 4177]\n",
      " [   3 4351]\n",
      " [   4 4072]\n",
      " [   5 3795]\n",
      " [   6 4137]\n",
      " [   7 4401]\n",
      " [   8 4063]\n",
      " [   9 4188]]\n"
     ]
    }
   ],
   "source": [
    "unique, counts = np.unique(np.array(y), return_counts=True)#understand distribution of y_train so as to choose metric for classification\n",
    "#As the distribution of y_train in this dataset is somewhat uniform we can use accuarcy as evaluation metric \n",
    "print (np.asarray((unique, counts)).T)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7650a664",
   "metadata": {},
   "source": [
    "Lets do some data preprocessing to our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9724c743",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.cast(X,float)\n",
    "X = X/255.0\n",
    "\n",
    "Xtest = tf.cast(Xtest,float)\n",
    "Xtest = Xtest/255.0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bd7e65aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "initializer1 = tf.keras.initializers.GlorotNormal()#Glorot Normal initializer/Xavier initialization works better for sigmoid layers\n",
    "initializer2 = tf.keras.initializers.HeNormal()#He initalizer works better for Relu layers\n",
    "\n",
    "def convolutional_model(input_shape):\n",
    "    \"\"\"\n",
    "    Implements the forward propagation for the model:\n",
    "    CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> DENSE\n",
    "    \n",
    "    Arguments:\n",
    "    input_img -- input dataset, of shape (input_shape)\n",
    "\n",
    "    Returns:\n",
    "    model -- TF Keras model (object containing the information for the entire training process) \n",
    "    \"\"\"\n",
    "    \n",
    "    input_img = tf.keras.Input(shape=input_shape)\n",
    "    # Adding Conv2D layers to our neural network\n",
    "    \n",
    "    Z1 = tf.keras.layers.Conv2D(8,4,strides =(1,1),padding = 'same', kernel_initializer =initializer2 )(input_img)\n",
    "    \n",
    "    \n",
    "    # Performing batch normalization so as to address the problem of covariance shift \n",
    "    \n",
    "    b1 = tf.keras.layers.BatchNormalization(3)(Z1)\n",
    "    \n",
    "    # Applying relu activation\n",
    "    A1 =tf.keras.layers.ReLU()(b1)\n",
    "    \n",
    "    # Finally adding maxpool layer so as to bring stability to the model using pool size 2 and stride 2\n",
    "    \n",
    "    P1 = tf.keras.layers.MaxPool2D(pool_size =(2,2),strides =2, padding = 'same')(A1)\n",
    "\n",
    "    Z2 = tf.keras.layers.Conv2D(16,2,strides = (1,1),padding ='same',kernel_initializer =initializer2)(P1)\n",
    "    \n",
    "    b2 = tf.keras.layers.BatchNormalization(3)(Z2)\n",
    "    \n",
    "    A2 = tf.keras.layers.ReLU()(b2)\n",
    "   \n",
    "    P2 = tf.keras.layers.MaxPool2D(pool_size = (4,4),strides =1, padding = 'same')(A2)\n",
    "    \n",
    "    \n",
    "    Z3 = tf.keras.layers.Conv2D(24,4,strides =(1,1),padding = 'same', kernel_initializer =initializer2 )(P2)\n",
    "    \n",
    "    b3 = tf.keras.layers.BatchNormalization(3)(Z3)\n",
    "    \n",
    "    A3 =tf.keras.layers.ReLU()(b3)\n",
    "    \n",
    "\n",
    "    F = tf.keras.layers.Flatten()(A3)\n",
    "    \n",
    "    #print(F.shape)\n",
    "    outputs = tf.keras.layers.Dense(10,activation =\"softmax\",kernel_initializer =initializer1)(F)\n",
    "    \n",
    "    model = tf.keras.Model(inputs=input_img, outputs=outputs)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2f90940f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using learning rate decay for our optimizer\n",
    "lr_schedule = keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=1e-2,decay_steps=1000,decay_rate=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ea6dd71c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 28, 28, 1)]       0         \n",
      "                                                                 \n",
      " conv2d (Conv2D)             (None, 28, 28, 8)         136       \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 28, 28, 8)        32        \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " re_lu (ReLU)                (None, 28, 28, 8)         0         \n",
      "                                                                 \n",
      " max_pooling2d (MaxPooling2D  (None, 14, 14, 8)        0         \n",
      " )                                                               \n",
      "                                                                 \n",
      " conv2d_1 (Conv2D)           (None, 14, 14, 16)        528       \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 14, 14, 16)       64        \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " re_lu_1 (ReLU)              (None, 14, 14, 16)        0         \n",
      "                                                                 \n",
      " max_pooling2d_1 (MaxPooling  (None, 14, 14, 16)       0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_2 (Conv2D)           (None, 14, 14, 24)        6168      \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 14, 14, 24)       96        \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " re_lu_2 (ReLU)              (None, 14, 14, 24)        0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 4704)              0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 10)                47050     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 54,074\n",
      "Trainable params: 53,978\n",
      "Non-trainable params: 96\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "conv_model = convolutional_model((28,28,1))\n",
    "conv_model.compile(optimizer=keras.optimizers.Adam(learning_rate=lr_schedule),\n",
    "                  loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
    "                  metrics=['accuracy'])\n",
    "# As our labels are not one hot encoded we can use sparse categorical cross entropy as loss function, if our labels are one hot\n",
    "#encoded then we can use categorical crossentropy as loss function\n",
    "conv_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bb9a02f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/70\n",
      "296/296 [==============================] - 17s 56ms/step - loss: 0.6061 - accuracy: 0.9183 - val_loss: 0.1678 - val_accuracy: 0.9510\n",
      "Epoch 2/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0677 - accuracy: 0.9789 - val_loss: 0.0630 - val_accuracy: 0.9790\n",
      "Epoch 3/70\n",
      "296/296 [==============================] - 16s 55ms/step - loss: 0.0483 - accuracy: 0.9849 - val_loss: 0.0858 - val_accuracy: 0.9743\n",
      "Epoch 4/70\n",
      "296/296 [==============================] - 17s 57ms/step - loss: 0.0368 - accuracy: 0.9891 - val_loss: 0.0516 - val_accuracy: 0.9855\n",
      "Epoch 5/70\n",
      "296/296 [==============================] - 16s 55ms/step - loss: 0.0305 - accuracy: 0.9905 - val_loss: 0.0647 - val_accuracy: 0.9805\n",
      "Epoch 6/70\n",
      "296/296 [==============================] - 16s 53ms/step - loss: 0.0273 - accuracy: 0.9913 - val_loss: 0.0507 - val_accuracy: 0.9857\n",
      "Epoch 7/70\n",
      "296/296 [==============================] - 16s 56ms/step - loss: 0.0244 - accuracy: 0.9920 - val_loss: 0.0497 - val_accuracy: 0.9860\n",
      "Epoch 8/70\n",
      "296/296 [==============================] - 17s 57ms/step - loss: 0.0199 - accuracy: 0.9937 - val_loss: 0.0440 - val_accuracy: 0.9879\n",
      "Epoch 9/70\n",
      "296/296 [==============================] - 17s 57ms/step - loss: 0.0169 - accuracy: 0.9948 - val_loss: 0.0429 - val_accuracy: 0.9864\n",
      "Epoch 10/70\n",
      "296/296 [==============================] - 17s 57ms/step - loss: 0.0144 - accuracy: 0.9950 - val_loss: 0.0569 - val_accuracy: 0.9860\n",
      "Epoch 11/70\n",
      "296/296 [==============================] - 18s 61ms/step - loss: 0.0128 - accuracy: 0.9962 - val_loss: 0.0495 - val_accuracy: 0.9848\n",
      "Epoch 12/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0133 - accuracy: 0.9956 - val_loss: 0.0593 - val_accuracy: 0.9855\n",
      "Epoch 13/70\n",
      "296/296 [==============================] - 16s 56ms/step - loss: 0.0097 - accuracy: 0.9968 - val_loss: 0.0547 - val_accuracy: 0.9840\n",
      "Epoch 14/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0084 - accuracy: 0.9974 - val_loss: 0.0554 - val_accuracy: 0.9840\n",
      "Epoch 15/70\n",
      "296/296 [==============================] - 17s 56ms/step - loss: 0.0080 - accuracy: 0.9970 - val_loss: 0.0442 - val_accuracy: 0.9876\n",
      "Epoch 16/70\n",
      "296/296 [==============================] - 17s 57ms/step - loss: 0.0088 - accuracy: 0.9964 - val_loss: 0.0511 - val_accuracy: 0.9888\n",
      "Epoch 17/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0056 - accuracy: 0.9982 - val_loss: 0.0444 - val_accuracy: 0.9886\n",
      "Epoch 18/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0027 - accuracy: 0.9991 - val_loss: 0.0436 - val_accuracy: 0.9900\n",
      "Epoch 19/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0025 - accuracy: 0.9991 - val_loss: 0.0567 - val_accuracy: 0.9888\n",
      "Epoch 20/70\n",
      "296/296 [==============================] - 16s 55ms/step - loss: 0.0022 - accuracy: 0.9994 - val_loss: 0.0409 - val_accuracy: 0.9895\n",
      "Epoch 21/70\n",
      "296/296 [==============================] - 16s 55ms/step - loss: 0.0044 - accuracy: 0.9984 - val_loss: 0.0556 - val_accuracy: 0.9855\n",
      "Epoch 22/70\n",
      "296/296 [==============================] - 16s 55ms/step - loss: 0.0044 - accuracy: 0.9985 - val_loss: 0.0497 - val_accuracy: 0.9883\n",
      "Epoch 23/70\n",
      "296/296 [==============================] - 16s 54ms/step - loss: 0.0014 - accuracy: 0.9996 - val_loss: 0.0468 - val_accuracy: 0.9905\n",
      "Epoch 24/70\n",
      "296/296 [==============================] - 15s 50ms/step - loss: 0.0013 - accuracy: 0.9996 - val_loss: 0.0456 - val_accuracy: 0.9895\n",
      "Epoch 25/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 3.6608e-04 - accuracy: 1.0000 - val_loss: 0.0393 - val_accuracy: 0.9917\n",
      "Epoch 26/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 1.2821e-04 - accuracy: 1.0000 - val_loss: 0.0400 - val_accuracy: 0.9919\n",
      "Epoch 27/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 7.5206e-05 - accuracy: 1.0000 - val_loss: 0.0398 - val_accuracy: 0.9919\n",
      "Epoch 28/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 6.0636e-05 - accuracy: 1.0000 - val_loss: 0.0394 - val_accuracy: 0.9921\n",
      "Epoch 29/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 5.2689e-05 - accuracy: 1.0000 - val_loss: 0.0404 - val_accuracy: 0.9921\n",
      "Epoch 30/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 4.8649e-05 - accuracy: 1.0000 - val_loss: 0.0408 - val_accuracy: 0.9924\n",
      "Epoch 31/70\n",
      "296/296 [==============================] - 10s 32ms/step - loss: 4.5837e-05 - accuracy: 1.0000 - val_loss: 0.0411 - val_accuracy: 0.9919\n",
      "Epoch 32/70\n",
      "296/296 [==============================] - 10s 32ms/step - loss: 3.6900e-05 - accuracy: 1.0000 - val_loss: 0.0422 - val_accuracy: 0.9919\n",
      "Epoch 33/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 3.5291e-05 - accuracy: 1.0000 - val_loss: 0.0423 - val_accuracy: 0.9924\n",
      "Epoch 34/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 3.4317e-05 - accuracy: 1.0000 - val_loss: 0.0418 - val_accuracy: 0.9926\n",
      "Epoch 35/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 3.0541e-05 - accuracy: 1.0000 - val_loss: 0.0426 - val_accuracy: 0.9924\n",
      "Epoch 36/70\n",
      "296/296 [==============================] - 10s 32ms/step - loss: 2.8514e-05 - accuracy: 1.0000 - val_loss: 0.0430 - val_accuracy: 0.9926\n",
      "Epoch 37/70\n",
      "296/296 [==============================] - 10s 32ms/step - loss: 2.6800e-05 - accuracy: 1.0000 - val_loss: 0.0435 - val_accuracy: 0.9924\n",
      "Epoch 38/70\n",
      "296/296 [==============================] - 10s 32ms/step - loss: 2.3799e-05 - accuracy: 1.0000 - val_loss: 0.0434 - val_accuracy: 0.9924\n",
      "Epoch 39/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 2.2662e-05 - accuracy: 1.0000 - val_loss: 0.0430 - val_accuracy: 0.9919\n",
      "Epoch 40/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 2.0411e-05 - accuracy: 1.0000 - val_loss: 0.0444 - val_accuracy: 0.9924\n",
      "Epoch 41/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 1.8694e-05 - accuracy: 1.0000 - val_loss: 0.0445 - val_accuracy: 0.9921\n",
      "Epoch 42/70\n",
      "296/296 [==============================] - 9s 32ms/step - loss: 1.7433e-05 - accuracy: 1.0000 - val_loss: 0.0449 - val_accuracy: 0.9924\n",
      "Epoch 43/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 1.9721e-05 - accuracy: 1.0000 - val_loss: 0.0450 - val_accuracy: 0.9924\n",
      "Epoch 44/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 1.5009e-05 - accuracy: 1.0000 - val_loss: 0.0461 - val_accuracy: 0.9921\n",
      "Epoch 45/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 1.4889e-05 - accuracy: 1.0000 - val_loss: 0.0452 - val_accuracy: 0.9919\n",
      "Epoch 46/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 1.2431e-05 - accuracy: 1.0000 - val_loss: 0.0466 - val_accuracy: 0.9926\n",
      "Epoch 47/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 1.1076e-05 - accuracy: 1.0000 - val_loss: 0.0462 - val_accuracy: 0.9924\n",
      "Epoch 48/70\n",
      "296/296 [==============================] - 10s 35ms/step - loss: 1.1294e-05 - accuracy: 1.0000 - val_loss: 0.0475 - val_accuracy: 0.9921\n",
      "Epoch 49/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 1.0370e-05 - accuracy: 1.0000 - val_loss: 0.0475 - val_accuracy: 0.9921\n",
      "Epoch 50/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 9.1358e-06 - accuracy: 1.0000 - val_loss: 0.0475 - val_accuracy: 0.9921\n",
      "Epoch 51/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 9.7007e-06 - accuracy: 1.0000 - val_loss: 0.0471 - val_accuracy: 0.9926\n",
      "Epoch 52/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 8.0322e-06 - accuracy: 1.0000 - val_loss: 0.0483 - val_accuracy: 0.9919\n",
      "Epoch 53/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 7.3632e-06 - accuracy: 1.0000 - val_loss: 0.0482 - val_accuracy: 0.9921\n",
      "Epoch 54/70\n",
      "296/296 [==============================] - 10s 35ms/step - loss: 6.3178e-06 - accuracy: 1.0000 - val_loss: 0.0490 - val_accuracy: 0.9919\n",
      "Epoch 55/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 5.7489e-06 - accuracy: 1.0000 - val_loss: 0.0492 - val_accuracy: 0.9926\n",
      "Epoch 56/70\n",
      "296/296 [==============================] - 10s 35ms/step - loss: 6.3505e-06 - accuracy: 1.0000 - val_loss: 0.0497 - val_accuracy: 0.9921\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57/70\n",
      "296/296 [==============================] - 10s 35ms/step - loss: 5.8198e-06 - accuracy: 1.0000 - val_loss: 0.0506 - val_accuracy: 0.9919\n",
      "Epoch 58/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 5.0719e-06 - accuracy: 1.0000 - val_loss: 0.0503 - val_accuracy: 0.9919\n",
      "Epoch 59/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 4.4134e-06 - accuracy: 1.0000 - val_loss: 0.0501 - val_accuracy: 0.9921\n",
      "Epoch 60/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 4.2838e-06 - accuracy: 1.0000 - val_loss: 0.0500 - val_accuracy: 0.9921\n",
      "Epoch 61/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 3.9808e-06 - accuracy: 1.0000 - val_loss: 0.0511 - val_accuracy: 0.9921\n",
      "Epoch 62/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 3.3904e-06 - accuracy: 1.0000 - val_loss: 0.0511 - val_accuracy: 0.9919\n",
      "Epoch 63/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 3.4299e-06 - accuracy: 1.0000 - val_loss: 0.0528 - val_accuracy: 0.9919\n",
      "Epoch 64/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 2.9309e-06 - accuracy: 1.0000 - val_loss: 0.0528 - val_accuracy: 0.9921\n",
      "Epoch 65/70\n",
      "296/296 [==============================] - 10s 34ms/step - loss: 2.8402e-06 - accuracy: 1.0000 - val_loss: 0.0529 - val_accuracy: 0.9921\n",
      "Epoch 66/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 3.1961e-06 - accuracy: 1.0000 - val_loss: 0.0531 - val_accuracy: 0.9917\n",
      "Epoch 67/70\n",
      "296/296 [==============================] - 10s 33ms/step - loss: 2.5946e-06 - accuracy: 1.0000 - val_loss: 0.0545 - val_accuracy: 0.9917\n",
      "Epoch 68/70\n",
      "296/296 [==============================] - 12s 40ms/step - loss: 2.2705e-06 - accuracy: 1.0000 - val_loss: 0.0536 - val_accuracy: 0.9919\n",
      "Epoch 69/70\n",
      "296/296 [==============================] - 12s 40ms/step - loss: 2.2777e-06 - accuracy: 1.0000 - val_loss: 0.0540 - val_accuracy: 0.9921\n",
      "Epoch 70/70\n",
      "296/296 [==============================] - 12s 40ms/step - loss: 2.1985e-06 - accuracy: 1.0000 - val_loss: 0.0552 - val_accuracy: 0.9921\n"
     ]
    }
   ],
   "source": [
    "history = conv_model.fit(X,y, epochs=70,validation_split = 0.1,batch_size = 128,verbose=1)\n",
    "# Fitting our model to training data and use a part of it for cross validation and train using over 70 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0a6c3fb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = conv_model.predict(Xtest)\n",
    "y_pred = np.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89c9362d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now I have submitted my predictions and got a score of 99.025% accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}