FashionMNIST_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Downloading the dataset\n",
    "train_set = torchvision.datasets.FashionMNIST(root = \".\", train= True ,download = True , transform = transforms.ToTensor())\n",
    "test_set = torchvision.datasets.FashionMNIST(root = \".\", train= False ,download = True , transform = transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_loader = torch.utils.data.DataLoader(train_set, batch_size = 32, shuffle = False)\n",
    "test_loader = torch.utils.data.DataLoader(test_set , batch_size = 32, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x2b5cfba8b10>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# B Question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a neural network class which extends the nn.Module base class\n",
    "\n",
    "#Created a network with ReLU activation function\n",
    "class Network_relu(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight) #Initializing the weights to the 1st convolutional layer\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight) #Initializing the weights to the 2nd convolutional layer\n",
    "\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight) #Initializing the weights to the 1st linear layer\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight) #Initializing the weights to the 2nd linear layer\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight) #Initializing the weights to the output layer\n",
    "\n",
    "    \n",
    "    #Mapping input tensor to the outpur tensor\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = F.relu(t) #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2) #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = F.relu(t) #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2) #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = F.relu(t)  #relu activation\n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = F.relu(t)  #relu activation\n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)\n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function calculating the accuacy and the loss of the network on the training loader dataset\n",
    "\n",
    "def train_set_accuracy(training_loader, network, learning_rate):\n",
    "    \n",
    "    #optimizer object of stochastic gradient descent type having particular learning rate\n",
    "    optimizer = optim.SGD(network.parameters(), lr = learning_rate)\n",
    "    \n",
    "\n",
    "    total_loss = 0  #Variable to calculate the total loss of the dataset\n",
    "    total_correct = 0  #Variable that calculates the total correct predictions done by the network\n",
    "\n",
    "    for batch in training_loader: # Get Batch\n",
    "        training_images, training_labels = batch  #Getting tensor of images and tensor of corresponding labels\n",
    "        training_images, training_labels = training_images.cuda(), training_labels.cuda() #Passing the tensor to the GPU\n",
    "\n",
    "        predictions = network(training_images) # Pass the batch to the network\n",
    "        training_loss = F.cross_entropy(predictions, training_labels) # Calculating the cross entropy loss\n",
    "\n",
    "        optimizer.zero_grad()#Set the gradient values to zero so that already present values are not added to the new batch\n",
    "        training_loss.backward() # Calculate the gradients\n",
    "        optimizer.step() # Update the weights\n",
    "\n",
    "        total_loss += training_loss.item() #Add the loss caclulated per batch\n",
    "        \n",
    "        #Caculate the element wise equality to measure the accuracy between the predictions and the labels\n",
    "        total_correct += predictions.argmax(dim=1).eq(training_labels).sum().item()  \n",
    "  \n",
    "    return total_correct*100/len(train_set), total_loss*100/len(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function calculating the accuacy and the loss of the network on the test loader dataset\n",
    "\n",
    "def test_set_accuracy(test_loader, network):\n",
    "\n",
    "    total_loss = 0 #Variable to calculate the total loss of the dataset\n",
    "    total_correct = 0 #Variable that calculates the total correct predictions done by the network\n",
    "\n",
    "    for batch in test_loader: # Get Batch\n",
    "        test_images, test_labels = batch #Getting tensor of images and tensor of corresponding labels\n",
    "        test_images, test_labels = test_images.cuda(), test_labels.cuda() #Passing the tensor to the GPU\n",
    "\n",
    "        predictions = network(test_images) # Pass the batch to the network\n",
    "        test_loss = F.cross_entropy(predictions, test_labels) # Calculating the cross entropy loss\n",
    "\n",
    "        total_loss += test_loss.item() #Add the loss caclulated per batch\n",
    "        \n",
    "        #Caculate the element wise equality to measure the accuracy between the predictions and the labels\n",
    "        total_correct += predictions.argmax(dim=1).eq(test_labels).sum().item()\n",
    "    \n",
    "    return total_correct*100/len(test_set), total_loss*100/len(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Change the device from CPU to GPU\n",
    "device = torch.device(\"cuda\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our relu network class\n",
    "network_relu = Network_relu().to(device) #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for ReLU\n",
    "\n",
    "relu_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    #Training on training loader\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu, learning_rate =0.1)\n",
    "    #Storing accuracy and loss after every epoch on training data\n",
    "    relu_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    #Testing on test loader\n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu)\n",
    "    #Storing accuracy and loss after every epoch on test data\n",
    "    relu_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Training Accuracy Obtained : 99.44833333333334\n",
      "Final Test Accuracy Obtained : 90.52\n"
     ]
    }
   ],
   "source": [
    "#Final train and test accuracy obtained on relu activation network\n",
    "print(\"Final Training Accuracy Obtained : {}\".format(relu_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Test Accuracy Obtained : {}\".format(relu_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABFTElEQVR4nO3dd3gVVfrA8e+bQkISakIvgqiIIASIKMWKsqgo2FaxLLoq6qqAZRV1/Ym6roh17YsuVgQRxAKIgAqsCmIoSpUaINQkhIQQQtr7++NMQoCUG0hyU97P88xz7507Z+adG5h3zjkzZ0RVMcYYY3wV4O8AjDHGVC2WOIwxxpSKJQ5jjDGlYonDGGNMqVjiMMYYUyqWOIwxxpSKJQ5TaYjINyIypKyX9ScRiRORC/0dhzFlKcjfAZiqTUTSCnwMAw4COd7nO1R1vK/rUtWLy2PZykhEvgHO9j6GAApkep8/VtU7S7m+UcBJqnpjmQVpTBEscZjjoqoRee9FJA64TVXnHLmciASpanZFxlaZFUx8IvI+EK+q//BfROVLRAQQVc31dyzm+FlTlSkXInKeiMSLyMMishN4T0QaiMg0EUkQkWTvfcsCZeaKyG3e+5tF5EcRecFbdpOIXHyMy7YVkfkisk9E5ojIGyLycRFx+xLj0yLyk7e+WSISVeD7m0Rks4gkichjx/jbDRCRZSKyV0R+FpHOBb57WES2edv+Q0T6ikh/4FHgWhFJE5HfiljvSBHZ4JVdJSJXHPH97SKyusD33bz5rUTkc+83SRKR1735owr+jiLSRkRURIIK/FbPiMhPQDpwoojcUmAbG0XkjiNiGOjte6oXa38RuUZEFh+x3AMi8sWx/L7m+FniMOWpKdAQOAEYivv39p73uTVwAHi9mPJnAn8AUcAY4L/emWtpl/0EWAREAqOAm4rZpi8xXg/cAjQGagEPAojIacBb3vqbe9trSSl4B+txwB1e+f8AX4lIiIi0B+4BzlDVOsCfgDhVnQn8C/hUVSNUtUsRq9+Aax6rBzwJfCwizbztXoP7bf4C1AUuB5JEJBCYBmwG2gAtgIml2KWbcH/7Ot46dgMDvG3cArxcIEH1AD4E/g7UB84B4oCvgLYi0qHAem8EPipFHKYMWeIw5SkXeEJVD6rqAVVNUtUpqpquqvuAZ4Bziym/WVXfUdUc4AOgGdCkNMuKSGvgDOD/VDVTVX/EHYgK5WOM76nqWlU9AEwCor35VwPTVHW+qh4EHvd+g9K4HfiPqv6iqjmq+gGu3+gsXN9RCHCaiASrapyqbvB1xar6mapuV9VcVf0UWAf08L6+DRijqr+qs15VN3vfNwf+rqr7VTXD+w199b6qrlTVbFXNUtXpqrrB28Y8YBaH+npuBcap6mwvxm2qusb7LT/FJQtEpCMuiU0rRRymDFniMOUpQVUz8j6ISJiI/MdrykkF5gP1vbPawuzMe6Oq6d7biFIu2xzYU2AewNaiAvYxxp0F3qcXiKl5wXWr6n4gqahtFeEE4AGvmWqviOwFWgHNVXU9MAJXM9gtIhNFpLmvKxaRvxRoAtsLdMLV0PC2UVgSaoVLysfaP3XYby0iF4vIQhHZ48VwiQ8xgDsZuN6rRd4ETPISivEDSxymPB059PIDQHvgTFWti2uKACiq+aks7AAaikhYgXmtiln+eGLcUXDd3jYjSxcuW4FnVLV+gSlMVScAqOonqtoHl2AUeM4rV+ww1yJyAvAOrqkrUlXrAys4tF9bgXZFxNM6r9/iCPtxV9LlaVrIMvlxiUgIMAV4AWjixTDDhxhQ1YW4q87OxjUVWjOVH1niMBWpDq7PYK+INASeKO8Nes0tscAoEaklIj2By8opxsnAABHpIyK1gKco/f+xd4A7ReRMccJF5FIRqSMi7UXkAu8AnOHFmXfp8y6gjYgUtb1w3EE8AUBEbsHVOPK8CzwoIt297Z7kJZtFuIQ42oslVER6e2WWAeeISGsRqQc8UsK+1cI1tSUA2eIuYOhX4Pv/Ard4Hf4BItJCRE4t8P2HuP6m7FI2l5kyZonDVKRXgNpAIrAQmFlB270B6IlrNvonrr28qGaOVzjGGFV1JXA3rjN+B5AMxJcmUFWNxfVzvO6VXw/c7H0dAoz2YtuJ65x/1PvuM+81SUSWFLLeVcCLwAJckjkd+KnA95/h+nM+AfYBXwANvT6jy4CTgC3e/lzrlZmN+y1/BxZTQp+D12c0DNcvlIyrOXxV4PtFeB3mQAowD1ezyvMRLtlZbcPPxB7kZGoaEfkUWKOq5V7jMWVHRGrjrsrqpqrr/B1PTWY1DlPticgZItLOa/7oDwzEnVGbquUu4FdLGv5nd46bmqAp8DmuozoeuEtVl/o3JFMa4kYlEGCQfyMxYE1VxhhjSsmaqowxxpRKjWiqioqK0jZt2vg7DGOMqVIWL16cqKqNjpxfIxJHmzZtiI2N9XcYxhhTpYjI5sLml1tTlYiME5HdIrKiwLyGIjJbRNZ5rw0KfPeIiKwXN+Lnn4pYZ5HljTHGVIzy7ON4H+h/xLyRwHeqejLwnfc5b1TR64COXpk3ixi/qNDyxhhjKk65JQ5VnQ/sOWL2QNxgZXivgwrMn+iNoroJd7dsD45WVHljjDEVpKKvqmqiqjsAvNfG3vwWHD6KZrw3z9fyRxGRoSISKyKxCQkJZRK8McaYynM5bmEjjx7XDSaqOlZVY1Q1plGjoy4KMMYYc4wqOnHsKvDEsWa4cWfA1TAKDnXdEtheivLGGGMqSEUnjq+AId77IcCXBeZf5z0esy1wMm44Z1/LG2OMqSDldh+HiEwAzgOiRCQe91yD0cAkEbkVN0TzNeCGoxaRScAqIBu42xvOGRF5F3jbG2660PLGGKOqKEpAkY8kKVtpmWls3ruZuL1xxO2NY/f+3YgIARJAoAQSIAEESABBAUHUC61H/dD61A+tT4PQBvnvI2pFUCuwFu7BhmUr+UAyv2z7hbNankX90Ppluu4aMVZVTEyM2g2AxlQP2bnZbEnZwqqEVaxOWM3qRG9KWE1QQBCPnf0Yfzvjb4QEhZTJ9nI1l1UJq/h568/8vPVnViasJG5vHInpiWWy/kAJJLxWOGHBYYQHhxNeK5zaQbUJCQohNCg0fwoJDKF+aH1a1W1F63qt86emEU0REVYnrGZB/AIWbF3AgvgFrE5cDcBX133FZe2Le3ZZ0URksarGHDXfEocxprJIy0xj6Y6lxG6PZenOpezav4uUjBRSDqbkv6ZnpR9Wpkl4Ezo06sBpUaexds9a5mycQ5v6bXjmgme4rtN1xdZAEtMT2Zuxl4zsDA5kHXCv2QdIz0pn+a7l/Bz/Mwu2LiDlYAoAjcIa0bVZV9rWb0ub+m3yp7b129I43F3kmau5h02ZOZmkHkxlb8Ze9mbsJTkjOf99WmYa6Vnp7M/cz/6s/e591n4OZB3gYM5BMrIzDpuS0pPYl7nvsH0ICggiJDCE/Vn7AWhYuyE9W/bkrJZn5b+G1wo/pr+HJQ5LHMaU2vo969mwZwMJ6QkkpieSsN97TU9ARGgQ2sBNtQ+9tq3flpjmMQQGFHYP7+E27NnA9HXT+XX7r8Ruj+WPxD9Q74LKZhHNaFWvFfVC6lEvtJ579d43r9OcDlEd6NCoAw1rNzxsnbM2zOLhOQ+zbOcyujXrxpgLx9D3xL7kai6rE1bz09af+HHLj/y45Uc27d1UZGyC0LFxR3q17EWvVm46qeFJ5dKsVBopGSlsTd3KlpQt+VNaZhrdm3WnZ6uenNzw5DKL0RKHJQ5jfPb7rt8ZNXcUU9dMPWx+oAQSGRZJozB3iXtyRjLJB5I5kH3gsOUa1m5I/5P6c+nJl9L/pP75B3dVZenOpXyx5gu+WPMFy3cvB6B5neZ0b9ad7s26E9M8hu7Nu9M0oukxx5+ruXyy/BP+8f0/2Jyymeim0Wzeu5nkjGQAGoc3pk/rPvRs2ZMm4U0IDQqldnBtagfVzn/ftn5bGtSu2aMaWeKwxGFMiVbuXsmT857ks1WfUTekLveddR8XnXgRjcIbERUWRf3Q+oU2/WRkZ7hmmAPJLN+9nOnrpvPNum9ISE8gQALo2bInpzU6jW83fMuWlC0ESABntz6bQacOYmD7gbRt0LZc9icjO4M3f32TSSsn0alxJ/q07kOf1n1o16Cd32sOVYElDkscpgZIPZjK7A2z+X3X7ySkJ7B7/+5Dr/sTOJhzkHYN2nFK5Cm0j2zPKZGncErkKYQEhTDmpzFMXDGR8FrhjDhzBPf3vP+4zrhzNZdft/3K9HXTmbFuBmsS19D3xL4Maj+Iy9pfRlRYVBnuuSkPljgscZgqIDMnk1UJq1iyY0n+tDJhJSfUO+Gwzs72Ue3zz/w3Jm9k2tppfL32a+bFzSMrNwtBaFi7IY3DG9MovBGNwhrROLwxwQHBrE9ez9qktWxK3kSOu+odgLDgMIb1GMaDvR4kMizSXz+BqUQscVjiMJVMTm4OaxLXsGjbIhZtW8Sv239l+e7lZOZkAhBRK4KuTbvSsVFHNu3dxC/bfmFvxl4A6ofW54zmZxCfGp9/2WWHqA4MOGUAA04ZQM+WPQkODC52+5k5mWxM3sjapLXsTNvJoFMH5V8ZZAxY4rDEYfwiMyeTXWm72Jm2M39av2c9i7YvInZ7LGmZaQDUDanLGc3PoHuz7nRr1o2uzbpyUsOTDutPyNVc1iatZWH8QhbGL2TRtkVEhkUy4GSXLNo1bOev3TTVlCUOSxymjO3Yt4OlO5eyNWUru/fvdlP67vz3O9N2sufAkU8WgOCAYKKbRtOjRY/86ZTIUyrsjmdjfFVU4qgRj4415nhtSdlC7PbY/H6HpTuXsjNt52HLNAhtQOPwxjQOb0yHqA6ce8K5NItoRtOIpjSNaEqzOu59k/AmJTYjGVOZWeIwpgjrktbx2arP+GzVZyzbuQxw9zF0bNyR/if1p2vTrnRt2pV2DdsRFRZFrcBa/g3YmApiicPUOOv3rGf+5vmEB4dTN6QudULquNdadUjPSmfqmql8tuozft/1OwA9W/bkxX4vcnbrszm9yemEBoX6eQ+M8S9LHKbGSM9K51//+xfP//x8/pVLRendqjev/OkVrjrtKlrWbVlBERpTNVjiMNWeqvLVH18xfOZwNqds5qbON/FIn0dQlH0H95F6MJXUg6nsy9yHqtKvXT9a1C3sycXGGLDEYaq5DXs2MGzmMGasm0HHRh2Zd/M8zjnhHH+HZUyVZonDVHm79+8mbm8cCfsTSEhPIGG/G2JjR9oOJq+aTHBgMC/1e4l7etxjVzMZUwYscZgqaWPyRj5f/Tmfr/6cBfELjvo+NCiUxuGN+XPHPzP6wtE0r9PcD1EaUz35JXGIyHDgdkCAd1T1FRH5FGjvLVIf2Kuq0YWUjQP2ATlAdmE3p5jqaXXCaqasnsKU1VPyL4/t1qwbT5//NNFNo/PHY2oU3ojw4HAb/dSYclLhiUNEOuGSRg8gE5gpItNV9doCy7wIpBSzmvNVtWye22gqtf2Z+5m0chJjl4xlYfxCAHq16sWL/V7kilOvKLfhuI0xRfNHjaMDsFBV0wFEZB5wBTDG+yzAn4EL/BCbqSR+3/U7YxeP5aPfPyL1YCqnRp3KS/1e4tpO11qzkzF+5o/EsQJ4RkQigQPAJUDBgaTOBnap6roiyiswS0QU+I+qji1sIREZCgwFaN26dVnFbspRRnYGk1ZO4q3Yt1gYv5CQwBCu6XgNQ7sNpU/rPtb0ZEwlUeGJQ1VXi8hzwGwgDfgNyC6wyGBgQjGr6K2q20WkMTBbRNao6vxCtjMWGAtukMMy2wFT5rambOWt2Ld4Z8k7JKYncmrUqbz8p5e5qfNN9lwIYyohv3SOq+p/gf8CiMi/gHjvfRBwJdC9mLLbvdfdIjIV11dyVOIwlZuqMjduLq//+jpfrPkCgMvbX849Z9zDBW0vsNqFMZWYv66qauwd+FvjEkVP76sLgTWqGl9EuXAgQFX3ee/7AU9VSNCmTKRlpvHRbx/x+q+vsyphFZG1I/l7r79zV8xdnFD/BH+HZ4zxgb/u45ji9XFkAXerarI3/zqOaKYSkebAu6p6CdAEmOqdjQYBn6jqzIoL2xyrtUlreWPRG7z/2/ukHkylW7NujLt8HNd1uo7awbX9HZ4xphT81VR1dhHzby5k3nZcBzqquhHoUq7BmTL1/abvGfPTGL7d8C3BAcFc0/Ea7u1xL2e2ONOao4ypouzOcVMuMrIzeGj2Q7y26DWa12nOU+c9xe3db6dpRFN/h2aMOU6WOEyZW75rOdd/fj0rdq9g+JnDGX3haHuGhTHViCUOU2ZUldcWvcZDsx+ifmh9vrnhG/qf1N/fYRljypglDlMmdqXt4pYvb+Gb9d9w6cmXMm7gOBqHN/Z3WMaYcmCJw5RaTm4Oa5PWsmTHEhbvWMySHUuI3R5Ljubw+sWv87cz/mYd38ZUY5Y4jM+W7FjCA7Me4Ndtv7I/az/ghi/v0qQLQ7oM4W9n/I2OjTv6OUpjTHmzxGF88snyT7j1q1uJrB3JrV1vpVuzbnRr1o0OjToQFGD/jIypSex/vClWTm4OI+eM5IUFL3DOCefw2TWfWd+FMTWcJQ5TpOQDyVw35TpmbZjF3Wfczct/etkevWqMscRhCrdy90oGThzIlpQtvHvZu9za7VZ/h2SMqSQscZjDxKfG88GyDxj902giakUw7+Z59GzVs+SCxpgawxKHISM7g6/++IpxS8cxe+NscjWXfu36Me7ycbSo28Lf4RljKhlLHDXY6oTVvPHrG3yy/BOSM5JpVbcVj539GEO6DKFdw3b+Ds8YU0lZ4qiBtqVu44m5T/DesvcIDgjmyg5Xckv0LVzQ9gICAwL9HZ4xppKzxFGDpGSkMOanMby88GWyc7MZfuZwHj37UaLCovwdmjGmCrHEUQNk5mTyduzbPDXvKZIOJHH96dfzz/P/SdsGbf0dmjGmCgrwx0ZFZLiIrBCRlSIywps3SkS2icgyb7qkiLL9ReQPEVkvIiMrNPAq6Ledv9Hl7S4MnzmcLk27EHt7LOOvHG9JwxhzzCq8xiEinYDbgR5AJjBTRKZ7X7+sqi8UUzYQeAO4CIgHfhWRr1R1VTmHXeWoKmMXj2X4zOE0rN2Q6ddP5+KTLrbBB40xx80fTVUdgIWqmg4gIvOAK3ws2wNY7z1CFhGZCAwELHEUkHowldu/vp1JKyfxp3Z/4sMrPrRhQowxZcYfTVUrgHNEJFJEwnDPE2/lfXePiPwuIuNEpEEhZVsAWwt8jvfmGc/i7Yvp9p9uTFk1hWf7PsuMG2ZY0jDGlKkKTxyquhp4DpgNzAR+A7KBt4B2QDSwA3ixkOKFtbNoYdsRkaEiEisisQkJCWUQeeX3+qLX6TWuFwdzDjLv5nmM7DOSAPFLN5Yxphrzy1FFVf+rqt1U9RxgD7BOVXepao6q5gLv4JqljhTPodoJQEtgexHbGKuqMaoa06hRo7LehUpn3NJx3PvNvfRr149ldyyjd+ve/g7JGFNN+euqqsbea2vgSmCCiDQrsMgVuCatI/0KnCwibUWkFnAd8FV5x1vZLd2xlLtn3E3ftn354toviAyL9HdIxphqzF/3cUwRkUggC7hbVZNF5CMRicY1PcUBdwCISHPgXVW9RFWzReQe4FsgEBinqiv9sgeVRPKBZK7+7Goia0fyyVWf2J3fxphy55fEoapnFzLvpiKW3Y7rQM/7PAOYUX7RVR25msuQL4awJWUL82+eb53gxpgKYXeOV2HP/fgcX6/9mlf7v2pDnxtjKoxdclNFfbfxO/7xwz8Y3Gkw9/S4x9/hGGNqEEscVVB8ajzXTbmOU6NOZexlY+1ucGNMhbLEUcVk5mRyzWfXkJGdwZQ/TyGiVoS/QzLG1DDWx1HFPDjrQRbGL2TS1ZM4NepUf4djjKmBrMZRhUxYPoHXFr3GfWfdxzUdr/F3OMaYGsoSRxWxcvdKbvv6Nvq07sNzFz7n73CMMTWYJY4qIPVgKldOupI6terw6dWfEhwY7O+QjDE1mPVxVHKqyl+//Csb9mzgu798R/M6zf0dkjGmhrPEUcm9tOAlpqyewvMXPc+5bc71dzjGGGNNVZXZ/M3zeXjOw1zZ4Uoe6PmAv8MxxhjAEkeltWPfDv782Z85scGJvDfwPbvJzxhTaVhTVSU1bOYwUg+mMucvc6gbUtff4RhjTD6rcVRCC+MXMnnVZB7q/RCdGnfydzjGGHMYSxyVjKry4KwHaRrRlAd7PejvcIwx5ijWVFXJfLHmC37a+hP/GfAfG4fKGFMpWY2jEsnKyeLhOQ/TIaoDf+36V3+HY4wxhfLXM8eHi8gKEVkpIiO8ec+LyBoR+V1EpopI/SLKxonIchFZJiKxFRl3eXtnyTus27OO5y58jqAAqwwaYyqnCk8cItIJuB3oAXQBBojIycBsoJOqdgbWAo8Us5rzVTVaVWPKPeAKknowlVFzR3HuCecy4JQB/g7HGGOK5I8aRwdgoaqmq2o2MA+4QlVneZ8BFgIt/RCb34z5aQwJ6Qk8f9Hzds+GMaZS80fiWAGcIyKRIhIGXAK0OmKZvwLfFFFegVkislhEhha1EREZKiKxIhKbkJBQJoGXl22p23hpwUtc1+k6zmhxhr/DMcaYYlV4Q7qqrhaR53BNU2nAb0BeTQMRecz7PL6IVfRW1e0i0hiYLSJrVHV+IdsZC4wFiImJ0TLejTL1fz/8Hzmaw78u+Je/QzHGmBL5pXNcVf+rqt1U9RxgD7AOQESGAAOAG1S10IO9qm73XncDU3F9JVXW8l3LeW/Ze9xzxj20bdDW3+EYY0yJ/HVVVWPvtTVwJTBBRPoDDwOXq2p6EeXCRaRO3nugH67pq0pSVf4+++/UC63HY+c85u9wjDHGJ/665nOKiEQCWcDdqposIq8DIbjmJ3Ad6HeKSHPgXVW9BGgCTPW+DwI+UdWZ/tmF4zdj3Qy+3fAtL/Z7kYa1G/o7HGOM8YkU0SJUrcTExGhsbOW65SMzJ5PT3zodgOV3LadWYC0/R2SMMYcTkcWF3fZgd5n5yRuL3mBt0lqmXz/dkoYxpkopsY9DRAaIiA1NUoYS9ifw5Lwn6X9Sfy45+RJ/h2OMMaXiS0K4DlgnImNEpEN5B1QTPP7D46RlpvFSv5f8HYoxxpRaiYlDVW8EugIbgPdEZIF3c12dco+uGvpt52+8s+Qd7ulxDx0aWR42xlQ9PjVBqWoqMAWYCDQDrgCWiMi95RhbtaOqjPh2BA1CG/DEuU/4OxxjjDkmJXaOi8hluCFA2gEfAT1Udbc3XMhq4LXyDbH6mLpmKnPj5vLGJW/QoHYDf4djjDHHxJerqq4BXj5yWA9VTRcRe2iEjzKyM3hw1oN0atyJod2LHGLLGGMqPV8SxxPAjrwPIlIbaKKqcar6XblFVs28vOBlNu3dxJyb5tizNowxVZovfRyfAbkFPud484yPEtMT+deP/2Jg+4H0PbGvv8Mxxpjj4kviCFLVzLwP3nu7Y60Uxvw0hvSsdJ7t+6y/QzGmyqgBg1pUWb4kjgQRuTzvg4gMBBLLL6TqZWfaTl5f9DrXn369XX5rjI8yM6FjR7jpJjh40N/RmCP5kjjuBB4VkS0ishU3gu0d5RtW9fHs/54lMyfTLr81frd8OTzzDKQXOvZ05TJ9OqxeDR9/DH/6EyQn+zsiU5AvNwBuUNWzgNOA01S1l6quL//Qqr741HjeXvw2N0ffzEkNT/J3OKaSK8+mmYUL4Zxz4B//gN69IS6u/LZVFj74AJo2hQ8/hJ9/hrPPhi1bil7+p5/gzDMhMhJeeQWyso592wcPwlVXQevW0Lw5NG4MDRpAnTpQu7Z7PeEEiI6G88+HK6+EW2+FZ5+FlJRj325V4tPlPSJyKdARCM17HraqPlWOcVULz8x/BlXl8XMe93cophJbtQqefhqmTHEHowsvdFOvXhAaevzr/+EHuOwydyB+8UW4/36IiYFPP4W+5XytRmYmLF0KP/4ISUnw5JMQHFx8mcREV+MYPtw1VbVsCYMGQc+eMGMGdOlyaNm4OHj4YZg0yR3ko6Phvvtg7FiXQPr1K33MDz0En38O113nkkRQkJuCg91rZqarAeVN69bBnj2wfTu8/DI89RTcdptbtqKogndorqgNarET8DbwIbAVd2nucuC/JZWrTFP37t21om3cs1GDngrSu6bdVeHbNlXDypWq112nKqIaEaF6662qvXurBgaqgmrt2qr9+qmOGaOanHxs25g+XTU0VLVjR9Xt2928tWtVTztNNSBA9cUXVXNzS7/e2bNVn3tO9c03VT/8UHXqVNU5c1R/+UX1m29UH3tM9bzz3D64w5qbPvqo5HW/+qpb9vffD837/XfVFi1U69Rx205JUR05UjUkxG3jiSdU09Lcvnz9tWq7dm4dAweqbtjg+35NmeLKDR9eut9DVTU2VvXss135Tp1Uv/229Os4Fs88oxoc7H6L+vVVmzZVbdNGtUMH1a5dVX/88djXDcRqYXmhsJmHLQC/H/EaAcwqqVxlmvyROG754hYNeTpE41PiK3zb1dX27aq7dqlmZpbP+nfsUJ0wQTUn5/jWs3+/6tChqt26qV57rer//Z/qxx+rLlqkunfv0QnjkUdUExIOlU9JcQe/4cPdAR5U27dXXbeudHFMmuQOKN27qyYmHv5daqrqlVe6dV9/vYvZV++842IvmBCOnAIDVWNiVEeMUJ082f3tOnZUPf30khNV9+7ugHekrVvdATkoSLVRI7edG29084+UkaE6erRqeLhqrVqqjz7q5hVnwwbVevVUzzhD9eBBn3+Ow+Tmuv1t29bFd+mlqqtXH9u6fPHcc4e289BDqvfeq3r77ao33aR69dWqAwaoLl587Os/nsSxyHtdCDTHPaVvXUnlKtNU0Ynjj8Q/NPDJQB3xzYgK3W51tXSp+w9Q8MAUEaHaqpVq586q55/vznKPVXa26htvuIMGuDP8Y7Vli0sYIi6uE090Z/ZHHlgLSxhFmTdPNTLSTfPn+xbHe++57fbp45JVYXJz3dmqiGp0tOqaNSWv9+23Xfz9+6vu2aO6c6dLaEuWuDinTVP9/nvVffuOLvvBB67s9OlFr3/FCrfMyy8X/v3eve4ged55rnZTkm3b3EEUVM88s/Ako+qSSkyM+zewcWPJ6y1JRob7d1S3rtt28+bu38Mdd7ha3rRp7nc7ltpenn//26372mtVs7KOP+bCHE/ieByoD1wF7MTdRf5USeVKWOdw3LPCVwIjvHkNgdnAOu+1QRFl+wN/AOuBkb5sr6ITx/VTrtewZ8J0576dFbrd6mbVKtVrrnH/SuvXd2fur72m+tRTqvfdpzpkiOrll7uzu9BQ1Z9/Lv02Fi92Z5ig2revOyAGB7sDYWn99JNqkyauOeXrrw/Nz8hwtYypU90Z4rPP+pYwClq3TvWUU9zZc3HNPVu3ujNPUL3oItd8U5Lp01UbNHBNPq++WnSN6/XXD53dlnT2XpjMTJfszzmn6GUeesjVKHbtKv36izN5skvWjRqpfvfd0d8PH+727fPPy3a7u3a5v/mQIao9e6o2bHj4CUSfPqoLFpR+vf/5jys/aFD51cBVjzFx4K666lXgcwhQr7gyJU1AJy9phOE65+cAJwNj8hIBMBJ4rpCygbjh3U/E3YT4G+5Kr0qTOFbsWqEySvShWQ9V2Darmw0bVP/yF3fGHBGh+o9/FN/Gv3u36kknuTPyP/7wbRspKe5gERDgDvbjx7uzv8RE1WbNXPtwerrvMb/7rks4J53kkkR52LPHnbWC6uOPHzpbzc11B8Mrr3RNRCLuLLs0B/dt21Qvvtit+4ILVDdvPvz7V17R/D6DY0kaeV5+2a2nsCSfne3OzC+77NjXX5zVq1VPPdX9zceMOfT7ff65HnO/xrFITHQnGS+95P7tgTtBWr/et/IffOD+xpdccnx/C18cT41jQUnLlGbCDZr4boHPjwMPebWIZt68ZsAfhZTtCXxb4PMjwCMlbbMiE8fVk67WiH9FaML+Up5SGlV1fQFBQa4G8cADLin4Yt061ago1zRU3NlqXht08+buP9/f/nZ0Upo1y/3PuOeekrebleXalfPO8Pfs8S3eY3XwoOtEz2uiePVVdzAElzgfflh106ZjW3dururYsS5Z162r+v77bt4LL7j1X3nlsbf959m3z9VuBg48+ruZM912Jk8+vm0UJzXVtf2D6lVXqf722/H3axyPfftcx35YmDvxGDas+NroxIku8fXtq3rgQPnHdzyJ40mvmUpKWtaXCegArAUivVrHAtzQ7HuPWC65kLJXH5F0bgJeL2I7Q4FYILZ169bl8qMe6Y/EP5RR6D+++0eFbK+6+fVXd2XIOee4M+DSWrjQNbeccUbhTTRbtrimLXBt+sW1kY8Y4ZabMaPoZeLiXFs7uKaz8mpnPlJurmv+yOugPvNMdxZaVgeSDRvc3wBcf03eGXFZNYk8/rhb56pVh88fPNgllfI+i87NVX3+eXcAFim7fo3jsX2769QOCHDxDBrkfvPrr3e177/+VfXmm12N8uyzfWuCLAvHkzj24QY5zARSvc+pJZUrYZ23AkuA+bjLfV/2MXEcWVu5CXitpO1VVI3jn/P+qYzCrqQ6Brt2ufbv1q19r2UU5ssv3X++AQMOHcizs92ZeUSESyzPP1/yQf7AAXcFT5MmR8eTm+vamOvUcev84INjj/d4/PijuwS0POTkuE7ckBB38CrLpLh7t/s73HLLoXkpKa6WeVcFXr3+/feqXboc3h/lbytXuhrR6ae7muRJJ7lLa1u0cP8WL77Y1ZoqyjEnjvKegH8Bf6sOTVVd3+6qPd/tedzrycpyTR6Jicd31UVVkZmpeu657sBxPJcO5nnrLfcv+447XFNEjx7u85/+VLozy99+c53Rl19+6O+webNrksrrCzjWZqGqIu/eiLJ2992uaSbvKqd333W/6cKFZb8tc+yKShy+PAHwnMLm6xEPdioNEWms7imCrYErvYTQFhgCjPZevyyk6K/AySLSFtgGXAdcf6xxlKUNezawdOdSXuz3os9lVOGJJ+DLL2HfPkhLc68ZGYeWqVULmjVzU/PmboqKckMqZGTAgQNuysiA3Fy49143pIQvvv7aDZFw442l3NlS7N+337phGs49t+jlHnwQ5s2Djz6Cbt2Of7t33umGp3j2WXcHcVQUjB8PgweX7u7azp1h9Gh3p/U777iyDzzgfuc334Q77oAAnx6+XHWFh5fPeh94AN5+293d/cILboiR9u2hR4/y2Z4pY4VlEz38LP/rAtNsIAX4vqRyJazzf8Aq3FVRfb15kcB3uMtxvwMaevObAzMKlL0E10eyAXjMl+1VRI3juR+fU0ahm5I3+bR8bq5rFwfXTn7jja6a/ve/u8tNX37ZTQ8/7K6QufBCdzNY/fquTECAu7kpMlK1ZUvVk092ncPBwa6DszjZ2W47eZcEPvOMb/u4ZInrIP3ss5Jvktu40Z3h523jwgtdH8aR8q7tHzHCtxh8lZurev/9qnfeefTNb6WRk+Niz9uPmlDLqCiDB7umvsWLS/fv0FQcyqqpCmgFTChtOX9OFZE4erzTQ2PGxvi8/KOPul9/2LDSNwVkZxdeZs+eQwfre+4pvDMzKelQU8tdd7mEBapPPln8Nr/80l35kTccRteu7iamI+PIynJ9CLVru4PCv//tLuWMitL8Tta8G81iY13z1PnnV1zH8rGIj3cxvvnm8d9Vbg5ZutT9m2jVynVSb9ni74jMkcoycQiwvLTl/DmVd+KIS45TRqGj/zfap+Wfftr98kOHln37cVaWu4wV3MGu4Nn2b7+5m+Vq1XJtyqouCQ0ZokfdG5AnN9d1koq4q5Xi490NaCee6MqcdZYbo0jV1Si6dnXzL7vs8ANBSoq77DAiwiWf224rm85wU7Xlnej07evvSExhjjlx4C6VfdWbXgd+BD4uqVxlmso7cbz080vKKHRdUsmDCY0Z4371IUPK9+z1gw/cFTFt26ouX+6u/w4Lc/cvHHmnak7OoXsDHnnkUPLIzHQdzHnXvBcczygz0zWJtWyp+Ze3BgS4AdYmTy46Ie7a5WpZwcFl1xluqq65c92/n08+8XckpjDHkziGFJhuAHqXVKayTeWdOHr/t7d2eatLicvljfp57bXuTL+8LVzo7oIODXXb7d3bDeRXmJycQ0ni7393N8XlNWmNHFl0kjtwwDVHdejgyvs6imtcnBuXyJgNG2rG1YNV0fEkjnAgsMDnQCCspHKVaSrPxBGfEq+MQp+e93SRy+zf74YXqIixZY60bZsbmnvYsJLvjM3NdZdJghtTJyhIddy4ionTGFP5FJU4fHnUyHfAhUCa97k2MAvo5UPZam/qmqkAXNXhqsPmJyfDtGkwdaq7JDU9HS65BCZOLPlBNmWpeXO3fV+IwGuvuYcHffwxzJrlnnBmjDEF+ZI4QlU1L2mgqmkiElaOMVUpU1ZP4bRGp9GhUQcyMtyjLidPdk9dy852B+6bb4YrrnAH4cBAf0dcPBF3Xf3zz1fwE8WMMVWGL4ljv4h0U9UlACLSHThQvmFVDbvSdjF/83we6/0448fDY4/B5s1wyinuBqcrroAzzqiaN4lZ0jDGFMWXxDEC+ExEtnufmwHXlltEVcgXa74gd8O5TJ7yEKt/d3c9jxvnahZ24DXGVFclJg5V/VVETgXa4+7hWKOqWeUeWSW3ciX847Zo+O0O9rdWPv7YDWlRFWsXxhhTGiUe5kTkbiBcVVeo6nIgQkT+Vv6hVV4//ACdOyuJf7Tn/Nu/4Y8/hBtusKRhjKkZfDnU3a6qe/M+qGoycHu5RVQFfPghhIZnwrB2PD+qMaGh/o7IGGMqji99HAEiIt41vYhIIO6xrTVSbi5Mnw71Oi2gcfO6dGtWBsO5GmNMFeJLjeNbYJKI9BWRC4AJwDflG1bl9euvkJAAu1uM4+oOVyPWC26MqWF8qXE8jHsM6124zvGluCuraqRp0yAgMJecE6dxRYdp/g7HGGMqXIk1DlXNBRYCG4EYoC+wupzjqrSmTYM2p2+HsGQ6RHXwdzjGGFPhiqxxiMgpuCfsDQaSgE8BVLXGDkIRHw/LlsEFQ1ewWQKpF1rP3yEZY0yFK66pag3uSX2Xqep6ABG5r0KiqqSmT3evUdELidwfSYDY9bfGmJqnuMRxFa7G8YOIzAQm4vo4jpuXgG4DFFgO3AJ8gLvJEKA+sFdVowspGwfsA3KAbFWNKYuYfDFtGpx4ImQ3XEGURlXUZo0xplIpMnGo6lRgqoiEA4OA+4AmIvIWMFVVZx3LBkWkBTAMOE1VD4jIJOA6Vb22wDIv4p5tXpTzVTXxWLZ/rNLTYc4cGDoUfjuQSGTtyIrcvDHGVBq+dI7vV9XxqjoAaAksA0Ye53aDgNoiEgSEAXnjYCHu+tY/4y77rTR++AEyMmDAAEhMTyQqzGocxpiaqVSN9Kq6R1X/o6oXHOsGVXUb8AKwBdgBpBxRezkb2KWq64paBTBLRBaLyNCitiMiQ0UkVkRiExISjjXcfF9/DRERcM45kHQgyRKHMabGqvDeXRFpAAwE2gLNgXARubHAIoMpvrbRW1W7ARcDd4vIOYUtpKpjVTVGVWMaNWp0XDGruv6Nfv2gVi21Gocxpkbzx2VBFwKbVDXBG2X3c7ynCXpNV1fiXfpbGFXd7r3uBqYCPco74N9+g23bXDNV6sFUsnOzLXEYY2osfySOLcBZIhLm9WcUvKHwQtyw7fGFFRSRcBGpk/ce6AesKO+Ap3k3iF9yievfACxxGGNqrApPHKr6CzAZWIK7FDcAGOt9fR1HNFOJSHMRmeF9bAL8KCK/AYuA6ao6s7xjnjYNevSAJk0OJQ67qsoYU1P5MlZVmVPVJ4AnCpl/cyHztgOXeO83Al3KO76Cdu2CRYvgySfd56QDSYDVOIwxNZfd+lyCb75xneMDBrjP1lRljKnpLHGUYNo0aN4coqPdZ0scxpiazhJHMTIz4dtvXW0j77EbiemJBAUEUTekrn+DM8YYP7HEUYz58yEt7VAzFbjEEVk70h7gZIypsSxxFGPaNAgNhb59D82zm/+MMTWdX66qqiquvhrat4ewsEPzbLgRY0xNZ4mjGH36uKmgxPREe/KfMaZGs6aqUrKmKmNMTWeJoxRyNZek9CS7a9wYU6NZ4iiFlIwUcjTHahzGmBrNEkcp2HAjxhhjiaNU7K5xY4yxxFEqljiMMcYSR6nkD6keZp3jxpiayxJHKViNwxhjLHGUSlJ6EsEBwdSpVcffoRhjjN9Y4iiFvJv/bIBDY0xN5pfEISL3ichKEVkhIhNEJFRERonINhFZ5k2XFFG2v4j8ISLrRWRkRcadeMDuGjfGmApPHCLSAhgGxKhqJyAQ96xxgJdVNdqbZhRSNhB4A7gYOA0YLCKnVVDoNtyIMcbgv6aqIKC2iAQBYcB2H8v1ANar6kZVzQQmAgPLKcajJKYn2hVVxpgar8ITh6puA14AtgA7gBRVneV9fY+I/C4i40SkQSHFWwBbC3yO9+YdRUSGikisiMQmJCSUSeyJ6YlE1bYahzGmZvNHU1UDXC2hLdAcCBeRG4G3gHZANC6hvFhY8ULmaWHbUdWxqhqjqjGNGjU67rhzNZc9B/ZYU5UxpsbzR1PVhcAmVU1Q1Szgc6CXqu5S1RxVzQXewTVLHSkeaFXgc0t8b+Y6Lnsz9pKruZY4jDE1nj8SxxbgLBEJE3dda19gtYg0K7DMFcCKQsr+CpwsIm1FpBauU/2rco8Yu/nPGGPyVPgTAFX1FxGZDCwBsoGlwFjgXRGJxjU9xQF3AIhIc+BdVb1EVbNF5B7gW9zVWONUdWVFxG3DjRhjjOOXR8eq6hPAE0fMvqmIZbcDlxT4PAM46lLd8mY1DmOMcezOcR8lpduzOIwxBixx+MxqHMYY41ji8FFieiIhgSGEB4f7OxRjjPErSxw+yrtr3AY4NMbUdJY4fGQDHBpjjGOJw0dJ6UmWOIwxBkscPrORcY0xxrHE4SMb4NAYYxxLHD7Iyc1hz4E9dte4McZgicMnyRnJKGpNVcYYgyUOn9jNf8YYc4glDh/YcCPGGHOIJQ4fWI3DGGMOscThA0scxhhziF+GVa9q8p/FUduuqjI1Q1ZWFvHx8WRkZPg7FFMBQkNDadmyJcHBwT4tb4nDB4npiYQGhRIWHObvUIypEPHx8dSpU4c2bdrY+GzVnKqSlJREfHw8bdu29amMNVX5IOmAG27E/gOZmiIjI4PISBvUsyYQESIjI0tVu/RL4hCR+0RkpYisEJEJIhIqIs+LyBoR+V1EpopI/SLKxonIchFZJiKxFRGvDTdiaiJLGjVHaf/WFZ44RKQFMAyIUdVOuGeHXwfMBjqpamdgLfBIMas5X1WjVTWm3APGEocxxhTkr6aqIKC2iAQBYcB2VZ2lqtne9wuBln6K7SiJ6YnWMW5MBUpKSiI6Opro6GiaNm1KixYt8j9nZmYWWzY2NpZhw4aVuI1evXqVVbgADB8+nBYtWpCbm1um662MKrxzXFW3icgLwBbgADBLVWcdsdhfgU+LWgUwS0QU+I+qji1sIREZCgwFaN269XHFbDUOYypWZGQky5YtA2DUqFFERETw4IMP5n+fnZ1NUFDhh6+YmBhiYkpujPj555/LJFaA3Nxcpk6dSqtWrZg/fz7nnXdema27oJycHAIDA8tl3aVR4YlDRBoAA4G2wF7gMxG5UVU/9r5/DMgGxhexit6qul1EGgOzRWSNqs4/ciEvoYwFiImJ0WONNzs3m70Zey1xmBprxMwRLNu5rEzXGd00mlf6v1KqMjfffDMNGzZk6dKldOvWjWuvvZYRI0Zw4MABateuzXvvvUf79u2ZO3cuL7zwAtOmTWPUqFFs2bKFjRs3smXLFkaMGJFfG4mIiCAtLY25c+cyatQooqKiWLFiBd27d+fjjz9GRJgxYwb3338/UVFRdOvWjY0bNzJt2rSjYvvhhx/o1KkT1157LRMmTMhPHLt27eLOO+9k48aNALz11lv06tWLDz/8kBdeeAERoXPnznz00UfcfPPNDBgwgKuvvvqo+J588kmaNWvGsmXLWLVqFYMGDWLr1q1kZGQwfPhwhg4dCsDMmTN59NFHycnJISoqitmzZ9O+fXt+/vlnGjVqRG5uLqeccgoLFy4kKurYj2n+uBz3QmCTqiYAiMjnQC/gYxEZAgwA+qpqoQd7Vd3uve4WkalAD+CoxFFWkg/YAIfGVBZr165lzpw5BAYGkpqayvz58wkKCmLOnDk8+uijTJky5agya9as4YcffmDfvn20b9+eu+6666j7FZYuXcrKlStp3rw5vXv35qeffiImJoY77riD+fPn07ZtWwYPHlxkXBMmTGDw4MEMHDiQRx99lKysLIKDgxk2bBjnnnsuU6dOJScnh7S0NFauXMkzzzzDTz/9RFRUFHv27ClxvxctWsSKFSvyL5cdN24cDRs25MCBA5xxxhlcddVV5Obmcvvtt+fHu2fPHgICArjxxhsZP348I0aMYM6cOXTp0uW4kgb4J3FsAc4SkTBcU1VfIFZE+gMPA+eqanphBUUkHAhQ1X3e+37AU+UZrN01bmq60tYMytM111yT31STkpLCkCFDWLduHSJCVlZWoWUuvfRSQkJCCAkJoXHjxuzatYuWLQ/vQu3Ro0f+vOjoaOLi4oiIiODEE0/MP1gPHjyYsWOPbhnPzMxkxowZvPzyy9SpU4czzzyTWbNmcemll/L999/z4YcfAhAYGEi9evX48MMPufrqq/MP3g0bNixxv3v06HHYPRavvvoqU6dOBWDr1q2sW7eOhIQEzjnnnPzl8tb717/+lYEDBzJixAjGjRvHLbfcUuL2SuKPPo5fRGQysATXJLUU16S0EgjBNT8BLFTVO0WkOfCuql4CNAGmet8HAZ+o6szyjNfuGjem8ggPD89///jjj3P++eczdepU4uLiiuxXCAkJyX8fGBhIdna2T8sU0ehxlJkzZ5KSksLpp58OQHp6OmFhYVx66aWFLq+qhV7+GhQUlN+xrqqHXQRQcL/nzp3LnDlzWLBgAWFhYZx33nlkZGQUud5WrVrRpEkTvv/+e3755RfGjy+qF8B3frmqSlWfUNVTVbWTqt6kqgdV9SRVbeVdZhutqnd6y273kgaqulFVu3hTR1V9prxjtRqHMZVTSkoKLVq0AOD9998v8/WfeuqpbNy4kbi4OAA+/bTw63UmTJjAu+++S1xcHHFxcWzatIlZs2aRnp5O3759eeuttwDXsZ2amkrfvn2ZNGkSSUlu1O28pqo2bdqwePFiAL788ssia1ApKSk0aNCAsLAw1qxZw8KFCwHo2bMn8+bNY9OmTYetF+C2227jxhtv5M9//nOZdK7bneMlsMRhTOX00EMP8cgjj9C7d29ycnLKfP21a9fmzTffpH///vTp04cmTZpQr169w5ZJT0/n22+/Pax2ER4eTp8+ffj666/597//zQ8//MDpp59O9+7dWblyJR07duSxxx7j3HPPpUuXLtx///0A3H777cybN48ePXrwyy+/HFbLKKh///5kZ2fTuXNnHn/8cc466ywAGjVqxNixY7nyyivp0qUL1157bX6Zyy+/nLS0tDJppgIQX6tjVVlMTIzGxh7bTeajfxzNI989wv5H99tYVabGWL16NR06dPB3GH6XlpZGREQEqsrdd9/NySefzH333efvsEotNjaW++67j//9739FLlPY31xEFhd2o7XVOEqQmJ5IWHCYJQ1jaqB33nmH6OhoOnbsSEpKCnfccYe/Qyq10aNHc9VVV/Hss8+W2TptdNwS2F3jxtRc9913X5WsYRQ0cuRIRo4cWabrtBpHCeyucWOMOZwljhJY4jDGmMNZ4ihB3rM4jDHGOJY4SmA1DmOMOZx1jhcjKyfLBjg0xg+SkpLo27cvADt37iQwMJBGjRoBbtymWrVqFVt+7ty51KpVq9ih0wcOHMju3btZsGBB2QVeQ1jiKMaeA+7OS7uqypiKVdKw6iWZO3cuERERRSaOvXv3smTJEiIiIti0aZPPz9oureKGf6/Kqt8elSG7a9wYGDECvGN4mYmOhldeKV2ZxYsXc//995OWlkZUVBTvv/8+zZo149VXX+Xtt98mKCiI0047jdGjR/P2228TGBjIxx9/zGuvvcbZZ5992LqmTJnCZZddRpMmTZg4cSKPPOIeOLp+/XruvPNOEhISCAwM5LPPPqNdu3aMGTOGjz76iICAAC6++GJGjx7NeeedxwsvvEBMTAyJiYnExMQQFxfH+++/z/Tp08nIyGD//v189dVXDBw4kOTkZLKysvjnP//JwIEDAY4aXv3NN9+kc+fOrF27luDgYFJTU+ncuTPr1q07akRff7LEUYykA24sGUscxviXqnLvvffy5Zdf0qhRIz799FMee+wxxo0bx+jRo9m0aRMhISHs3buX+vXrc+eddxZbS5kwYQJPPPEETZo04eqrr85PHDfccAMjR47kiiuuICMjg9zcXL755hu++OILfvnlF8LCwnwaBn3BggX8/vvvNGzYkOzsbKZOnUrdunVJTEzkrLPO4vLLL2fVqlVHDa9ep04dzjvvPKZPn86gQYOYOHEiV111VaVKGmCJo1hW4zCm9DWD8nDw4EFWrFjBRRddBLgBA5s1awZA586dueGGGxg0aBCDBg0qcV27du1i/fr19OnTBxEhKCiIFStWcMIJJ7Bt2zauuOIKAEJDQwGYM2cOt9xyC2FhbvQIX4ZBv+iii/KXU1UeffRR5s+fT0BAANu2bWPXrl18//33hQ6vfttttzFmzBgGDRrEe++9xzvvvFOKX6piWOIohiUOYyoHVaVjx46FdmRPnz6d+fPn89VXX/H000+zcuXKYtf16aefkpycnN+vkZqaysSJE3nooYeK3HZJw6BnZGQc9l3BAQrHjx9PQkICixcvJjg4mDZt2hQ7DHrv3r2Ji4tj3rx55OTk0KlTp2L3xx/sctxi5D+LI8w6x43xp5CQEBISEvITR1ZWFitXriQ3N5etW7dy/vnnM2bMGPbu3UtaWhp16tRh3759ha5rwoQJzJw5M38Y9MWLFzNx4kTq1q1Ly5Yt+eKLLwBXy0lPT6dfv36MGzeO9HT3fLnChkGfPHlykbGnpKTQuHFjgoOD+eGHH9i8eTNAkcOrA/zlL39h8ODBZTaabVmzxFGMxPREwoPDCQ0K9XcoxtRoAQEBTJ48mYcffpguXboQHR3Nzz//TE5ODjfeeCOnn346Xbt25b777qN+/fpcdtllTJ06lejo6MNGhI2Li2PLli35Q5EDtG3blrp16/LLL7/w0Ucf8eqrr9K5c2d69erFzp076d+/P5dffjkxMTFER0fzwgsvAPDggw/mP0M8MTGxyNhvuOEGYmNjiYmJYfz48Zx66qkARQ6vnlcmOTm52MfV+pMNq16Md5e8y4KtC/jvwP+WQ1TGVF42rLp/TZ48mS+//JKPPvqowrZZmmHV/dLHISL3AbcBCiwHbgHCgE+BNkAc8GdVTS6kbH/g30Ag7pGyo8srztu63cZt3W4rr9UbY8xR7r33Xr755htmzJjh71CKVOFNVSLSAhgGxKhqJ1wCuA4YCXynqicD33mfjywbCLwBXAycBgwWkdMqKnZjjClvr732GuvXr+eUU07xdyhF8lcfRxBQW0SCcDWN7cBA4APv+w+AQYWU6wGs9549nglM9MoZY8pYTWjGNk5p/9YVnjhUdRvwArAF2AGkqOosoImq7vCW2QE0LqR4C2Brgc/x3ryjiMhQEYkVkdiEhISy3AVjqr3Q0FCSkpIsedQAqkpSUlL+fSu+qPA+DhFpgKsltAX2Ap+JyI2+Fi9kXqH/slV1LDAWXOd46SM1puZq2bIl8fHx2ElXzRAaGkrLli19Xt4fneMXAptUNQFARD4HegG7RKSZqu4QkWbA7kLKxgOtCnxuiWvmMsaUoeDg4HIb+M9Uff7o49gCnCUiYeJum+wLrAa+AoZ4ywwBviyk7K/AySLSVkRq4TrVv6qAmI0xxngqvMahqr+IyGRgCZANLMU1KUUAk0TkVlxyuQZARJrjLru9RFWzReQe4Fvc1VjjVLX48QWMMcaUKbsB0BhjTKGKugGwRiQOEUkANpewWBRQ9LgB1Zftd81i+13zHM++n6CqjY6cWSMShy9EJLawzFrd2X7XLLbfNU957LsNcmiMMaZULHEYY4wpFUsch4z1dwB+Yvtds9h+1zxlvu/Wx2GMMaZUrMZhjDGmVCxxGGOMKZUanzhEpL+I/CEi60XkqGeAVCciMk5EdovIigLzGorIbBFZ57028GeM5UFEWonIDyKyWkRWishwb3613ncRCRWRRSLym7ffT3rzq/V+g3t2j4gsFZFp3udqv88AIhInIstFZJmIxHrzynzfa3TiqIEPhnof6H/EvBIfoFUNZAMPqGoH4Czgbu/vXN33/SBwgap2AaKB/iJyFtV/vwGG48bAy1MT9jnP+aoaXeDejTLf9xqdOKhhD4ZS1fnAniNm+/IArSpNVXeo6hLv/T7cAaUF1Xzf1UnzPgZ7k1LN91tEWgKXAu8WmF2t97kEZb7vNT1x+PxgqGrMlwdoVRsi0gboCvxCDdh3r8lmGe4xBbNVtSbs9yvAQ0BugXnVfZ/zKDBLRBaLyFBvXpnvuz+ex1GZ+PxgKFP1iUgEMAUYoaqpblT/6k1Vc4BoEakPTBWRTn4OqVyJyABgt6ouFpHz/ByOP/RW1e0i0hiYLSJrymMjNb3GYQ+G8h6gBVDMA7SqPBEJxiWN8ar6uTe7Ruw7gKruBebi+riq8373Bi4XkThc0/MFIvIx1Xuf86nqdu91NzAV1xxf5vte0xOHPRjKtwdoVWneA8P+C6xW1ZcKfFWt911EGnk1DUSkNu7pm2uoxvutqo+oaktVbYP7//y9qt5INd7nPCISLiJ18t4D/YAVlMO+1/g7x0XkElybaN6DoZ7xb0TlR0QmAOfhhlneBTwBfAFMAlrjPUBLVY/sQK/SRKQP8D9gOYfavR/F9XNU230Xkc64ztBA3EniJFV9SkQiqcb7ncdrqnpQVQfUhH0WkRNxtQxw3RCfqOoz5bHvNT5xGGOMKZ2a3lRljDGmlCxxGGOMKRVLHMYYY0rFEocxxphSscRhjDGmVCxxGHMcRCTHG4k0byqzwfNEpE3BkYyNqSxq+pAjxhyvA6oa7e8gjKlIVuMwphx4z0V4znsexiIROcmbf4KIfCciv3uvrb35TURkqvfsjN9EpJe3qkARecd7nsYs7w5wRGSYiKzy1jPRT7tpaihLHMYcn9pHNFVdW+C7VFXtAbyOG50A7/2HqtoZGA+86s1/FZjnPTujG7DSm38y8IaqdgT2Ald580cCXb313Fk+u2ZM4ezOcWOOg4ikqWpEIfPjcA9R2ugNsLhTVSNFJBFopqpZ3vwdqholIglAS1U9WGAdbXBDoZ/sfX4YCFbVf4rITCANN2TMFwWeu2FMubMahzHlR4t4X9QyhTlY4H0Oh/olL8U9vbI7sFhErL/SVBhLHMaUn2sLvC7w3v+MG7UV4AbgR+/9d8BdkP/wpbpFrVREAoBWqvoD7oFF9YGjaj3GlBc7SzHm+NT2nrCXZ6aq5l2SGyIiv+BO0AZ784YB40Tk70ACcIs3fzgwVkRuxdUs7gJ2FLHNQOBjEamHexjZy97zNoypENbHYUw58Po4YlQ10d+xGFPWrKnKGGNMqViNwxhjTKlYjcMYY0ypWOIwxhhTKpY4jDHGlIolDmOMMaViicMYY0yp/D+pmdyCqyYfMAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plotting the accuracy graph of training and test data on rely activation network\n",
    "epochs = range(1,51)\n",
    "plt.plot(epochs, relu_training_accuracy_per_epoch, 'g', label='Training Accuracy')\n",
    "plt.plot(epochs, relu_test_accuracy_per_epoch, 'b', label='Test Accuracy')\n",
    "plt.title('Training and Test accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9OElEQVR4nO3dd3hUZfbA8e9JJZBAqEYIEBBQqgGyWJGgrrKAi7h2dAELylrWtrb97dpldV0LigUEQVFARRSVtYBSRBcJSAdRIEjoBBI6pJzfH+8kBEjCBHIzycz5PM99ZubOO/eeS3TO3LeKqmKMMSZ0hQU6AGOMMYFlicAYY0KcJQJjjAlxlgiMMSbEWSIwxpgQZ4nAGGNCnCUCU6mIyH9FpH95lw0kEUkXkQsDHYcxJbFEYE6YiOwusuWLyL4ir/uV5Viq+gdVHVPeZSsjXyIr+HfKEZGDRV6/fhzHe1RExh6jjCUlc5SIQAdgqj5VjS14LiLpwE2qOvXIciISoaq5FRlbZaaqfyh4LiKjgQxV/b/ARWRCld0RGM+ISKqIZIjIAyKyCXhLRGqLyGcislVEdvieJxb5zHQRucn3fICIfCciz/nKrhGRPxxn2WYiMlNEdonIVBEZVtKvZz9jfEJEZvuO95WI1Cvy/vUislZEMkXk78f5b9dbRBaISJaIfC8iHYq894CIrPed+2cRuUBEegAPA1f57igWlvF80SLyoohs8G0viki07716vn+DLBHZLiKzRCSspFiO53pNYFkiMF5LAOoATYFBuP/m3vK9bgLsA14p5fNnAD8D9YBngZEiIsdR9j3gR6Au8ChwfSnn9CfGa4GBQAMgCrgPQETaAK/5jt/Qd75EykBEOgGjgFt8n38DmOz7sj4VuB34narGARcD6ar6BfA0MEFVY1X19LKcE/g7cCaQDJwOdAEK7k7uBTKA+sBJuISjJcVSxvOaSsASgfFaPvCIqh5Q1X2qmqmqE1V1r6ruAp4CupXy+bWqOkJV84AxwMm4LyO/y4pIE+B3wD9V9aCqfgdMLumEfsb4lqquVNV9wPu4L1CAy4HPVHWmqh4A/uH7NyiLm4E3VHWOqub52kEO4L6o84BooI2IRKpquqquKuPxi9MPeFxVt6jqVuAxDiXLHNy/ZVNVzVHVWeomKfMqFlPBLBEYr21V1f0FL0Skuoi84as62QnMBOJFJLyEz28qeKKqe31PY8tYtiGwvcg+gHUlBexnjJuKPN9bJKaGRY+tqnuAzJLOVYKmwL2+qpgsEckCGgMNVfVX4C7cXc0WERkvIg3LePziNATWFnm91rcP4N/Ar8BXIrJaRB4E8DAWU8EsERivHTm97b3AqcAZqloTOM+3v6TqnvKwEagjItWL7GtcSvkTiXFj0WP7zlm3bOGyDnhKVeOLbNVVdRyAqr6nqufiEoYCz/g+dyJTCW/wHa9AE98+VHWXqt6rqs2BS4B7CtoCSonFVCGWCExFi8PVuWeJSB3gEa9PqKprgTTgURGJEpGzcF9oXsT4IdBbRM4VkSjgccr+/9kI4FYROUOcGiLSS0TiRORUETnf15C73xdnnu9zm4GkgobcUkSKSLUiWwQwDvg/Eanva/j+JzAWChuuW/jaW3b6zpd3jFhMFWKJwFS0F4EYYBvwP+CLCjpvP+AsXDXNk8AEXL17cV7kOGNU1aXAbbjG6Y3ADlxDq99UNQ3XTvCK7/O/AgN8b0cD//LFtgnXWP2w770PfI+ZIjK/lFNMwX1pF2yP4v5N0oBFwGJgvm8fQEtgKrAb+AF4VVWnHyMWU4WILUxjQpGITABWqKrndyTGVHZ2R2BCgoj8TkROEZEwX5/7PsDHAQ7LmErBRhabUJEAfIRruM0ABqvqT4ENyZjKwaqGjDEmxFnVkDHGhLgqVzVUr149TUpKCnQYxhhTpcybN2+bqtYv7r0qlwiSkpJIS0sLdBjGGFOliMjakt6zqiFjjAlxlgiMMSbEWSIwxpgQV+XaCIqTk5NDRkYG+/fvP3Zh47lq1aqRmJhIZGRkoEMxxvghKBJBRkYGcXFxJCUlUfKaJaYiqCqZmZlkZGTQrFmzQIdjjPFDUFQN7d+/n7p161oSqAREhLp169rdmTFVSFAkAsCSQCVifwtjqpagSQTGGFNVbN8Oo0ZBZZnhxxJBOcjMzCQ5OZnk5GQSEhJo1KhR4euDBw+W+tm0tDTuvPPOY57j7LPPLpdYp0+fTu/evcvlWMaY4/Pcc3DjjTBrVqAjcYKisTjQ6taty4IFCwB49NFHiY2N5b777it8Pzc3l4iI4v+pU1JSSElJOeY5vv/++3KJ1RgTWKowfrx7/t57cN55pZevCHZH4JEBAwZwzz330L17dx544AF+/PFHzj77bDp27MjZZ5/Nzz//DBz+C/3RRx/lhhtuIDU1lebNmzN06NDC48XGxhaWT01N5fLLL+e0006jX79+FMwgO2XKFE477TTOPfdc7rzzzjL98h83bhzt27enXbt2PPDAAwDk5eUxYMAA2rVrR/v27XnhhRcAGDp0KG3atKFDhw5cffXVJ/6PZUwISUuDNWugTh344AM4RqVBhfDsjkBEGgNv4+aBzweGq+pLR5RJBT4B1vh2faSqj5/Iee/64i4WbFpwIoc4SnJCMi/2eLHMn1u5ciVTp04lPDycnTt3MnPmTCIiIpg6dSoPP/wwEydOPOozK1as4Ntvv2XXrl2ceuqpDB48+Kj++D/99BNLly6lYcOGnHPOOcyePZuUlBRuueUWZs6cSbNmzbjmmmv8jnPDhg088MADzJs3j9q1a3PRRRfx8ccf07hxY9avX8+SJUsAyMrKAuBf//oXa9asITo6unCfMcY/48dDZCQMHQrXXQdffgmXlLaCdgXw8o4gF7hXVVsDZwK3iUibYsrNUtVk33ZCSaCyueKKKwgPDwcgOzubK664gnbt2nH33XezdOnSYj/Tq1cvoqOjqVevHg0aNGDz5s1HlenSpQuJiYmEhYWRnJxMeno6K1asoHnz5oV998uSCObOnUtqair169cnIiKCfv36MXPmTJo3b87q1au54447+OKLL6hZsyYAHTp0oF+/fowdO7bEKi9jzNHy8+H996FHD7jySqhb11UPBZpn/xer6kbc4t2o6i4RWQ40ApZ5dU7guH65e6VGjRqFz//xj3/QvXt3Jk2aRHp6OqmpqcV+Jjo6uvB5eHg4ubm5fpU5kQWGSvps7dq1WbhwIV9++SXDhg3j/fffZ9SoUXz++efMnDmTyZMn88QTT7B06VJLCMb44fvvISMD/vUvd1dw5ZUwejTs3g2+2t+AqJA2AhFJAjoCc4p5+ywRWSgi/xWRtiV8fpCIpIlI2tatW70M1TPZ2dk0atQIgNGjR5f78U877TRWr15Neno6ABMmTPD7s2eccQYzZsxg27Zt5OXlMW7cOLp168a2bdvIz8/nT3/6E0888QTz588nPz+fdevW0b17d5599lmysrLYvXt3uV+PMcFowgSoVg3++Ef3+tprYd8++OSTwMbl+c84EYkFJgJ3qerOI96eDzRV1d0i0hO3mHjLI4+hqsOB4QApKSmVpOdt2dx///3079+f559/nvPPP7/cjx8TE8Orr75Kjx49qFevHl26dCmx7LRp00hMTCx8/cEHHzBkyBC6d++OqtKzZ0/69OnDwoULGThwIPn5+QAMGTKEvLw8rrvuOrKzs1FV7r77buLj48v9eowJNnl5rnG4Vy+Ii3P7zj4bmjRx1UP9+gUuNk/XLBaRSOAz4EtVfd6P8ulAiqpuK6lMSkqKHrkwzfLly2nduvUJRlv17d69m9jYWFSV2267jZYtW3L33XcHJBb7m5hQceAADB4MLVrAww+XXO6bb+CCC1wyuPzyQ/sfegj+/W/YuBHqF7t+WPkQkXmqWmxfdc+qhsTNMzASWF5SEhCRBF85RKSLL55Mr2IKdiNGjCA5OZm2bduSnZ3NLbfcEuiQjAlqBw/CFVfAW2/B//0ffPddyWXHj4caNaBnz8P3X3vtobuFQPHsjkBEzgVmAYtx3UcBHgaaAKjq6yJyOzAY18NoH3CPqpY6csruCKoG+5uYYJeT45LAJ5/Af/4DL78M0dGwYIFrBziybEKC6y307rtHH6t9e6hVq/REcqJKuyPwstfQd0Cps4+p6ivAK17FYIwxXsjJgWuucUng5Zfh9tuhQwf4/e/hscdgyJDDy0+d6uYXuuqq4o937bWuWik9HZKSvI7+aDay2BgTFDIzXQ8cr+XmuobdiRPhxRddEgC48EI3f9C//w3z5x/+mQkT3C/+iy8u/pgFA/QLpp6oaJYIjDFVniqcdZb7IvZSbi5cf72rz//Pf+Cvfz38/eeegwYN4IYb3F0DwP79MGkS9O3rqo6K06yZ60FUXLVRRbBEYIyp8lavhl9+cV/QGzd6cw5V9wU/fjw88wzcc8/RZeLj4bXXYOFCVwbcFBI7dx761V+Sfv1gyRJYvLjcQz8mSwTl4ESmoQY3kVxJs4uOHj2a2wvuPY0xxZoxwz3m5sKbb3pzjpEj4Z134NFH4f77Sy7Xp49rC3jiCVi2zCWOunXhWMOHrrgCwsMDM+WEJYJyUDAN9YIFC7j11lu5++67C19HRUUd8/OlJQJjzLFNn+764P/+9zB8uEsI5WntWncH0L07/OMfxy7/8stu0NiAAfDpp27cwBFzRx6lfn246CIYN87NSVSRLBF4ZN68eXTr1o3OnTtz8cUXs9F3v3rkFM7p6em8/vrrvPDCCyQnJzPLz5Uqnn/+edq1a0e7du148cUXAdizZw+9evXi9NNPp127doXTTDz44IOF5yy6ToIxwUDV3RF06wZ/+Yuby+fzz8vv+Pn5rkpI1a0qFubHt2b9+m520blzYc+eknsLHenaa13SKc/4/RF0M4XddZfrx1uekpNd7wB/qSp33HEHn3zyCfXr12fChAn8/e9/Z9SoUUdN4RwfH8+tt9561GI2pZk3bx5vvfUWc+bMQVU544wz6NatG6tXr6Zhw4Z87vuvKDs7m+3btzNp0iRWrFiBiNi00SbopKfDb7/B3/4GvXtDYqKrp+/Tp3yO//rrblTwG2+UrWvnNde4mUYXLPB/8ZnLLnMT0l1/PcyeDW2LnX2t/NkdgQcOHDjAkiVL+P3vf09ycjJPPvkkGRkZQPlM4fzdd9/Rt29fatSoQWxsLJdddhmzZs2iffv2TJ06lQceeIBZs2ZRq1YtatasSbVq1bjpppv46KOPqF69enleqjEBV9A+kJoKERFw882ugXbVqhM/9qpVLsFcfLE7blmIHEoEvtnoj6l6dZgyBWJi3AjkTZvKHPJxCbo7grL8cveKqtK2bVt++OGHo94rbgrn4zl+cVq1asW8efOYMmUKDz30EBdddBH//Oc/+fHHH5k2bRrjx4/nlVde4ZtvvinzOY2prKZPd42xbXyrndx4Izz+uPsF/+yzx3/c/HwYONDV7b/5pvtiL6uoKLeVRZMm8Nln7i7ikkvc9RWZ0d4TdkfggejoaLZu3VqYCHJycli6dGmJUzjHxcWxa9cuv49/3nnn8fHHH7N371727NnDpEmT6Nq1Kxs2bKB69epcd9113HfffcyfP5/du3eTnZ1Nz549efHFFwvXVjYmWBS0DxTU3Tdq5KqFRo1yffiP10svucXlX3rJVTdVpM6dXW+j+fMPzUXkJUsEHggLC+PDDz/kgQce4PTTTyc5OZnvv/++cArn9u3b07Fjx8IpnC+55BImTZpUYmPx6NGjSUxMLNwaNGjAgAED6NKlC2eccQY33XQTHTt2ZPHixXTp0oXk5GSeeuop/u///o9du3bRu3dvOnToQLdu3QrXHTYmGKxd69oIunU7fP/gwW6k8YcfHt9xf/7ZTflwySXw5z+fcJjH5ZJLXIPz5MnFj1koT55OQ+0Fm3SuarC/iakIb78N/fu7AVwdOhzan58Pp53meu/Mnl22Y+bmwrnnwsqVsHQpnHxy+cZcVvfeC88/76q9jxzJXBYBmYbaGGO8NmMG1KkD7dodvj8sDG691S0NuWiR/8dTdZ+bMweGDQt8EgA3d1HfvnD33d6tZGaJwBhTZU2fDl27Ft+3f8AANx30a6/5dyxV10No5Eg3aOyaa8oz0uMXFgZjx7q5lLzqRRQ0iaCqVXEFM/tbmIqwbp2bYyg1tfj369RxA7nGjgV/+mI8/bSbSO6OO9xU0pVJ9eowcyZ4tdZUUCSCatWqkZmZaV9AlYCqkpmZSbUjV+YwIWHlSqioMYsF4weObCguavBg2L3bJYPSDBvmVhi7/npXF388XUW95u9YhOMRFOMIEhMTycjIYOvWrYEOxeASc2JF97czAZedDZ06ua6Ws2Yd3/q7+fnul+9777npGd5/H1q2LL7sjBluts+ijcRH6tIFOnaEBx90Db8DB7oYi37Rv/uuW1Pgj3901UL+TCERdFS1Sm2dO3dWY0zl8/rrqqAaGanaubNqdrZ/n8vPV503T/Xee1UbNXLHqFHDbd26qeblFf+5Fi1UL7nk2Mdftkz1qqtUo6Pdsdu3V33+edUtW1QnT1YND1ft3l113z6/L7VKAtK0hO/VUMx9xhg/5ObCjz/6X37kSLf27kcfuWkVLr209AFdqm5a59at3QCqoUMPDaTavNlV0cyY4QaGHWnDBvj115LbB4pq3dodc+NGVwUUHe365Tds6GYF7dTJ9cYJ6drMkjJEZd3sjsCYivHii+4X9LRpxy67aJEr++KL7vXYse51nz6qOTlHl1+3TrVnT1cmJUV1+HDVzMzDy+TnuzuC+HjVjRsPf++999xn09KO58pUFy92dyB9+qhu23Z8x6hqKOWOIOBf7GXdLBEY4738fNXWrd03xAUXHLv8XXepRkUd/qU6dKj7/IAB7ngFxx0+XLVmTdXq1V2Zkqp+VFV//tlV6VxxxeH7Bw1yx8jNLfu1harSEoFVDRljjjJ7Nixf7qpqpk1zDbclOXDAVfH06eMmfytwxx3wyCMwerTrn79mjVs4ZtAgSElxSzLecUfpjbOtWsE//+mWoJw8+dD+GTPc+AEve9KEEksExpijjBjhVtj67DOoXRuGDCm57Kefunl9ils4/pFHXI+c//zHTfnw449uVtCpU6F5c/9i+dvfXNvDX/7i1v7duNHNBeRP+4DxT1B0HzXGlJ8dO1y3zf79ISHBfZEXrL9bMNVzUSNHQuPGcOGFR78n4mbvzM93o2JfeMFNs1wWkZEuMZ11lpsIrmtXt7+08QOmbOyOwBhzmHffdb19Bg1yr++8041sfeaZo8uuW+cWgRkwoORqmrAw11tn4sSyJ4ECZ5zhqpFefdUlk7g4Nz7AlA9LBMaYQqpu8fdOndwGUK+eSwrvvuumfC5qzBj3mYEDvY/tySfdncecOW520ONc4M8UwxKBMabQjz+6Rtwjl2W89173y/655w7ty893ffzPPx+aNfM+tri4QxPInX++9+cLJZYIjDGFRoxw1UDXXnv4/sREt0DLyJFusBe4mT/XrCm+kdgrPXu6qaVvu63izhkKLBEYYwDXI2f8eLj6aqhZ8+j377/fdRUtWBd81Cg310/fvhUZpWs0jomp2HMGO0sExhgAxo2DPXsONRIfqVUrNyXDq6+6toKJE92dg30pV32eJQIRaSwi34rIchFZKiJHLbImzlAR+VVEFolIJ6/iMSbUDRvm5tkvaf6fESNcf/0uXUo+xkMPuTuHXr3ccSqyWsh4x8s7glzgXlVtDZwJ3CYiR/ZC/gPQ0rcNAvxcS8gYUxbp6XDXXfDoo5Cc7OrZi5o/H+bNc3cDpc3F37Ej9OjhxhQkJx/qWWSqNs8SgapuVNX5vue7gOVAoyOK9QHe9k2F8T8gXkQqwSqhxgSXf/3LfcGPGQP79rnul/fcA3v3uvdHjHCzb/brd+xjPfywe7zpJu/iNRWrQtoIRCQJ6AjMOeKtRsC6Iq8zODpZICKDRCRNRNJs8RljymbdOtewe+ONrufPkiVugfYXXnCLuvz3v26MwBVXuOkkjqVrV7cg/ODB3sduKobniUBEYoGJwF2quvPIt4v5yFHrTarqcFVNUdWU+sez7JExQSgnx79yzzzjBn09+KB7HRfnGny/+cbt79nTrelbUiNxcdq3D9GVvIKUp39KEYnEJYF3VfWjYopkAI2LvE4ENngZkzHBYNEiN+K3uEVbilq/3lX7DBgATZse/l737u4499wD11wD55zjWbimkvOy15AAI4Hlqvp8CcUmA3/29R46E8hW1Y1exWRMsHjtNdd75y9/gbS0kss9+yzk5R2q1z9SjRpuZtD33qucC7abiuHlbB3nANcDi0VkgW/fw0ATAFV9HZgC9AR+BfYCFTBjiTFV29697ou7d2/3i/7yy12Pn6JrAYCbrnn4cNcuUBFTQJiqy7NEoKrfUXwbQNEyCthgcWPK4MMP3d3Affe56SDOPdf19vn888NnAH3uOTh4sOS7AWMKWHOPMRVowYKjZ/Asq5EjoUULOO88+N3v4OWX3VTQjz9+qMyWLa76qF8/V9aY0lgiMKaC/PgjnHmm+wW/ZcvxHWPlSpg503UFLajTv/lm1xj8+OPurgBcvf+BA/D3v5dL6CbIWSIwpgJs2ACXXgoNGrhlHa+91jXiltWoUa76p3//Q/tEXHfQ5GS47jqXcIYNc5PHnXpqeV2BCWaWCEzI2bjRja6tKPv2uSSwc6f7xT5smFsQ/rHHynacnBw3MrhXLzj5iPH3MTFuEjhwA7727rW7AeM/SwQmpCxb5n4lX3AB5OZ6fz5VV3Uzdy6MHesGYt1wg1vR64kn3Khef02Z4tb9LWmit+bN3TkOHnSjhItbX9iY4lgiMCEjK8v9Mgf44Yey/yI/Hs8+66ZvePLJQ+cGd1dw+umuKmftWv+ONXKkuxPo2bPkMr16uaQzcuQJhW1CjCUCExLy8ly9/Jo17pf1DTfAU0+5Vba88umnbtrmq646ugtnTIzrBpqb6369HzhQ+rE2bHDVSv37H3ut3pQUiI09sdhNaLFEYELCP//pqmFeftn12hk6FFq2dN0rt20r//MtXeoST8eOroG3uFG7LVrA6NHuF/w995R+vDFj3BrBN9xQ/rEaY4nABL0PP4Snn3bTJt9yi9tXo4ZblnHbNlfnrkdNdVh2qvDTT/DII3DRRe4cH3/sBn2VpG9fNzDs1Vfh7beLL5Of76p6unVzycuY8maJwAS1JUtcH/szz4RXXjn8l3nHjm5mzsmT3Rfx8cjNhW+/hb/+FZKS3EItTz4Jp5ziqnIaNz7mIRgyxA0O698fLrnEJZOiZs6EVatsNTDjHUsEpkp4/33XLfLTT/3/9b59O/Tp46ZdnjgRoqOPLvPXv7rG13vvdfP2+Gv1avjb3yAhAc4/H954wzX+jhzpuqfOnAmdO/t3rIgI127x9NMwe7ZLJpdf7qqXAN58E2rVgj/9yf/4jCkTVa1SW+fOndWElvx81bZtVV0KUO3aVfX770svP3++6vnnq0ZGll5WVXXzZtWEBNXWrVX37Cm5XF6e6pdfqvburSqiGh6uevnlqhMnqu7adXzXdqSsLNVHHlGNi3PnuOYa1WrVVAcPLp/jm9AFpGkJ36ui5VE5WoFSUlI0rbR5d03Q+d//4Kyz3Nw54Nbd3bwZLrvM/Yo+9VRXj/7DD/DRR25LT3cjcEeMcH32j2XqVFev36WLW7Wrdm2oU+fQlpHhqo9WrnSjg2+5xW2NjlpPr3xkZsK//+0at/fudVNN+3uHYUxxRGSeqqYU+54lAlPZ3XQTTJjgqlxiY2H3brfM4rPPulG7PXq4aZg3bYKoKLjwQpck/vhHKMuCdi+95Kp4duxwX8RHrgB2xhlw++2uu2dx1Uxe2LwZli+H1NSKOZ8JXpYITJW1a5cbRHXNNe7XfVFbtrixAB984FbXuuwyV99fq9aJn1fV/RLfvt1tUVHQuvWJH9eYQCktEXi5MI0xJ2z8eNizx90VHKlBA/cr/qWXyv+8Iq77Z40a/vX8MaYqs15DplJ7801o187V3RtjvGGJwFRaixa5KZVvusnW0zXGS5YITKU1cqSrm7/uukBHYkxws0RgKqX9++Gdd1wD8JGLshtjypclAlMpTZrkunEW10hsjClfIZMI5mTM4fpJ17N1z9ZAh2L88OabbqGV7t0DHYkxwS9kEsG2vdsYu2gsq3asCnQo5hhWrYJvvnGTrIWFzH+hxgROyPxvlhSfBEB6VnpA4zDHNnKkSwADBgQ6EmNCQ8gkgqbxTQFLBJVdbi689ZZbcrFhw0BHY0xoCJmRxbFRsdSrXs8SQYBt3+6mihgxwk3jcNppbtK4gsf0dDdnkDUSG1NxQiYRgKseskQQGMuWueUh337bTRSXmgonnQQrVrj2gP37D5U91gLtxpjyFXKJYPHmxYEOI6R8+SX85z/w9dduxs7rroM773RTPRfIz4d161xS+PlnSE4+9gLtxpjyE1L/uyXVSuKzlZ+5hRhszgLPffIJXHqpq+t/6ikYNAjq1Tu6XFgYNG3qtosvrvAwjQl5oZUI4pPYn7ufTbs3cXLcyYEOJ6jt3Am33Qbt28PcuRU3f78xpuw86zUkIqNEZIuILCnh/VQRyRaRBb7tn17FUsC6kFachx6CDRtco7AlAWMqNy+7j44GehyjzCxVTfZtj3sYCwDNajcDLBF47fvv3bKSd9zhVvUyxlRuniUCVZ0JbPfq+MejaS0bS+C1Awfg5pshMRGefDLQ0Rhj/BHoAWVnichCEfmviLQtqZCIDBKRNBFJ27r1+OcKqhFVg/rV61siOIb334d333WDu8rqmWdcV9HXXoO4uPKPzRhT/gKZCOYDTVX1dOBl4OOSCqrqcFVNUdWU+mVZjbwYSfFJpGenn9AxgtXBgzB4MFx1levmeeqpro7/4EH/Pr98uesddNVVbmSwMaZqCFgiUNWdqrrb93wKECkixXQuLF82qKx4mzbB+efD66/DAw/Axx9DnTquy2eLFjBs2OGDvo6Un+/K1qjhzRrCxhjvBCwRiEiC+Drzi0gXXyyZXp83KT6JtVlrydd8r09VZcyZA507w08/wYQJ8K9/QZ8+bpnIL76AJk3g9tuhWTN45BGXJFauPLzqaMQI+O47eO45N2LYGFN1eDaOQETGAalAPRHJAB4BIgFU9XXgcmCwiOQC+4CrVVW9iqdAUnwSB/IOsHn3ZhtLAIwa5aqDGjWCH344fMSviBvgddFFMH26a/x9vEjfrqgoV33Upo1LGN27w8CBFX4JxpgT5FkiUNVrjvH+K8ArXp2/JEXHEoRyIjhwAO65B159FS68EMaPL3lJSBH3Jd+9uxsotmKFaw9Ytsxtc+e6pPDGG7bIvDFVUUiNLIbDE8FZjc8KbDABsmIFXH01LFwI997rqoL8ndunZk3o0sVtxpjg4FcbgYjUEJEw3/NWIvJHEYn0NjRvhPJYAlW36EvnzpCRAZMnuzp9m+DNmNDmb2PxTKCaiDQCpgEDcSOHq5xQHUuQleXuAm66Cc48ExYtgksuCXRUxpjKwN9EIKq6F7gMeFlV+wJtvAvLW6E2luD7793UzhMnwpAh8NVXtvqXMeYQfysFRETOAvoBN5bxs5VOUnwSCzcvDHQYFWLxYujWDRo3htmzbe4fY8zR/L0juAt4CJikqktFpDnwrWdReSyUxhK8+aab73/OHEsCxpji+fWrXlVnADMAfI3G21T1Ti8D81KojCU4eNDNGXTppXCCM3MYY4KYv72G3hORmiJSA1gG/Cwif/M2NO+EyroEU6ZAZib07x/oSIwxlZm/VUNtVHUncCkwBWgCXO9VUF4LlUQwZgwkJLiRwcYYUxJ/E0Gkb9zApcAnqpoDeD4dhFdCYSzB1q3w2WduFlEbJ2CMKY2/ieANIB2oAcwUkabATq+C8loojCUYN85NCmfVQsaYY/G3sXgoMLTIrrUi0t2bkCpGsI8lGDMGOnWCdu0CHYkxprLzt7G4log8X7BKmIj8B3d3UGUlxSexZseaQIfhicWLYf58uxswxvjH36qhUcAu4ErfthN4y6ugKkJSfBJrs4NzLMGYMRAZCddeG+hIjDFVgb+J4BRVfURVV/u2x4DmXgbmtaT4JA7mHWTT7k2BDqVM3nzTzRd04EDx7+fmwtixbqnIep6v92aMCQb+9ifZJyLnqup3ACJyDm4xmSqraBfShnFVY+KdrCw3bfTOnbBtG3zwgfvlX9RXX8HmzVYtZIzxn793BLcCw0QkXUTScQvK3OJZVBWgWXwzoGp1IR02zCWBO+6ATz6B66+HvLzDy4we7e4EevYMSIjGmCrI315DC4HTRaSm7/VOEbkLWORhbJ5qGl+1xhLs2QMvvOC+4IcOdesI/+1vEBPj1hgIC4MdO1yCuPVWt2KYMcb4o0xDjXyjiwvcA7xYrtFUoOqR1WlQo0GVSQQjRrjpIv7+d/f6vvtg7163mHxMjLtbmDDBzS9k1ULGmLI4kTGnVX512qT4pCqRCA4cgH//G1JT4eyzD+3/xz/cncKzz7pkMHu2GzfQsWPAQjXGVEEnkgiq3BQT+fmuCqVAUnwSP238KXAB+entt2HDBlf/X5SIW2947154/nm377nnbAF5Y0zZlNpYLCK7RGRnMdsuoGp0tfH5+mto29bNwVMgqVblH0uQm+u+7H/3O7jwwqPfF4GXXoKbb4ZataBfv4qP0RhTtZWaCFQ1TlVrFrPFqWqVmsosMRF++QUef/zQvqowlmDCBFi9Gh5+uORf+mFhMHw4bNzoZhs1xpiy8Lf7aJXXujUMGgSvvw4//+z2VfbpqPPz3RrDbdvCH/947PIxMd7HZIwJPiGTCAAefdR9Wd5/v3td2RPB5MmwdKm7GwgLqb+UMaYihdTXS4MG7kt18mSYPr1yjyVQhaefhubN4corAx2NMSaYhVQiAPjrX91grHvvhWrhlXcswdSpMHcuPPigLSxjjPFWyCWCmBj3S3v+fDc5W2UcSzB3Ltx1FzRqBH/+c6CjMcYEu5BLBADXXAMpKW6UbmJMq0qTCFavdrF16eK6ub7xBkRHBzoqY0ywC8lEEBbmBmBlZMDWqf0CPpYgMxPuvhtOO821X/zjH7BqlZtK2hhjvOZZIhCRUSKyRUSWlPC+iMhQEflVRBaJSCevYilO167Qty/8+P75HMyqHZCxBLt2wTPPwCmnuInk+vc/NNYhLq7CwzHGhCgv7whGAz1Kef8PQEvfNgh4zcNYivXMM5B7MAKmP1ah1UNbtsD//Z9rtH7wQTjnHFi40E0s17BKjdc2xgQDzxKBqs4EtpdSpA/wtjr/A+JF5GSv4ilOy5ZwzcAsmH8Tb43dS77HtUOrV8Ntt0HTpq7B+vzzYc4c+PxzW2TeGBM4gWwjaASsK/I6w7fvKCIySETSRCRta9HJgsrB80/HE31SOm8+fCFt2iijRrmpnMvTzz+79YNbtnS/+vv1g+XLYeJE1zBsjDGBFMhEUNzMOcXOaKqqw1U1RVVT6tevX65B1K8XxpivfoLLr2S/7ODGG90gruefh927T+zY69e7yeDatoVPP3VjF9LT3brDp55aLuEbY8wJC2QiyAAaF3mdCGwIRCBXtLuM0y9YScTgM5jy3zxatXJf2k2awF/+4lb92rnz2McpsGOHq/tv0QLGjIHbb3fVQs8+a20AxpjKJ5BjVicDt4vIeOAMIFtVNwYikDAJ47HUx7h0wqVsOultvvlmIHPmwH/+49YCeO01CA+Hs86Ciy5yW+vWbsGY/fvdtm+fe5w+3U0Ul50N113negAlJQXiqowxxj+i6s36MiIyDkgF6gGbgUeASABVfV1EBHgF17NoLzBQVdOOddyUlBRNSztmsTJTVX434ndk7stk5e0riQyPBFx7wQ8/wFdfuW3ePDcPUGl69XKNwR06lHuYxhhzXERknqqmFPueV4nAK14lAoApv0yh13u9eKP3GwzqPKjYMlu3wrRpsG6dm64iJgaqVXNbTIxb98ASgDGmsrFE4CdV5exRZ7N+53p+ueMXoiNsfgdjTHAoLRGE5BQTJRERHk99nHU71/Hm/DcDHY4xxlQISwRHuLD5hZzb5Fye/u5p9uXsC3Q4xhjjOUsERxARnuj+BBt2beCNeW8EOhxjjPGcJYJipCal0j2pO0O+G8Keg3sCHY4xxnjKEkEJHu/+OFv2bOHF/70Y6FCMMcZTlghKcG6Tc/lT6z/x2IzHSNvgTS8lY4ypDCwRlGL4JcNJiE3g6g+vZueBMswxYYwxVYglglLUianDe396jzVZa7j1s1upamMujDHGH5YIjuHcJufyWOpjjFsyjtELRgc6HGOMKXeWCPzw0LkPkZqUyu3/vZ0V21YEOhxjjClXlgj8EB4WzruXvUv1yOpc9eFV7M/dH+iQjDGm3Fgi8FPDuIaM7jOaRZsXcd9X9wU6HGOMKTeWCMqgV6te3H3m3QybO4xJyycFOhxjjCkXlgjKaMgFQ+h8cmf6f9yfxZsXBzocY4w5YZYIyig6IppJV00iLjqOnu/1ZP3O9YEOyRhjToglguPQuFZjPr/2c7L2Z9HrvV7sOrAr0CEZY8xxs0RwnJITkvngig9YsmUJV354Jbn5uYEOyRhjjoslghPQo0UPXuv1Gl/8+gW3fX6bjTw2xlRJEYEOoKq7ufPNrMlaw5DvhtCsdjMePPfBQIdkjDFlYomgHDx5/pOsyVrDQ9MeIik+iavbXR3okIwxxm+WCMpBmIQxus9o1u9cT/+P+xNfLZ4eLXoEOixjjPGLtRGUk+iIaD65+hPa1G9D3wl9+XbNt4EOyRhj/GKJoBzVjqnNV9d9RfPazblk3CXM/m12oEMyxphjskRQzurXqM+0P0+jUc1G9Hyvp61uZoyp9CwReCAhNoFpf55G3Zi6XPTORSzctDDQIRljTIksEXgksWYi3/T/hhpRNfj9O79n2dZlgQ7JGGOKZYnAQ0nxSXzz528IDwvngrcvsEVtjDGVkiUCj7Ws25Jpf56GqnLeW+exaPOiQIdkjDGHsURQAdrUb8OMATOICo8idXQqc9fPDXRIxhhTyNNEICI9RORnEflVRI6ae0FEUkUkW0QW+LZ/ehlPIJ1a71RmDZxFfLV4Lnj7AmatnRXokIwxBvAwEYhIODAM+APQBrhGRNoUU3SWqib7tse9iqcyaFa7GbMGzqJhXEMuHnsxX6/6OtAhGWOMp3cEXYBfVXW1qh4ExgN9PDxfldCoZiNmDJhBy7ot6T2uN5/+/GmgQzLGhDgvE0EjYF2R1xm+fUc6S0QWish/RaSth/FUGifFnsS3/b/l9JNO57L3L2PUT6MCHZIxJoR5mQikmH1HTtg/H2iqqqcDLwMfF3sgkUEikiYiaVu3bi3fKAOkTkwdpv55KqlJqdw4+UYGfTqI/bn7Ax2WMSYEeZkIMoDGRV4nAhuKFlDVnaq62/d8ChApIvWOPJCqDlfVFFVNqV+/vochV6ya0TX5b7//8uA5DzJi/gi6vtWVtVlrAx2WMSbEeJkI5gItRaSZiEQBVwOTixYQkQQREd/zLr54Mj2MqdKJCItgyIVD+Piqj1mZuZJOwzvx1aqvAh2WMSaEeJYIVDUXuB34ElgOvK+qS0XkVhG51VfscmCJiCwEhgJXa4iu99jntD6k3ZxGw7iG9Bjbg6dmPkW+5gc6LGNMCJCq9r2bkpKiaWnBO6PnnoN7uOWzW3h38buc0/gcHu/+ON2TuuO7cTLGmOMiIvNUNaW492xkcSVTI6oG7/R9h5F/HMmarDVc8PYFpI5JZXr69ECHZowJUpYIKiER4YaON7DqzlW8/IeX+XX7r3Qf053uY7ozI31GoMMzxgQZSwSVWLWIatze5XZW3bmKl3q8xM/bfiZ1TCrnjDqHEfNGsGPfjkCHaIwJAtZGUIXsy9nHiPkjeC3tNVZsW0FUeBSXtLqE6ztczx9a/oGo8KhAh2iMqaRKayOwRFAFqSrzN87nnUXvMG7JOLbs2UKdmDpc2eZK/tTmT3Rr2o3I8MhAh2mMqUQsEQSx3Pxcvl71Ne8seodPfv6EvTl7qV2tNr1b9abvaX256JSLqBFVI9BhGmMCzBJBiNibs5evV33NpBWT+HTlp2zft52YiBgubnExN3e6mR4tehAm1ixkTCiyRBCCcvJymPXbLCYtn8SHyz9k0+5NnFL7FP7yu78wMHkgtWNqBzpEY0wFskQQ4nLycvho+Ue8MvcVvvvtO6pHVue69tdxW5fb6HBSh0CHZ4ypAJYITKEFmxYw7MdhvLv4Xfbl7qNRXCPan9SedvXb0a6B29rUb0NMZEygQzXGlCNLBOYo2/dt591F7zJ3w1yWbFnCsq3LOJB3AABBOK/peTx07kNcdMpFNr2FMUHAEoE5ptz8XFZtX8WSLUtYsGkBoxeOJmNnBh0TOvJw14fpe1pfwsPCAx2mMeY4WSIwZXYw7yBjF43lmdnPsDJzJa3qtuLBcx6kX4d+NnDNmCrIEoE5bnn5eXy0/COGfDeEnzb9RGxULC3qtOCU2qfQvHZzTql9CqfUOYXT6p1GYs3EQIdrjCmBJQJzwlSVL1d9yZRfprBqxypWbV/Fmqw1HMw7WFgmpWEKV7e9mivbXknjWo1LOZoxpqJZIjCeyMvPY/2u9azesZq56+fy/rL3Sdvg/jbnNjmXq9tezeVtLuek2JMCHKkxxhKBqTC/bv+VCUsmMH7peJZsWUKYhJGckEy3pt1ITUqla5OuNpjNmACwRGACYsmWJUxcNpFv07/lfxn/40DeAQShw0kd6Na0G6cnnE6z+GY0r92cxJqJ1ivJGA9ZIjABtz93Pz+u/5EZ6TOYsXYG36/7nn25+wrfjwiLoGmtpjSr3YyE2ASqR1SneuThW6OajejdqjfVI6sH8EqMqZosEZhKJzc/l3XZ61i9YzVrstYc9rht7zb25uxlb85e9hzcQ57mFX4uLiqOq9pexQ0db+DMxDNtsJsxfiotEURUdDDGgLsDaFa7Gc1qNztm2Zy8HPbm7C0c6Pbekvd486c3ObXuqQxMHsgVba9AVdm+bzvb920nc18m2/dtJ3t/Nok1E2nboC2n1TuN2KjYCrgyY6oeuyMwVc6uA7v4YNkHvLXgLb777Tu/P5cUn0Sb+m1oW78tzWs3p2Fcw8LtpBonWRuFCWpWNWSC1i+Zv/D16q+JjYqlTkwd6sTUoW5MXerE1CEuOo61WWtZtnUZS7cuZenWpSzbuowV21YcNv4BIEzCSIhNoHHNxu5OxdeI3Sze3bU0imtEVHiUVUWZKssSgTFF5OXnsXnPZjbs2sCGXRtYv3O9e9y1nnU7XbvFb9m/kZufe9RnI8IiiAyLJDI8ksiwSKpFVKNhXEOS4pNoWqspTeObFj4mxSdRM7rmMeNRVbbt3UZOfg4nx55sycZ4wtoIjCkiPCy8sEqoJLn5uWTszGDNDteAvXH3RnLycsjJzyEnL4fc/Fxy8l3bRcbODBZuXsinKz9lf+7+w45TJ6YOSfFJNItvVvh4MO8ga7LWuG3HGtKz0tmTsweA+GrxtK3flrb129KuQTvaNmhLq7qtiI2KpVpENaLDo49KFHtz9pK5N5PMfZlk7s0ka38WnU7u5Ff7izFgdwTGlBtVZcueLazNXkt6VnrhtiZrTeHzgkQRFxVHs9qHkkOz+GaESVhhNdaSLUvYsX/HUecQhGoR1YiJjCEqPIrs/dmHdcMtqmuTrvQ/vT9XtL3CrzuTY8nLz2PxlsXMWjuLtI1p1IupR8u6LWlRpwUt67Skca3GthRqJWZVQ8ZUAvmaz5Y9W4gMi6ROTJ1Sq4BUlU27N7FkyxJW71jNvtx97MvZd9jjwbyD1IquRd3qdakbU7fwMTYqlq9WfcXohaNZmbmSmIgY+rbuS//T+3NW4lnu+CiqWviYr/nk5PvudHx3Prn5uWTuzWT2utnM+m0Ws3+bTfaBbAASYhPI2p912B1QdHg0zWs3JzkhmbMSz+LMxDNJTkgmMjyyXP799hzcw/gl4xm/dDwJsQmc0/gczm58Nm3rt7WGfj9YIjAmBKkqc9bPYcyCMYxfOp6s/VnHfazW9VrTtUlXzmt6Hl2bdqVJrSbkaz4bdm3gl8xf+GX7L/y6/VdWZq4kbUMa63etB6BaRDVSGqZwZqMzaX9Se06OPZmE2AROij2JetXr+XUHsXzrcl5Pe50xC8eQfSCbVnVbkb0/m817NgNQM7omZyaeydmJZ5MQm0Bufm5h1V3B85iIGJrUakLT+KY0qdWEBjUaVJq7l90Hd7M2ay2/Zf/G2uy1bN69mRZ1WtDp5E60qtuq3JKcJQJjQtz+3P18tvIz1uxYg4ggyGGPYRJW2Ahe0CAeERZBbFQsXRp1oX6N+mU637rsdfyQ8QP/y/gfP2T8wPyN84/qqRUu4TSo0aAwMSTEJnBSjUOPufm5jFowiunp04kKj+LyNpczOGUw5zQ+B4A1WWv4ft33zP5tNt9nfM/izYtR/Ps+iw6PpnGtxjSt1bRwKvWij3HRcaV+fl/OvsIOBut3rmf9rvXsPLDzsHakg3kH3fP8HA7kHuBA3oHCx4N5B9l1YBfrdq5j+77tJZ6nemR1khOS6ZTQiU4nd6Jr0660qNPCr2s8kiUCY0xAHcg9wG/Zv7Fp96ajtz2b2Lx7M5v3bGbz7s3k5OcUfi4pPolbO9/KwI4DaVCjQann2HVgF7sO7iIiLOKwZBYRFsHug7v5Lfu3wm1ttvsFnp6Vzqodq9i2d9thx6oTU4dqEdUIkzDCJZzwsHDCJRwRYdvebSV+eRftURYZHklUeBSRYZFER0QTHR5NdEQ0UeFRRIdHUyOqBo1rumRU9G6lfvX6/LL9F+ZvnF+4/bTpJ3Yf3M39Z9/PM79/5rj+BpYIjDFVgqqyY/8ONu/ezN6cvSQnJFdI/X/2/mxW71hduNbGb9m/cTDvIHma57b8Q491Y+rSqGYjGsU1IrFmYuHzmtE1Pev6m6/5/JL5C9UiqtE0vulxHSNgiUBEegAvAeHAm6r6ryPeF9/7PYG9wABVnV/aMS0RGGNM2ZWWCDxrLRGRcGAY8AegDXCNiLQ5otgfgJa+bRDwmlfxGGOMKZ6XzeZdgF9VdbWqHgTGA32OKNMHeFud/wHxInKyhzEZY4w5gpeJoBGwrsjrDN++spZBRAaJSJqIpG3durXcAzXGmFDmZSIortXkyAYJf8qgqsNVNUVVU+rXL1s3NmOMMaXzMhFkAI2LvE4ENhxHGWOMMR7yMhHMBVqKSDMRiQKuBiYfUWYy8GdxzgSyVXWjhzEZY4w5gmezj6pqrojcDnyJ6z46SlWXisitvvdfB6bguo7+ius+OtCreIwxxhTP02moVXUK7su+6L7XizxX4DYvYzDGGFO6KjeyWES2AmuPUawesO0YZYKRXXfoCdVrt+suu6aqWmxvmyqXCPwhImkljaALZnbdoSdUr92uu3xVjnlYjTHGBIwlAmOMCXHBmgiGBzqAALHrDj2heu123eUoKNsIjDHG+C9Y7wiMMcb4yRKBMcaEuKBLBCLSQ0R+FpFfReTBQMfjFREZJSJbRGRJkX11RORrEfnF91g7kDF6QUQai8i3IrJcRJaKyF99+4P62kWkmoj8KCILfdf9mG9/UF93AREJF5GfROQz3+ugv24RSReRxSKyQETSfPs8ue6gSgR+LoYTLEYDPY7Y9yAwTVVbAtN8r4NNLnCvqrYGzgRu8/2Ng/3aDwDnq+rpQDLQwzc/V7Bfd4G/AsuLvA6V6+6uqslFxg54ct1BlQjwbzGcoKCqM4EjV9DuA4zxPR8DXFqRMVUEVd1YsJypqu7CfTk0Isiv3bd4027fy0jfpgT5dQOISCLQC3izyO6gv+4SeHLdwZYI/FroJoidVDB7q++xQYDj8ZSIJAEdgTmEwLX7qkcWAFuAr1U1JK4beBG4H8gvsi8UrluBr0RknogM8u3z5Lo9nXQuAPxa6MZUfSISC0wE7lLVnSLF/emDi6rmAckiEg9MEpF2AQ7JcyLSG9iiqvNEJDXA4VS0c1R1g4g0AL4WkRVenSjY7ghCfaGbzQVrPvsetwQ4Hk+ISCQuCbyrqh/5dofEtQOoahYwHddGFOzXfQ7wRxFJx1X1ni8iYwn+60ZVN/getwCTcFXfnlx3sCUCfxbDCWaTgf6+5/2BTwIYiyfE/fQfCSxX1eeLvBXU1y4i9X13AohIDHAhsIIgv25VfUhVE1U1Cff/8zeqeh1Bft0iUkNE4gqeAxcBS/DouoNuZLGI9MTVKRYshvNUYCPyhoiMA1Jx09JuBh4BPgbeB5oAvwFXqOqRDcpVmoicC8wCFnOozvhhXDtB0F67iHTANQ6G437Ava+qj4tIXYL4uovyVQ3dp6q9g/26RaQ57i4AXBX+e6r6lFfXHXSJwBhjTNkEW9WQMcaYMrJEYIwxIc4SgTHGhDhLBMYYE+IsERhjTIizRGCMj4jk+WZ6LNjKbSIzEUkqOlOsMZVJsE0xYcyJ2KeqyYEOwpiKZncExhyDb174Z3zrAfwoIi18+5uKyDQRWeR7bOLbf5KITPKtHbBQRM72HSpcREb41hP4yjdCGBG5U0SW+Y4zPkCXaUKYJQJjDok5omroqiLv7VTVLsAruJHr+J6/raodgHeBob79Q4EZvrUDOgFLfftbAsNUtS2QBfzJt/9BoKPvOLd6c2nGlMxGFhvjIyK7VTW2mP3puEVhVvsmvNukqnVFZBtwsqrm+PZvVNV6IrIVSFTVA0WOkYSbOrql7/UDQKSqPikiXwC7cVOEfFxk3QFjKoTdERjjHy3heUllinOgyPM8DrXR9cKtrNcZmCci1nZnKpQlAmP8c1WRxx98z7/HzYgJ0A/4zvd8GjAYCheTqVnSQUUkDGisqt/iFl+JB466KzHGS/bLw5hDYnwrgBX4QlULupBGi8gc3I+na3z77gRGicjfgK3AQN/+vwLDReRG3C//wcDGEs4ZDowVkVq4hZVe8K03YEyFsTYCY47B10aQoqrbAh2LMV6wqiFjjAlxdkdgjDEhzu4IjDEmxFkiMMaYEGeJwBhjQpwlAmOMCXGWCIwxJsT9P85ieaHscLxBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plotting the loss graph of training and test data\n",
    "epochs = range(1,51)\n",
    "plt.plot(epochs, relu_training_loss_per_epoch, 'g', label='Training Loss')\n",
    "plt.plot(epochs, relu_test_loss_per_epoch, 'b', label='Test Loss')\n",
    "plt.title('Training and Test Loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# C Question"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tanh Activation Function Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a neural network class which extends the nn.Module base class\n",
    "\n",
    "#Created a network with Tanh activation function\n",
    "class Network_tanh(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "\n",
    "        #Fully Connected Layer Initialization\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight)\n",
    "\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = torch.tanh(t)   #tanh activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = torch.tanh(t)   #tanh activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = torch.tanh(t)   #tanh activation\n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = torch.tanh(t)   #tanh activation\n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)    \n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our tanh network class\n",
    "network_tanh = Network_tanh().to(device) #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for Tanh\n",
    "\n",
    "tanh_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "tanh_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "tanh_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "tanh_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the tanh network\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_tanh, learning_rate =0.1)\n",
    "    tanh_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    tanh_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_tanh)\n",
    "    tanh_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    tanh_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Tanh Training Accuracy Obtained : 100.0\n",
      "Final Tanh Test Accuracy Obtained : 91.35\n"
     ]
    }
   ],
   "source": [
    "#Final train and test accuracy obtained on tanh activation network\n",
    "print(\"Final Tanh Training Accuracy Obtained : {}\".format(tanh_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Tanh Test Accuracy Obtained : {}\".format(tanh_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sigmoid  Activation Function Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a neural network class which extends the nn.Module base class\n",
    "\n",
    "#Created a network with Sigmoid activation function\n",
    "class Network_sigmoid(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter \n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "\n",
    "        #Fully Connected Layer Initialization\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight)\n",
    "\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = torch.sigmoid(t)   #sigmoid activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = torch.sigmoid(t)   #sigmoid activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = torch.sigmoid(t)   #sigmoid activation\n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = torch.sigmoid(t)   #sigmoid activation\n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)\n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our sigmoid network class\n",
    "network_sigmoid = Network_sigmoid().to(device) #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for Sigmoid\n",
    "\n",
    "sigmoid_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "sigmoid_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "sigmoid_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "sigmoid_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the sigmoid network\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_sigmoid, learning_rate =0.1)\n",
    "    sigmoid_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    sigmoid_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_sigmoid)\n",
    "    sigmoid_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    sigmoid_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Sigmoid Training Accuracy Obtained : 92.63333333333334\n",
      "Final Sigmoid Test Accuracy Obtained : 90.51\n"
     ]
    }
   ],
   "source": [
    "print(\"Final Sigmoid Training Accuracy Obtained : {}\".format(sigmoid_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Sigmoid Test Accuracy Obtained : {}\".format(sigmoid_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elu  Activation Function Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network_elu(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter \n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "\n",
    "        #Fully Connected Layer Initialization\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight)\n",
    "\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = F.elu(t)   #elu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = F.elu(t)   #elu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = F.elu(t)   #elu activation\n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = F.elu(t)   #elu activation\n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)\n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our elu network class\n",
    "network_elu = Network_elu().to(device)  #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for elu\n",
    "\n",
    "elu_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "elu_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "elu_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "elu_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the elu network\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_elu, learning_rate =0.1)\n",
    "    elu_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    elu_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_elu)\n",
    "    elu_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    elu_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Elu Training Accuracy Obtained : 99.35\n",
      "Final Elu Test Accuracy Obtained : 90.28\n"
     ]
    }
   ],
   "source": [
    "print(\"Final Elu Training Accuracy Obtained : {}\".format(elu_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Elu Test Accuracy Obtained : {}\".format(elu_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying Learning rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Trying different learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu\n",
    "\n",
    "relu2_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu2_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu2_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu2_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our relu network class\n",
    "network_relu_2 = Network_relu().to(device)  #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network\n",
    "\n",
    "#Training with learning rate = 0.001\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_2, learning_rate =0.001)\n",
    "    relu2_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu2_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_2)\n",
    "    relu2_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu2_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after changing learning rate to 0.001 :-> [49.285, 69.14166666666667, 72.89, 75.06833333333333, 76.81166666666667, 78.04166666666667, 79.06833333333333, 79.97, 80.68333333333334, 81.41666666666667, 81.96, 82.48, 82.91666666666667, 83.29166666666667, 83.65666666666667, 83.99666666666667, 84.29333333333334, 84.62833333333333, 84.91, 85.22333333333333, 85.47166666666666, 85.725, 85.89666666666666, 86.08666666666667, 86.28666666666666, 86.475, 86.61833333333334, 86.745, 86.90666666666667, 87.05666666666667, 87.165, 87.29333333333334, 87.41, 87.52, 87.645, 87.74, 87.83, 87.925, 88.02166666666666, 88.115, 88.21, 88.315, 88.42666666666666, 88.52666666666667, 88.60166666666667, 88.70833333333333, 88.79833333333333, 88.875, 88.98333333333333, 89.08166666666666]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training accuracy after changing learning rate to 0.001 :->{}\".format(relu2_training_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy after changing learning rate to 0.001 :->[66.21, 72.26, 74.59, 76.14, 77.49, 78.31, 79.25, 80.03, 80.74, 81.3, 81.88, 82.33, 82.69, 83.15, 83.42, 83.7, 83.96, 84.18, 84.42, 84.83, 84.81, 85.01, 85.26, 85.38, 85.53, 85.62, 85.77, 85.97, 86.12, 86.34, 86.41, 86.46, 86.55, 86.68, 86.82, 86.85, 86.99, 87.08, 87.17, 87.35, 87.37, 87.4, 87.52, 87.62, 87.71, 87.77, 87.78, 87.87, 87.89, 87.95]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test accuracy after changing learning rate to 0.001 :->{}\".format(relu2_test_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying Learning rate = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu\n",
    "\n",
    "relu3_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu3_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu3_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu3_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our relu network class\n",
    "network_relu_3 = Network_relu().to(device)  #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network\n",
    "\n",
    "#Training with learning rate = 0.5\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_3, learning_rate =0.5)\n",
    "    relu3_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu3_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_3)\n",
    "    relu3_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu3_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after changing learning rate to 0.5 :->[75.49666666666667, 85.16, 86.89333333333333, 88.01, 88.0, 87.47333333333333, 88.40333333333334, 89.16666666666667, 89.80333333333333, 90.32333333333334, 89.64, 90.11166666666666, 89.34166666666667, 87.44, 89.10833333333333, 89.64166666666667, 89.62, 89.70833333333333, 89.69833333333334, 90.18333333333334, 90.17, 89.92333333333333, 90.08, 89.69, 89.44333333333333, 89.75166666666667, 89.49333333333334, 88.71666666666667, 88.59666666666666, 89.33333333333333, 87.90833333333333, 10.658333333333333, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035, 10.035]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training accuracy after changing learning rate to 0.5 :->{}\".format(relu3_training_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy after changing learning rate to 0.5 :->[83.99, 85.68, 86.47, 85.97, 84.63, 85.93, 86.2, 85.84, 86.26, 86.74, 87.09, 86.64, 83.14, 85.11, 85.3, 85.38, 85.89, 85.14, 85.74, 86.15, 85.45, 86.18, 85.32, 84.05, 85.55, 85.05, 84.88, 85.36, 85.97, 84.37, 76.92, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test accuracy after changing learning rate to 0.5 :->{}\".format(relu3_test_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying Learning rate = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu\n",
    "\n",
    "relu4_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu4_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu4_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu4_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our relu network class\n",
    "network_relu_4 = Network_relu().to(device)  #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network\n",
    "\n",
    "#Training with learning rate = 1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_4, learning_rate =1)\n",
    "    relu4_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu4_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_4)\n",
    "    relu4_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu4_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after changing learning rate to 1 :->[10.085, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334, 10.058333333333334]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training accuracy after changing learning rate to 1 :->{}\".format(relu4_training_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy after changing learning rate to 1 :->[10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test accuracy after changing learning rate to 1 :->{}\".format(relu4_test_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying Learning rate = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu\n",
    "\n",
    "relu5_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu5_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu5_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu5_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our relu network class\n",
    "network_relu_5 = Network_relu().to(device)  #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network\n",
    "\n",
    "#Training with learning rate = 10\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_5, learning_rate =10)\n",
    "    relu5_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu5_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_5)\n",
    "    relu5_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu5_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after changing learning rate to 10 :->[10.001666666666667, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Training accuracy after changing learning rate to 10 :->{}\".format(relu5_training_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy after changing learning rate to 10 :->[10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test accuracy after changing learning rate to 10 :->{}\".format(relu5_test_accuracy_per_epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# D question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding droput of 0.3 to the 2nd fully connected layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a neural network class which extends the nn.Module base class\n",
    "\n",
    "#Created a network with ReLU activation function with network having dropout\n",
    "class Network_relu_dropout(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight) #Initializing the weights to the 1st convolutional layer\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight) #Initializing the weights to the 2nd convolutional layer\n",
    "\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight) #Initializing the weights to the 1st linear layer\n",
    "        self.dropout = nn.Dropout(p = 0.3)  #Initializing dropout of 0.3\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight) #Initializing the weights to the 2nd linear layer\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight) #Initializing the weights to the output layer\n",
    "\n",
    "    \n",
    "    #Mapping input tensor to the outpur tensor\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "       # t = self.dropout(t) \n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = self.dropout(t) #Adding dropout\n",
    "        t = F.relu(t)   #relu activation\n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)\n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our network class\n",
    "network_relu_dropout = Network_relu_dropout().to(device) #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu with dropout\n",
    "\n",
    "relu_dropout_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu_dropout_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu_dropout_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu_dropout_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network with dropout\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    #Training on training loader\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_dropout, learning_rate =0.1)\n",
    "    #Storing accuracy and loss after every epoch on training data\n",
    "    relu_dropout_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu_dropout_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    #Testing on test loader\n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_dropout)\n",
    "    #Storing accuracy and loss after every epoch on test data\n",
    "    relu_dropout_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu_dropout_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Training Accuracy Obtained on adding dropout: 98.76666666666667\n",
      "Final Test Accuracy Obtained on adding dropout: 90.0\n"
     ]
    }
   ],
   "source": [
    "#Final train and test accuracy obtained after dropout = 0.3\n",
    "print(\"Final Training Accuracy Obtained on adding dropout: {}\".format(relu_dropout_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Test Accuracy Obtained on adding dropout: {}\".format(relu_dropout_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABHi0lEQVR4nO3deZxN9f/A8dd7FsPMWMeQNRIKMTFJUikqidCqtGjRt76t2pdvy7fVt2/96qtF0aIolEhFQtaIjMguQsg2BsOYGWZ5//74nOEadxbMnTtm3s/H4z7uvWd9nzN3zvt8Pp9zPkdUFWOMMSa3kGAHYIwxpmSyBGGMMcYvSxDGGGP8sgRhjDHGL0sQxhhj/LIEYYwxxi9LEKWIiPwgIrcU9bTBJCLrRaRzsOMwh4jI3SKyTURSRCSmmNapInJqcazLHGIJIsi8f7KcV7aIpPl873M0y1LVy1T106KetiTyElzOfsoQkQM+398/huU9LyLDAxFraSIi4cD/AZeoajRwhohsCnJYQSEiDbzEFRbsWAKl1G7YicL7JwPc2TJwh6pOyT2diISpamZxxlaSqeplOZ9FZCiwSVX/FbyIAktEBBBVzQ5yKDWB8sCyolhYUfyuRSRUVbOKIh5zOCtBlFAi0lFENonI4yKyFfhERKqKyPcikigiu7zPdX3mmS4id3if+4rIzyLyujftOhG57BinbSgiM0Vkr4hMEZF38zrbLmSML4rIbG95k0Skus/4m0TkLxFJEpGnj3HfdRORRSKyW0TmiEhLn3GPi8jf3rpXiUgnEekCPAVc55VAfs9juU+IyJ/evMtFpFeu8f1EZIXP+Nbe8HoiMsbbJ0ki8o43/LBSS+4zUm9fvSwis4FU4BQRudVnHWtF5B+5YujhbfseL9YuInKNiCzINd3DIvJNHtvpdx0i0gRY5U22W0SmAT8AtX1Kb7VFJMRnXyWJyJciUi3XNt4uIhuAqXnE8KiIbBGRzSJyW65xQ0VkkIhMEJF9wIUicrq3v3aLyDIRuSLX9O+LyGRvm2aIyMk+49uLyHwRSfbe2/uMO6yKM9ffbKbPvkgRkXP8bcsJTVXtVUJewHqgs/e5I5AJ/AeIACoAMcBVQCRQEfgK+MZn/um4EghAXyAD6AeEAncDm3FnoUc77S/A60A5oAOwBxiexzYUJsY/gSbeNk0HBnjjmgEpwPneNv+ftw86F7DfhgIveZ9bA9uBs71tucXbrxFAU2AjUNubtgHQyPv8fF7b5LOea4DauBOr64B9QC2fcX8DZwECnAqc7MXwO/AmEIU7++7gb51ePAqE+eyrDUBzXGk/HLgcaOSt4wJc4mjtTd8WSAYu9mKsA5zmbftO4HSfdS0ErspjO/NbR+4YO+JKb77zPwjMBep66/4AGJFr/s+8/VHBz/q7ANuAFt40X3jznOrz904GzvW2syKwBpfkywEXAXuBpj7T7+XQ7+p/wM/euGrALuAmbx9f732Pyf0/mftvlntflMZX0AOwl88f48gEcQAon8/0ccAun+/TOfygv8ZnXKT3Yz7paKYF6uMO0pE+44dTwMG0gBj/5fP9n8BE7/OzwEifcVHePjiaBDEIeDHX+FW4A92puOTRGQjPNc3Bf/yj+HstAnp4n38EHvAzzTlAor+DSO515j7gePvqhQJi+CZnvbgD8Zt5TDcIeNn73Bx3EIwo5Hb6riN3jB05MkGsADr5fK+FOwEJ85n/lHzW9zHeSYP3vQlHJojPfMafB2wFQnyGjQCe95ne93cVDWQB9XCJ4ddc6/8F6Jv7fzL33yz3viiNL6tiKtkSVTU954uIRIrIB14VzB5cEbeKiITmMf/WnA+qmup9jD7KaWsDO32GgTsL96uQMW71+ZzqE1Nt32Wr6j4gKa915eFk4GGvqmG3iOzGHQhqq+oa3Nnt88B2ERkpIrULu2ARudmn6mo37gw3p3qsHq5klFs94C899nr2w/a1iFwmInNFZKcXQ9dCxADwKXCDiAjuoPilqu73N2EB6yiMk4GxPvtpBe6AXDOv7cqldq7xf/mZxnd8bWCjHt4+8xeuBHXE9KqagitR1fZeuZefe94yyxJEyZa7q92HcdUkZ6tqJVyRGVxVQKBsAaqJSKTPsHr5TH88MW7xXba3zqO9jHIj7ky5is8rUlVHAKjqF6raAXcQU1wVHhy5rw/j1VkPAe7FVT9UAZZyaLs24qpl/MVTX/xf6bIPV1rLcZKfaQ7GJSIRwNe46r6aXgwTChEDqjoXVxo7D7gBGOZvukKsI8/4fGwELsv1Nyivqn8XMF+Ow34HuFJsfuvdDNQTEd/jWX1clV8O399VNK5qabP3OpnD+c6b39+o1HeFbQnixFIRSMM1ilUDngv0ClX1LyABeF5EynkNcd0DFONooJuIdBCRcsALHP1vdAhwl4icLU6UiFwuIhVFpKmIXOQdBNO9OHOuftkGNMh1kPEVhTsgJIJryMWVIHJ8CDwiIm289Z7qJZVfcQe8AV4s5UXkXG+eRcD5IlJfRCoDTxawbeVwdeiJQKa4Cwku8Rn/EXCruIb3EBGpIyKn+Yz/DHgHyFTVn49xHbltA2K8+HO8D7yc0xAsIrEi0qOAbfP1JdBXRJp5JwkF/Ybm4Q7kj4lIuIh0xP1GR/pM09Xnd/UiME9VN+KSXxMRuUFEwkTkOlxb2PfefIuA3t5y44GrfZaZCGQDpxzFtp1QLEGcWN7CNezuwDUCTiym9fbB1aUnAS8BowC/1RMcR4yqugy4B9couQVXT35U19iragKusf0db/41uDYWcAe+AV5sW4EauIZNcI3pAEki8puf5S4H3sDVT28DzgBm+4z/CnjZi30vrt6+mrrLL7vj2j82eNtznTfPZNy+XAws4NBBKa9t2wvcjzuA7sKVBL71Gf8rcCuuQTwZmMHhZ8fDcEnNb+mhMOvwM/1KXH3/Wq9KqTauEfhbYJKI7MX9Ds7Ob9tyLfMH3O9oKu7v5/dKJ5/pDwBXAJfh/rbvATd7seX4ApdodgJtcL9pVDUJ6IYr+SYBjwHdVHWHN98zuFLZLuDf3nJy1puK+5vP9ra9XWG38USRc5WKMYUmIqOAlaoa8BKMKToiUgHXSN9aVVcHO57iImXgPplAsRKEKZCInCUijbxqiy5AD9wZsjmx3A3ML0vJwRwfu5PaFMZJwBhcg/Em4G5VXRjckMzREHeXvgA9gxuJOZFYFZMxxhi/rIrJGGOMX6Wqiql69eraoEGDYIdhjDEnjAULFuxQ1Vh/40pVgmjQoAEJCQnBDsMYY04YIuLvTnXAqpiMMcbkwRKEMcYYvyxBGGOM8csShDHGGL8sQRhjjPHLEoQxxhi/LEEYY4zxq1TdB2GMMaWJqrJt3zaWbV/G8sTl7Erf5R4H6j2rKOdzdLloHjv3sSJfvyUIY4wpIfbu38tXy78iYXMCS7cvZVniMnam7SxwvpOiT7IEYYwxR2ND8gbem/8eq5JWcUurW+jepDuhIXk9wv3YLNu+jKGLhrIqaRUVIypSqVwlKkUcetWIqkG7uu04uUruJ5se8vvW3xmUMIjPl3xOyoEUKkdUpnmN5lx1+lU0j21O8xrNaR7bnBpRNRARBME9XjywLEEYY0oVVWXGXzMYOG8g41aNAyA2MpZvVn7DKVVP4f6293PrmbdSKaLSMa9jV9ouRi4dySeLPmH+5vmEhYTRLLYZ+w7sY8/+PezZv4f9WYc/dLF+5fqcV/889zr5PBpWacjXK75mUMIg5mycQ/mw8vRu0Zu72txF2zptiyUBFKRUdfcdHx+v1heTMWVLtmaTciCF5PRkfljzA2//+jZLty8lpkIM/Vr34+6z7qZ2xdp8s/Ib3pr7FrM3zqZiuYrcfubt3NH6DjKzM9m0Z9Oh195NbN67mXKh5ahSvgqVIypTOaIyVcpXIbpcNDP+msE3K79hf9Z+WtZsya1xt9LnjD7ERh3e393+zP3sPbCXjckb+XnDz8zcMJNZf81i275tAIRKKFmaReNqjbkr/i76xvWlWoVqxb7/RGSBqsb7HWcJwhhTku07sI+l25eyZPsSFm9bzJLtS9iWsu3gmfreA3sPm/7Mk87kvrb30btFbyqEVzhiefP/ns//5v2PUctGkZmdedi4EAmhVnQtalWsRWZ2JsnpyexO303y/mSyNRuAahWq0eeMPtwadytxJ8Ud1Zm+qrJm5xpmbZjFsu3LuKzxZVzU8CJCJHgXlFqCMMaUaNmazea9m1mzc83B1x9Jf7Bk+xL+3Pnnwat2ostF06JGC+pVqkeliEpULFfxsPr+FjVa0K5uu0IdtDfv3cz4P8ZTrUI16laqS91KdakZXZOwkCNr3lWVfRn7SE5PJjYqlnKh5Yp8HwSLJQhjTMCpKit3rGRD8gayNIus7CwyszPJUveenplOUmoSSWlJB993pu1k275trN21lvTM9IPLKhdajlOqnkLz2Oa0rNny4KtBlQZBPdsujfJLENZIbYzxKz0znS17t1C1QlUqR1T2e1aenJ7MT+t+YuKaiUxcM5GNezYWuNxQCSUmMoaYCjFUq1CNJjFNuOzUyzi12qkHX/Uq1Svyq43M0bMEYUwZpqqsSlrFnI1zWLtrLet3r2fd7nWs27WOLSlbDk5XIawCtSrWonbF2tSuWJsakTVYtG0Rv2z8hSzNolJEJTo17MS/zv8XzWObExYSRmhIqHsX9x4RFkFMhRgqRVQqEVfomIJZgjCmjNmZtpOf1v7EpD8nMWntJDYkbwDcmX29yvVoUKUBXU7tQoMqDahTsQ6703ezJWULm/duZkvKFhZtXcTWlK00rtaYx899nC6ndqFd3XaEh4YHectMUQtoghCRB4B+gABDVPUtERkFNPUmqQLsVtU4P/OuB/YCWUBmXnVkxpj8JacnM2fjHGZtmMXUdVOZv3k+2ZpN5YjKdDqlE091eIoLG15IwyoN7SBvDhOwBCEiLXDJoS1wAJgoIuNV9Tqfad4AkvNZzIWquiNQMRpT0mVrNgmbE6gRVYP6lesX2ECblZ3Fpj2bmL95PjP/msmsDbNYvG0x2ZpNWEgYZ9U+i2fOf4ZLGl1C2zpt/V6xY0yOQP46TgfmqmoqgIjMAHoBr3nfBbgWuCiAMRhzwpqzcQ73/3A/C7YsAFw7QJOYJjSt3pTTYk6jcUxjdqfvPnhZ6J+7/mTtrrUcyDpwcPpz6p3DM+c/w/knn8/Zdc4mqlxUMDfJnGACmSCWAi+LSAyQBnQFfK9BPQ/Ypqqr85hfgUkiosAHqjrY30QicidwJ0D9+vWLKnZjgmbz3s08PuVxhi8eTu2Ktfmg2wcIwsodK1mVtIoFmxcwevnogzduRYVH0ahaI5rFNqN7k+40qtqIVie1ok2tNlZlZI5LwBKEqq4Qkf8Ak4EU4HfA97bF64ER+SziXFXdLCI1gMkislJVZ/pZz2BgMLj7IIpsA4wpZvsz9/PW3Ld4ceaLZGRn8FSHp3jyvCeJLhftd9p1u9dRpXwVakbVtKuCTEAEtAJSVT8CPgIQkVeATd7nMOBKoE0+82723reLyFhcW8YRCcKYYFNV0jPTSc1IJTUjlbTMNFIzUqkQVoHYqFiqlK/it+0gKTWJVUmrWLVjFauSVvH1iq9Zs3MNPZr24I1L3qBRtUZ5rjMiLILTqp8WyM0yJuBXMdXwDvD1cQnhHG9UZ2Clqm7KY74oIERV93qfLwFeCGSsxhSWqjJn4xyGLR7GmBVjSExNzHf6nBvDYiNjiY2K5UDWAVbtWEVSWtLBacJDwok7KY4fb/yRSxpdEuhNMKZQAn0Jw9deG0QGcI+q7vKG9yZX9ZKI1AY+VNWuQE1grFdsDgO+UNWJAY7VmHyt2bmGYb8PY/iS4azdtZbI8Eh6NO1B42qNiQyPJDI8kgrhFdx7WAVSM1LZkbqDxNREEvcluvfURMJDwrnq9KtoWr0pTWOa0rR6UxpUaWBXFJkSx/piMiYfB7IO8OWyL3lv/nv8sukXBOGihhdxU8ubuPL0K6kYUTHYIRpzXKwvJmOO0s60nXyQ8AHvzH+HzXs30ySmCQM6DaBPyz7UrVQ32OEZUywsQRjj44+kP3hr7lt8+vunpGak0vmUzgzpPoQup3axXkRNmWMJwpQpS7cv5dHJj5KwOYFszUZV3Tt68Mlk5ULLceMZN/Jguwc5o+YZwQ7ZmKCxBGHKhKTUJJ6d9izvL3ifyhGVubb5tYSHhCMihEjIwYfA14yqSd+4vtSMrhnskI0JOksQ5oS3M20nkeGRlA8rf8S4jKwMBiUM4vnpz7Nn/x7+Gf9Pnu/4PDGRMUGI1JgTiyUIc8Lad2AfT099moHzBqIotaJr0bBqQxpWaUiDKg2oGVWTQQmDWLFjBRefcjFvXvomzWs0D3bYxpwwLEGYE9L09dO5/dvbWbtrLf1a96NepXruQTe71/Hzhp8ZsXQE2ZrNqdVOZVzvcXRv0t26ozDmKFmCMCeUvfv38viUxxmUMIhGVRsx/ZbpXNDggiOmy8jKYEvKFmpF17IO64w5RpYgzAlj0p+T6PddPzYmb6R/u/68dNFLRIZH+p02PDSc+pWtd19jjoclCFPizdk4h1dmvcL41eM5rfppzL5tNufUO6fgGY0xx8UShCmRVJUpa6fw8qyXmfHXDGIqxPDShS/xcPuH/V6tZIwpepYgTImSlZ3Fd398xyuzXmH+5vnUqViHNy99k36t+9nT0IwpZpYgTNDs3b+XJduX8PvW3/l92+8s2rqIJduXkJqRyilVT2Fwt8Hc3OpmIsIigh2qMWWSJQhT7FYkruCfE/7J9PXTDw6rUr4KrWq2ol/rfnSo34Gep/W07q+NCTL7DzTFZn/mfgb8PIBXfn6F6HLRPHv+s8TXjqfVSa2oV6me3adgTAljCcIUizkb59Dvu34sT1zO9S2u560ub1EjqkawwzLG5COg/ReLyAMislRElonIg96w50XkbxFZ5L265jFvFxFZJSJrROSJQMZpAmfP/j3cO+FeOnzcgZQDKYy/YTxfXPWFJQdjTgABK0GISAugH9AWOABMFJHx3ug3VfX1fOYNBd4FLgY2AfNF5FtVXR6oeE3RUVV+/ftXvljyBSOWjmBH6g7uP/t+XrroJaLLRQc7PGNMIQWyiul0YK6qpgKIyAygVyHnbQusUdW13rwjgR6AJYgSbHni8oNJYe2utUSERnB5k8t5rP1jnF337GCHZ4w5SoFMEEuBl0UkBkgDugIJQBJwr4jc7H1/WFV35Zq3DrDR5/smwO8RRkTuBO4EqF/fulYoblnZWYxaNor/zvkvi7YuIkRC6NSwE8+c/wy9TutF5fKVgx2iMeYYBSxBqOoKEfkPMBlIAX4HMoFBwIuAeu9vALflmt3f5Syax3oGA4MB4uPj/U5jil5WdhZfLvuSF2a+wModK2lRowUDuwzk2ubX2sN2jCklAnoVk6p+BHwEICKvAJtUdVvOeBEZAnzvZ9ZNQD2f73WBzQEM1RRSVnYWXy3/ihdmvMCKHStoHtucr675iitPv9Ke2WxMKRPQBCEiNVR1u4jUB64EzhGRWqq6xZukF64qKrf5QGMRaQj8DfQGbghkrCZ/2ZrNmBVjeG76cyxPXE6z2GaMunoUVze72hKDMaVUoO+D+Nprg8gA7lHVXSIyTETicFVG64F/AIhIbeBDVe2qqpkici/wIxAKfKyqywIcq/FDVfnxzx95eurT/LblN06rfhojrxrJ1c2uJjQkNNjhGWMCKNBVTOf5GXZTHtNuxjVk53yfAEwIXHSmILP+msXTU59m1oZZNKjSgE97fkqfM/pYYjCmjLA7qc0Rlm5fyqOTH2XimonUiq7Fe13f4/bWt1MutFywQzPGFCNLEOagbM1m4LyBPD7lcaLLRfNa59e4p+09eT61zRhTulmCMABsTdlK32/68uOfP9K9SXc+uuIjYqNigx2WMSaILEEYvv/je24bdxt7D+zlva7vcVf8XdazqjHGEkRZlpaRxqOTH+Xd+e/SqmYrvrjqC5rFNgt2WMaYEsISRBmUmZ3J54s/58WZL/Lnrj/p364/r3Z61Z7cZow5jCWIMiQzO5Phi4fz0syX+HPXn8SdFMfkmybT+ZTOwQ7NGFMCWYIoAzKyMhi2eBgvz3qZtbvWcuZJZzKu9zi6N+lubQ3GmDxZgijl5v89nz5j+rB652ra1GrDt72/pVuTbpYYjDEFsgRRSqkqb//6No9MeoRaFWtZYjDGHDVLEKXQ7vTd3P7t7YxZMYbuTboztOdQqlWoFuywjDEnGEsQpcyCzQu4dvS1bEjewOsXv85D5zxkpQZjzDGxBFFKqCrvzX+PhyY9RM2omszsO5Nz6p0T7LCMMScwSxClQGpGKv2+68cXS76ga+OufNbzM2IiY4IdljHmBGcJ4gS3btc6eo3qxeJti3nxwhd56ryn7AE+xpgiYQniBDbpz0n0Ht0bRRl/w3gua3xZsEMyxpQidqp5AlJVBvw8gC7Du1C3Ul0S+iVYcjDGFLlAP5P6AaAfIMAQVX1LRP4LdAcOAH8Ct6rqbj/zrgf2AllApqrGBzLWE0XKgRRuHXcro5ePpneL3nzY/UOiykUFOyxjTCkUsBKEiLTAJYe2QCugm4g0BiYDLVS1JfAH8GQ+i7lQVeMsOTjJ6clcPOxixqwYw+sXv84XV35hycEYEzCBrGI6HZirqqmqmgnMAHqp6iTvO8BcoG4AYyg1ktOTuXT4pSRsTmD0NaN5uP3Ddn+DMSagApkglgLni0iMiEQCXYF6uaa5Dfghj/kVmCQiC0TkzrxWIiJ3ikiCiCQkJiYWSeAlze703Vw87GJ+2/Ibo68ZTa/TewU7JGNMGRCwNghVXSEi/8FVKaUAvwM5JQdE5Gnv++d5LOJcVd0sIjWAySKyUlVn+lnPYGAwQHx8vBbxZgTdzrSdXDLsEpZsX8LX135N96bdgx2SMaaMCOhVTKr6kaq2VtXzgZ3AagARuQXoBvRRVb8HdVXd7L1vB8bi2jLKlKTUJDp/1pkl25cw9rqxlhyMMcUqoAnCO/tHROoDVwIjRKQL8Dhwhaqm5jFflIhUzPkMXIKrsiozdqTuoNNnnVieuJxxvcfRtXHXYIdkjCljAn2j3NciEgNkAPeo6i4ReQeIwFUbgWvIvktEagMfqmpXoCYw1hsfBnyhqhMDHGuJ8dfuv+j6RVfW7lrLuN7juPTUS4MdkjGmDApoglDV8/wMOzWPaTfjGrJR1bW4S2PLnN+2/MblX1xOWkYaP/T5gY4NOgY7JGNMGWV3UpcgE9dM5PxPzic8JJzZt8225GCMCSpLECXER799RLcvutE4pjFz75hL8xrNgx2SMaaMswQRZKrKs9Oe5Y7v7qDzKZ2Z2XcmtSvWDnZYxhhjvbkGk6rS77t+fLTwI24/83YGXT6I8NDwYIdljDGAlSCCauC8gXy08COe6vAUQ7oPseRgjClRLEEEyewNs3lk8iP0PK0nL130kvWrZIwpcSxBBMH2fdu5dvS1NKjSgKE9hlpyMMaUSNYGUcwyszPpPbo3O9N2MuGGCVQuXznYIRljjF+WIIrZs9OeZdr6aXzS4xNanVQm7wU0xpwgCqxiEpFuImJVUUXgu1Xf8erPr9KvdT/6xvUNdjjGGJOvwhz4ewOrReQ1ETk90AGVVmt3reWmsTfRulZrBl42MNjhGGNMgQpMEKp6I3Am7vnRn4jIL95DeioGPLpSIj0znau/vBoRYfQ1oykfVj7YIRljTIEKVXWkqnuAr4GRQC2gF/CbiNwXwNhKjUcnPcrCrQsZ1msYDas2DHY4xhhTKIVpg+guImOBqUA40FZVL8P1tvpIgOM74X2z8hvemf8O/dv1p1uTbsEOxxhjCq0wVzFdA7yZ+3GfqpoqIrcFJqzSYWPyRm4bdxuta7Xm1U6vBjscY4w5KoVJEM8BW3K+iEgFoKaqrlfVnwIW2QkuMzuTPmP6kJGdwcirRhIRFhHskIwp1ebOBVU455xgR1J6FKYN4isg2+d7ljesQCLygIgsFZFlIvKgN6yaiEwWkdXee9U85u0iIqtEZI2IPFGY9ZUkL818iVkbZjHo8kE0jmkc7HBMCbF9O/zwAyxeDLt2uQOaP3v2wO+/w9ix8PnnsH9/4GNThcRE+O03GDcO3nkHXnnFxVxYf/4J69YFLkZ/9uyBu+5yieHCC+GXX4p3/cVhyxZ48EG3nZMmQUZGMa1YVfN9AYv8DPu9EPO1wD1HOhJXUpkCNAZeA57wpnkC+I+feUNxV02dApQDfgeaFbTONm3aaEkwfd10Dfl3iN489uZgh6KpqapTp6pu2BDsSMyWLar16qm6Q7F7RUWpnnaa6sUXq151lWp8vGq1aodPA6rXXaealRWYuL7/3sVQvvyR6wXV009X3by54OVMm6YaHe226euvAxNrbuPHq9atqxoSovrAA6qNGqnWqKG6fn3xrD/QUlJU//1vt0/Dw93+BdWqVVX79lX97jvV9PTjWweQoHkdx/MacXACmAxc4fO9B/BTIea7BveM6ZzvzwCPAauAWt6wWsAqP/OeA/zo8/1J4MmC1lkSEsSOfTu0zht1tPHAxrp3/97gxLBDdehQ1V69VCMjD/2jt27tfmy//66anR2U0Mqs1FTVs892f4+vv1b98kvVN95wB7Urr1Q96yzVpk1VL7lE9a67VP/zHzdNQoLqyy+7v9+jjxZ9XNOmqUZEqDZrpvrww6pvvaU6Zozq/PmqW7e68VFRqk2aqG7alPdyvv/eJZhmzdx2gupzzwUuqe3YoXrjjW49zZurzp3rhq9YoVq5suoZZ6ju2ROYdReHzEzVDz9UrVXLbePVV6uuWaOalqY6bpzqTTe57QTVSpVU+/RRPXDg2NZ1vAmiETAX2ABsBOYApxZivtOBP4AYrxTxC/A2sDvXdLv8zHt1ruRyE/BOHuu5E0gAEurXr39se6iIZGdna/cvumu5F8vpb5t/K6Z1qm7b5v6R33hD9YIL3NkUqNapo3r33e4HNWCA6jnnqIq4cQ0auIPTsmWFX9cXX6j27Km6fXuANibAUlJUFy50B94ff1Tdv7941pudrXr99W6/jxlzbPP/859u/nfeKbq4EhJUK1Z0B/UdO/Ke7uef3XSNGqn+9deR40eNUg0LU23TRjUx0R3E+vZ18fbqpbr3GM+TsrNVk5NdaWDRIvcbHzNG9X//c6WEsDDVZ5898gx60iTV0FDVbt3cgbY4fPCB6oUXuv+3t99WnTLFlbqO5URs8mSX4EC1XTvV2bP9T7d/v+oPP6jefrtqly7HHvtxJQg9dCCOBioWdnpvntuB34CZwPvAm4VMELlLHzcBbxe0vmCXIN785U3lefStX9467mWNH696772qjzyi+q9/ubPIN95Qffdd1TffVL3zTtUOHVRjYg6VEEC1RQvVp592Z4D+fpxbtqgOGaJ6+eXuzDEiwi0zvx9yRoZq//6H1tG+vTsIBNqBA6rz5qm+9pr7Z2/cWLVrV9UnnnDJaunSw8+aMjLcAeznn1VHjHBn4Xfeqdqxo0uWuatOqlRRvflml0D9bU9mpurKlapffaX6wguq99yjeu217kBwxhmqJ53kSgR33eUOZHn597/d+l599dj3RWamavfuLvmPG3fsy8mxYoVq9eqqJ5+cf8kgxy+/uDPWBg1U1607NPyjj1xM5513+D7Izna/05AQ95v888/Dl7dpk/sb/uMfqm3bulJAo0aqtWu7KrYKFQ6dzPh7tWnjSsJ5efddN93DDxd+nxyrnTtdAj3ppENn9b6/sa5d3UlcYYwa5bb7lFPc7644Svr5JQhx4/MnIpcDzYGDtwCr6gsFznj4Ml4BNgEPAB1VdYuI1AKmq2rTXNOeAzyvqpd635/01pnvtaLx8fGakJBwNGEVmTkb53DB0Avo1qQbY64dc1xdeH/6Kdx6K0RGup9ZWtqRjZnVqkGzZodep58OzZtDnTqFX8/27XDLLTBxIvTqBR99BFVzXTKQmAi9e8PUqXDffa4h8IYb4NprYcQICCniXrq2boWPP4YZM2D2bNi3zw1v0gRatIA1a2DFikONdOXKwamnuobKzZshO/vw5cXEuHmbNIHGjQ+9b9oEX3/tGmN37YLoaLj8cmjTBlaudI3Iy5a5fZ+jalWIjXWv6tXd+4EDMHw41K4NH3wAXbsevv5Ro9z+u/lmGDoUjqdn9337XCPs0qUwfTq0bXtsy9mwAc491+3Dn392+68wEhLgkkvcvpo2Db77Dvr3h0svhTFj3O81tylT3G9FBJ5+2u3TGTNcYzZApUpw1llQpQqUL3/oVaGCe69c2e33qlXdNDmf69cv+Ld3332uoX3IELjjjsLvH1WYNQvi4/1vU24vvADPPQeLFkHLlu43vHy5+50uW+b+n0891f0PVa+e93ImT3a/wXbt4Mcf3T4oDiKyQFXj/Y7MK3PkvHBn/p/hqpeeA5YAHxU0nzdvDe+9PrASqAr8l8MbqV/zM18YsBZoyKFG6uYFrS9YJYhtKdu0zht1tNH/GumutF3HtawhQ9wZROfOqvv2uWHZ2a44mZzszkQSE4vuzCIrS/W//3XF9fr13dl3jgUL3LCICNemkeM//3FnR088kf+y165VveEGVz9dGKmph4rWLVq4apVRo45sIN2/X3XxYtXhw1Ufe0z1iitUb7nFlbQ++MAVu5cuzf+sPseBA666qV8/1dhYt+7YWLf/H3rIbfdvv+VfYpo711XTgKsbzqmumTfP1ct36HD8DYk5tm5VbdjQxZj7rLwwtm1z7QmVK7tqm6O1cKErteacKV95ZcHbtnr1of1TrZpqjx6q//d/7vcVyCqgjAzVSy91v+2pUws3T3a26lNPuVhvuaXg6ZOTXYPxFVfkPc2UKe530LJl3lV58+a5tp6WLVV37SpcrEWF42yDWJzrPRqYVNB83rSzgOXeAb6TNywG+AlY7b1X84bXBib4zNsV14bxJ/B0YdYXjASRmZWpnT/rrBEvRujCLQuPa1nvvef+Il26FE8Vjq9581yxNjRU9aWXVD/7zP2o69VzddW+srNd1QCoDh585LKyslwRPyrKTRMZmX91QI5+/dz0EyYUzTYdrYwMl3yPRXq66jPPuINRjRqq77/vqhwaNiz6NpuVK92BtkkT1U8+UR00yFXnvPKKq5N/7DFXzfj6666h8+uv3QFy/nzVM8901Te+JwJHa8kSd+XQbbe5fVYYaWmqf/wRuEbrvOze7ZJTpUqq336b/7TZ2a5KF1x1FxS8n1591U3366/5TzdpkjvRiotTTUo6fNyKFS7pnnJK4a4WK2rHmyB+9d7negfxCGB1QfMF4xWMBPHs1GeV59EPF3yY5zTLl7srVK65xp2t+vsneest99fo3r3ozjaP1u7d7nLKnPrTCy7Iu+40I8MlstBQt0051q1zdfTgLt2cN8/VKzdsmH9D6OefF65UUtItWuTqx3OuLlm6NDDr+flnd6D3Vz8fEXHoQoXcr/BwV8I6XifSVXB//eWu4APVxx/3n9Sys91FG+Da/vbudSdHrVrlnQRTUlw7zqWXFi6OiRPd3+bMM127harqxo1uPTVruquUguF4E8QzQBXgKmAr7q7qFwqaLxiv4k4QP6z+QeV50b7f9NXsPP5jtmxxDYExMYeubz/5ZNfouXGjm+a//9WDV3wU15U1ecnOVv30U9UXXyz4srnkZFckrljRlRDee8+VGipWdCWLnF0yd65quXKqnTr5/2dbtcpd333uuYU/Iy3JMjJcCWLOnMCuZ+dOl5C3bHHVEmlph/Z5VtahK4AWLjx0BVBhSnKlUVraoVLvBRe4fZYjK8tdfQTuYoycfTh6tBs2cKD/Zf7f/2mhShm+Jkxw/wtt2riEcPrp7kTit+K56NGvY04QuDut2/t8jwAq5zdPMF/FmSD+2v2XxvwnRlsOaqn7DuzzO83eve7MJTLSVdOkpamOHOkOlODO8nKuGb/mmmO/jjmYNm50JYTwcLcdnTv7vxTy44/d+IceOnx4aqo7S4uJOZQwjQmUzz5zJa+TTlKdPt0lhzvucL/Nxx47vGSUne3uS6lUybX9+EpLc/codOx49DF8/737fwkLcyWK6dOPb5uO1/GWIH4paJqS8iquBLE/c7+ePeRsrfhKRf1jxx9+p8nIcJeShoT4b6T9809XT1yvnmsMO5HPnBcudHcAf/BB/lUP993nfnGffXZoWM5Z3fjxAQ/TGFV1bShNmrj/zXPPdb+/f/3L/2931Sp3ML85V6cIOZfR/vTTscXw7bfuf3/s2GObvygdb4L4t1e9JAVNG+xXcSWIl2a8pDyPjl422u9430bc998vlpBOCAcOuDOuiAjXYDpixKEzN2OK05497p4WcPep5CfnqqaZM933/fvdwb19+xOrLSYv+SWIAu+DEJG9QBSQCaQD4q6O1Ur5zhgExXEfhKrSaGAjGlVrxOSbJvudZsAAePJJeOIJeNV6+T5MYqK77j0zE5KT3XXj06dDeHiwIzNljaq7H6Zevfyn27fP3WtUubLryHDoUOjXDyZMgMsuK5ZQAyq/+yAK88jRiqoaoqrlVLWS973EJYfiMnvjbNbtXsdNLW/yO/6LL1xyuOEGePnlYg7uBBAb63oo3bnT3eQ2cqQlBxMcIgUnB4CoKHjrLViyBP73P3fS16YNdOkS8BCDrsDnQYjI+f6Ga64HCJUVwxcPJzI8kitPv/KIcd9+6+6AvuACdzdwUd9lXFqceaa7g7d8+cL9gxoTbD17uoTw6KOu5DF27PHdFX+iKMwDgx71+VweaAssAC4KSEQl2P7M/Xy57Et6ndaL6HLRh40bOND11x4f7348EfZ8oHy1bh3sCIwpPBH3P96iBTRtCldcEeyIikeBCUJVu/t+F5F6uGc6lDnjV49nV/oubmx548FhWVnw0EPux9Orl+uXpzD9txhjTiyNG7s+qGrXLju1A4UpQeS2CfcwoDJn+OLh1IyqSedTOgOu8eqGG1zVUv/+8N//QmhokIM0xgRM+/bBjqB4FaYN4m0g51KnECAO17dSmbIzbSff//E997a9l7CQMLZuhW7dYOFC12PkPfcEO0JjjClahSlB+F43mgmMUNXZAYqnxPpy2ZdkZGdwU8ubWLnSdXGclOS6i+7WLdjRGWNM0StMghgNpKtqFoCIhIpIpKqmBja0kmX44uE0i21GvfA42nZ1D5GfOdMaW40xpVdhmlp+AnwfXVEBmBKYcEqmtbvWMnvjbG5odjO9ewt//+1KDpYcjDGlWWFKEOVVNSXni6qmiEiZuk5n+OLhAPw1+p/89JO7x+Hss4MclDHGBFhhShD7ROTgubKItAHS8pm+VFFVhi0exml/v8yQdyty333uZjhjjCntClOCeBD4SkQ2e99rAdcVZuEi0h+4A3cV1BLgVuBTIOcZ1FWA3aoa52fe9cBeIAvIzKuvkED79e9fWbOkMmGfPkbHjvDGG8GIwhhjil9hbpSbLyKn4Q7qAqxU1YyC5hOROsD9QDNVTRORL4HeqnqdzzRvAMn5LOZCVd1R0LoC6YMZ38Cob6h9Ughffmn9Bhljyo4Cq5hE5B4gSlWXquoSIFpE/lnI5YcBFUQkDIgEckohiIgA1wIjjj7s4pGSdoBh/7qC0PRYvvkmhNjYYEdkjDHFpzBtEP1UdXfOF1XdBfQraCZV/Rt4HdiAe0xpsqpO8pnkPGCbqq7OaxHAJBFZICJ35rUeEblTRBJEJCExMbHgrTkKN9+7gcz159D/1WWceWaRLtoYY0q8wiSIEO9sH3D3QQDlCppJRKoCPYCGQG0gSkRu9JnkevIvPZyrqq2By4B78ulVdrCqxqtqfGwRnuJnZ8OE0bFEtBzHKw+cUWTLNcaYE0VhEsSPwJci0klELsId1H8oxHydgXWqmui1WYwB2gN4VU5XAqPymllVN3vv24GxuF5ki83ixbB/T2WanrOa8FBreDDGlD2FSRCP426Wuxu4B1jM4TfO5WUD0E5EIr0SSCdghTeuM66xe5O/GUUkSkQq5nwGLgGWFmKdRWay97C4xmdtKM7VGmNMiVGYJ8plA3OBtUA8hx/o85tvHq6bjt9wl7iGAIO90b3JVb0kIrVFZIL3tSbws4j8DvwKjFfViYXZoKIyeTKE1FhOvTrWPasxpmzK8zJXEWmCO5BfDyThVQep6oWFXbiqPgc852d4Xz/DNgNdvc9rgVaFXU9RS0+HWbOU7FaTqB5ZPVhhGGNMUOVXgliJKy10V9UOqvo27qa1Uu/nnyE9XaDRZGIiY4IdjjHGBEV+CeIqYCswTUSGiEgn3I1ypd6UKRAWng0nzyCmgiUIY0zZlGeCUNWx3l3PpwHTgf5ATREZJCKXFFN8QTF5MjQ7cw9E7LMqJmNMmVWYRup9qvq5qnYD6gKLgCcCHViw7NjhnhLX7Gx3gZVVMRljyqqjevS2qu5U1Q9U9aJABRRsP/0EqnBya3eDt5UgjDFl1VEliLJg8mSoXBki67srea0NwhhTVlmC8KHqEsRFF8HujB1EhUcRERYR7LCMMSYoLEH4WL0aNmyAiy+GpLQkq14yxpRpliB8TPGetH3xxbAjdYc1UBtjyjRLED4mT4YGDaBRI0hKTbL2B2NMmWYJwpOZCVOnutKDiCtBWBWTMaYsswThmT8f9uyBzp3d96Q0K0EYY8o2SxCeyZNdyaFTJ8jMzmR3+m4rQRhjyjRLEJ7Jk6F1a4iJgZ1pOwG7i9oYU7ZZggD27oW5c137A7gGarCb5IwxZZslCGDGDNdInZMgdqTuAKybDWNM2WYJAle9VKECtG/vvieleSUIq2IyxpRhAU0QItJfRJaJyFIRGSEi5UXkeRH5W0QWea+ueczbRURWicgaEQlo77GTJ8N550H58u57ThWTlSCMMWVZwBKEiNQB7gfiVbUFEIp7hCnAm6oa570m+Jk3FHgXuAxoBlwvIs0CEWd6OsTGQlefNJVTxWRtEMaYsizPZ1IX4fIriEgGEAlsBhoUYr62wBrv2dSIyEigB7C8qAMsX961QfhKSksiIjSCyPDIol6dMcacMAJWglDVv4HXgQ3AFiBZVSd5o+8VkcUi8rGIVPUzex1go8/3Td6wI4jInSKSICIJiYmJRRJ7UqrrqE+kTDxh1Rhj/ApkFVNV3Fl/Q6A2ECUiNwKDgEZAHC5xvOFvdj/D1N96VHWwqsaranxsbGxRhM6ONOuozxhjAtlI3RlYp6qJqpoBjAHaq+o2Vc1S1WxgCK46KbdNQD2f73Vx1VPFwjrqM8aYwCaIDUA7EYkUV1fTCVghIrV8pukFLPUz73ygsYg0FJFyuMbtbwMY62Gsoz5jjAlgI7WqzhOR0cBvQCawEBgMfCgicbgqo/XAPwBEpDbwoap2VdVMEbkX+BF39dPHqrosULHmZh31GWNMgK9iUtXngOdyDb4pj2k3A119vk8AjrgENtCyNZudaTutBGGMKfPsTupcdqfvJluzrZHaGFPmWYLIxTrqM8YYxxJELtZRnzHGOJYgcrGO+owxxrEEkYt11GeMMY4liFysoz5jjHEsQeSSlJZEWEgYlSIqBTsUY4wJKksQueR0s2Ed9RljyjpLELlYR33GGONYgsjFOuozxhjHEkQu1lGfMcY4liBysY76jDHGsQThQ1UPPk3OGGPKOksQPvYe2EtGdoY1UhtjDJYgDmMd9RljzCGWIHxYR33GGHNIQBOEiPQXkWUislRERohIeRH5r4isFJHFIjJWRKrkMe96EVkiIotEJCGQceawjvqMMeaQgCUIEakD3A/Eq2oL3KNDewOTgRaq2hL4A3gyn8VcqKpxqhofqDh9WRWTMcYcEugqpjCggoiEAZHAZlWdpKqZ3vi5QN0Ax1BoVsVkjDGHBCxBqOrfwOvABmALkKyqk3JNdhvwQ16LACaJyAIRuTOv9YjInSKSICIJiYmJxxVzUloSglClfJXjWo4xxpQGgaxiqgr0ABoCtYEoEbnRZ/zTQCbweR6LOFdVWwOXAfeIyPn+JlLVwaoar6rxsbGxxxVzUmoS1SpUIzQk9LiWY4wxpUEgq5g6A+tUNVFVM4AxQHsAEbkF6Ab0UVX1N7OqbvbetwNjgbYBjBWwjvqMMcZXIBPEBqCdiESK6zu7E7BCRLoAjwNXqGqqvxlFJEpEKuZ8Bi4BlgYwVsA66jPGGF+BbIOYB4wGfgOWeOsaDLwDVAQme5ewvg8gIrVFZII3e03gZxH5HfgVGK+qEwMVaw7rqM8YYw4JC+TCVfU54Llcg0/NY9rNQFfv81qgVSBj8ycpLYkza51Z3Ks1JmgyMjLYtGkT6enpwQ7FBFj58uWpW7cu4eHhhZ4noAniRJOUmkT1ClaCMGXHpk2bqFixIg0aNLCnKJZiqkpSUhKbNm2iYcOGhZ7PutrwpGakkpaZZo3UpkxJT08nJsYesVvaiQgxMTFHXVK0BOGxu6hNWWXJoWw4lr+zJQhPTj9M1khtjDGOJQhPTjcbVsVkTPFJSkoiLi6OuLg4TjrpJOrUqXPw+4EDB/KdNyEhgfvvv7/AdbRv376owgXggQceoE6dOmRnZxfpcksia6T2WBWTMcUvJiaGRYsWAfD8888THR3NI488cnB8ZmYmYWH+D1Px8fHExxfcj+ecOXOKJFaA7Oxsxo4dS7169Zg5cyYdO3YssmX7ysrKIjQ0+D06WILwWEd9pqx7cOKDLNq6qEiXGXdSHG91eeuo5unbty/VqlVj4cKFtG7dmuuuu44HH3yQtLQ0KlSowCeffELTpk2ZPn06r7/+Ot9//z3PP/88GzZsYO3atWzYsIEHH3zwYOkiOjqalJQUpk+fzvPPP0/16tVZunQpbdq0Yfjw4YgIEyZM4KGHHqJ69eq0bt2atWvX8v333x8R27Rp02jRogXXXXcdI0aMOJggtm3bxl133cXatWsBGDRoEO3bt+ezzz7j9ddfR0Ro2bIlw4YNo2/fvnTr1o2rr776iPj+/e9/U6tWLRYtWsTy5cvp2bMnGzduJD09nQceeIA773Td0k2cOJGnnnqKrKwsqlevzuTJk2natClz5swhNjaW7OxsmjRpwty5c6le/diPaZYgPDltENUqVAtyJMaYP/74gylTphAaGsqePXuYOXMmYWFhTJkyhaeeeoqvv/76iHlWrlzJtGnT2Lt3L02bNuXuu+8+4pr/hQsXsmzZMmrXrs25557L7NmziY+P5x//+AczZ86kYcOGXH/99XnGNWLECK6//np69OjBU089RUZGBuHh4dx///1ccMEFjB07lqysLFJSUli2bBkvv/wys2fPpnr16uzcubPA7f71119ZunTpwUtRP/74Y6pVq0ZaWhpnnXUWV111FdnZ2fTr1+9gvDt37iQkJIQbb7yRzz//nAcffJApU6bQqlWr40oOYAnioKTUJCpHVCY8tPA3kRhTmhztmX4gXXPNNQerWJKTk7nllltYvXo1IkJGRobfeS6//HIiIiKIiIigRo0abNu2jbp1D3+aQNu2bQ8Oi4uLY/369URHR3PKKaccPChff/31DB48+IjlHzhwgAkTJvDmm29SsWJFzj77bCZNmsTll1/O1KlT+eyzzwAIDQ2lcuXKfPbZZ1x99dUHD9LVqhV88tm2bdvD7lMYOHAgY8eOBWDjxo2sXr2axMREzj///IPT5Sz3tttuo0ePHjz44IN8/PHH3HrrrQWuryCWIDzWUZ8xJUdUVNTBz8888wwXXnghY8eOZf369XnW+0dERBz8HBoaSmZmZqGmyaO/0CNMnDiR5ORkzjjjDABSU1OJjIzk8ssv9zu9qvq9tDQsLOxgA7eqHtYY77vd06dPZ8qUKfzyyy9ERkbSsWNH0tPT81xuvXr1qFmzJlOnTmXevHl8/nleHWUXnl3F5LGO+owpmZKTk6lTpw4AQ4cOLfLln3baaaxdu5b169cDMGrUKL/TjRgxgg8//JD169ezfv161q1bx6RJk0hNTaVTp04MGjQIcA3Me/bsoVOnTnz55ZckJbnq65wqpgYNGrBgwQIAxo0bl2eJKDk5mapVqxIZGcnKlSuZO3cuAOeccw4zZsxg3bp1hy0X4I477uDGG2/k2muvLZJGbksQHuuoz5iS6bHHHuPJJ5/k3HPPJSsrq8iXX6FCBd577z26dOlChw4dqFmzJpUrVz5smtTUVH788cfDSgtRUVF06NCB7777jv/9739MmzaNM844gzZt2rBs2TKaN2/O008/zQUXXECrVq146KGHAOjXrx8zZsygbdu2zJs377BSg68uXbqQmZlJy5YteeaZZ2jXrh0AsbGxDB48mCuvvJJWrVpx3XXXHZzniiuuICUlpUiqlwCksMWrE0F8fLwmJCQc07wN/9eQDvU7MKzXsCKOypiSa8WKFZx++unBDiPoUlJSiI6ORlW55557aNy4Mf379w92WEctISGB/v37M2vWLL/j/f29RWSBqvq9XthKEB7rqM+YsmvIkCHExcXRvHlzkpOT+cc//hHskI7agAEDuOqqq3j11VeLbJnWSA0cyDrA3gN7rZHamDKqf//+J2SJwdcTTzzBE088UaTLtBIEdhe1Mcb4E9AEISL9RWSZiCwVkREiUl5EqonIZBFZ7b1XzWPeLiKySkTWiEjRpsVcrKM+Y4w5UsAShIjUAe4H4lW1BRAK9AaeAH5S1cbAT9733POGAu8ClwHNgOtFpFmgYrWO+owx5kiBrmIKAyqISBgQCWwGegCfeuM/BXr6ma8tsEZV16rqAWCkN19AWBWTMcYcKWCN1Kr6t4i8DmwA0oBJqjpJRGqq6hZvmi0iUsPP7HWAjT7fNwFn+1uPiNwJ3AlQv379Y4rVOuozJjiSkpLo1KkTAFu3biU0NJTY2FjA9UtUrly5fOefPn065cqVy7dL7x49erB9+3Z++eWXogu8jAhYgvDaFnoADYHdwFcicmNhZ/czzO8NG6o6GBgM7j6Io4/0UBuEVTEZU7wK6u67INOnTyc6OjrPBLF7925+++03oqOjWbdu3VE9j/lo5Nct+YkskFvUGVinqokAIjIGaA9sE5FaXumhFrDdz7ybgHo+3+viqqcCIik1iajwKMqHlQ/UKowp8R58ELxjdZGJi4O33jq6eRYsWMBDDz1ESkoK1atXZ+jQodSqVYuBAwfy/vvvExYWRrNmzRgwYADvv/8+oaGhDB8+nLfffpvzzjvvsGV9/fXXdO/enZo1azJy5EiefPJJANasWcNdd91FYmIioaGhfPXVVzRq1IjXXnuNYcOGERISwmWXXcaAAQPo2LEjr7/+OvHx8ezYsYP4+HjWr1/P0KFDGT9+POnp6ezbt49vv/2WHj16sGvXLjIyMnjppZfo0cPVjOfu9vu9996jZcuW/PHHH4SHh7Nnzx5atmzJ6tWrj+iBNpgCmSA2AO1EJBJXxdQJSAD2AbcAA7z3cX7mnQ80FpGGwN+4xu0bAhWoddRnTMmgqtx3332MGzeO2NhYRo0axdNPP83HH3/MgAEDWLduHREREezevZsqVapw11135VvqGDFiBM899xw1a9bk6quvPpgg+vTpwxNPPEGvXr1IT08nOzubH374gW+++YZ58+YRGRlZqO65f/nlFxYvXky1atXIzMxk7NixVKpUiR07dtCuXTuuuOIKli9ffkS33xUrVqRjx46MHz+enj17MnLkSK666qoSlRwgsG0Q80RkNPAbkAksxFUFRQNfisjtuCRyDYCI1AY+VNWuqpopIvcCP+KufvpYVZcFKlbrqM+Yoz/TD4T9+/ezdOlSLr74YsB1fFerVi0AWrZsSZ8+fejZsyc9e/YscFnbtm1jzZo1dOjQAREhLCyMpUuXcvLJJ/P333/Tq1cvAMqXdzUHU6ZM4dZbbyUyMhIoXPfcF1988cHpVJWnnnqKmTNnEhISwt9//822bduYOnWq326/77jjDl577TV69uzJJ598wpAhQ45iTxWPgFaaqepzwHO5Bu/HlSZyT7sZ6OrzfQIwIZDx5bCO+owpGVSV5s2b+21QHj9+PDNnzuTbb7/lxRdfZNmy/M8ZR40axa5duw62O+zZs4eRI0fy2GOP5bnugrrnTk9PP2ycb0d7n3/+OYmJiSxYsIDw8HAaNGiQb/fc5557LuvXr2fGjBlkZWXRokWLfLcnGOxOalwjtVUxGRN8ERERJCYmHkwQGRkZLFu2jOzsbDZu3MiFF17Ia6+9xu7du0lJSaFixYrs3bvX77JGjBjBxIkTD3bPvWDBAkaOHEmlSpWoW7cu33zzDeBKLampqVxyySV8/PHHpKamAv675x49enSesScnJ1OjRg3Cw8OZNm0af/31F0Ce3X4D3HzzzVx//fVF1vtqUbMEgVUxGVNShISEMHr0aB5//HFatWpFXFwcc+bMISsrixtvvJEzzjiDM888k/79+1OlShW6d+/O2LFjiYuLO6wH0/Xr17Nhw4aDXWQDNGzYkEqVKjFv3jyGDRvGwIEDadmyJe3bt2fr1q106dKFK664gvj4eOLi4nj99dcBeOSRRw4+Y3rHjh15xt6nTx8SEhKIj4/n888/57TTTgPIs9vvnHl27dqV72NOg6nMd/etqtw09iYubXQpN7W6KUCRGVMyWXffwTV69GjGjRvHsGHF85iBo+3uu/RduHuURIThVw4PdhjGmDLmvvvu44cffmDChGJpaj0mZT5BGGNMMLz99tvBDqFA1gZhTBlXmqqZTd6O5e9sCcKYMqx8+fIkJSVZkijlVJWkpKSD93wUllUxGVOG1a1bl02bNpGYmBjsUEyAlS9fnrp16x7VPJYgjCnDwsPDA9aBnTnxWRWTMcYYvyxBGGOM8csShDHGGL9K1Z3UIpII/FXAZNWBvO+XL71su8sW2+6y5Xi2+2RVjfU3olQliMIQkYS8bisvzWy7yxbb7rIlUNttVUzGGGP8sgRhjDHGr7KYIAYHO4Agse0uW2y7y5aAbHeZa4MwxhhTOGWxBGGMMaYQLEEYY4zxq8wkCBHpIiKrRGSNiDwR7HgCRUQ+FpHtIrLUZ1g1EZksIqu996rBjDEQRKSeiEwTkRUiskxEHvCGl+ptF5HyIvKriPzubfe/veGlertziEioiCwUke+972Vlu9eLyBIRWSQiCd6wIt/2MpEgRCQUeBe4DGgGXC8izYIbVcAMBbrkGvYE8JOqNgZ+8r6XNpnAw6p6OtAOuMf7G5f2bd8PXKSqrYA4oIuItKP0b3eOB4AVPt/LynYDXKiqcT73PxT5tpeJBAG0Bdao6lpVPQCMBHoEOaaAUNWZwM5cg3sAn3qfPwV6FmdMxUFVt6jqb97nvbiDRh1K+bark+J9DfdeSinfbgARqQtcDnzoM7jUb3c+inzby0qCqANs9Pm+yRtWVtRU1S3gDqRAjSDHE1Ai0gA4E5hHGdh2r5plEbAdmKyqZWK7gbeAx4Bsn2FlYbvBnQRMEpEFInKnN6zIt72sPA9C/Ayz63tLIRGJBr4GHlTVPSL+/vSli6pmAXEiUgUYKyItghxSwIlIN2C7qi4QkY5BDicYzlXVzSJSA5gsIisDsZKyUoLYBNTz+V4X2BykWIJhm4jUAvDetwc5noAQkXBccvhcVcd4g8vEtgOo6m5gOq4NqrRv97nAFSKyHldlfJGIDKf0bzcAqrrZe98OjMVVoxf5tpeVBDEfaCwiDUWkHNAb+DbIMRWnb4FbvM+3AOOCGEtAiCsqfASsUNX/8xlVqrddRGK9kgMiUgHoDKyklG+3qj6pqnVVtQHu/3mqqt5IKd9uABGJEpGKOZ+BS4ClBGDby8yd1CLSFVdnGQp8rKovBzeiwBCREUBHXPe/24DngG+AL4H6wAbgGlXN3ZB9QhORDsAsYAmH6qSfwrVDlNptF5GWuAbJUNwJ35eq+oKIxFCKt9uXV8X0iKp2KwvbLSKn4EoN4JoJvlDVlwOx7WUmQRhjjDk6ZaWKyRhjzFGyBGGMMcYvSxDGGGP8sgRhjDHGL0sQxhhj/LIEYUwBRCTL6zUz51VkHcCJSAPfnneNKUnKSlcbxhyPNFWNC3YQxhQ3K0EYc4y8Pvn/4z2P4VcROdUbfrKI/CQii733+t7wmiIy1nt2w+8i0t5bVKiIDPGe5zDJuyMaEblfRJZ7yxkZpM00ZZglCGMKViFXFdN1PuP2qGpb4B3cnfp4nz9T1ZbA58BAb/hAYIb37IbWwDJveGPgXVVtDuwGrvKGPwGc6S3nrsBsmjF5szupjSmAiKSoarSf4etxD+tZ63UUuFVVY0RkB1BLVTO84VtUtbqIJAJ1VXW/zzIa4Lrobux9fxwIV9WXRGQikILrKuUbn+c+GFMsrARhzPHRPD7nNY0/+30+Z3GobfBy3JMQ2wALRMTaDE2xsgRhzPG5zuf9F+/zHFwPowB9gJ+9zz8Bd8PBh/xUymuhIhIC1FPVabiH4lQBjijFGBNIdkZiTMEqeE9syzFRVXMudY0QkXm4k63rvWH3Ax+LyKNAInCrN/wBYLCI3I4rKdwNbMljnaHAcBGpjHvg1Zve8x6MKTbWBmHMMfLaIOJVdUewYzEmEKyKyRhjjF9WgjDGGOOXlSCMMcb4ZQnCGGOMX5YgjDHG+GUJwhhjjF+WIIwxxvj1/x54l11kFYZOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plotting the accuracy graph of training and test data after adding dropout\n",
    "epochs = range(1,51)\n",
    "plt.plot(epochs, relu_dropout_training_accuracy_per_epoch, 'g', label='Training Accuracy')\n",
    "plt.plot(epochs, relu_dropout_test_accuracy_per_epoch, 'b', label='Test Accuracy')\n",
    "plt.title('Training and Test accuracy after dropout')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Increasing droput on 0.5 to the 2nd fully connected layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a neural network class which extends the nn.Module base class\n",
    "\n",
    "#Created a network with ReLU activation function with network having dropout\n",
    "class Network_relu_dropout2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight) #Initializing the weights to the 1st convolutional layer\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight) #Initializing the weights to the 2nd convolutional layer\n",
    "\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight) #Initializing the weights to the 1st linear layer\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight) #Initializing the weights to the 2nd linear layer\n",
    "        self.dropout = nn.Dropout(p = 0.5) #Initializing dropout of 0.5\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight) #Initializing the weights to the output layer\n",
    "\n",
    "    \n",
    "    #Mapping input tensor to the outpur tensor\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        #t = self.dropout(t) \n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = self.dropout(t) #Adding dropout\n",
    "        t = F.relu(t)   #relu activation        \n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)\n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our network class\n",
    "network_relu_dropout2 = Network_relu_dropout2().to(device) #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu with dropout\n",
    "\n",
    "relu_dropout2_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu_dropout2_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu_dropout2_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu_dropout2_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network with dropout\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    #Training on training loader\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_dropout2, learning_rate =0.1)\n",
    "    #Storing accuracy and loss after every epoch on training data\n",
    "    relu_dropout2_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu_dropout2_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    #Testing on test loader\n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_dropout2)\n",
    "    #Storing accuracy and loss after every epoch on test data\n",
    "    relu_dropout2_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu_dropout2_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Training Accuracy Obtained after increasing dropout: 98.18666666666667\n",
      "Final Test Accuracy Obtained after increasing dropout: 90.6\n"
     ]
    }
   ],
   "source": [
    "#Final train and test accuracy obtained after dropout = 0.5\n",
    "print(\"Final Training Accuracy Obtained after increasing dropout: {}\".format(relu_dropout2_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Test Accuracy Obtained after increasing dropout: {}\".format(relu_dropout2_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decreasing droput to 0.2 on the 2nd fully connected layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a neural network class which extends the nn.Module base class\n",
    "\n",
    "#Created a network with ReLU activation function with network having dropout\n",
    "class Network_relu_dropout3(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        #Defining the neural network layers as class attributes\n",
    "        #Our Network has five layers comprising of two convolutional layers, 2 linear layers and 1 output layer.\n",
    "        \n",
    "        # in_channels = No of colour channels of the input image\n",
    "        # out_channels = No of output channels \n",
    "        # kernel_size = Height and width of the filter\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv1.weight) #Initializing the weights to the 1st convolutional layer\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size = 5)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight) #Initializing the weights to the 2nd convolutional layer\n",
    "\n",
    "        # in_features = length of the flattened output from the previous layer\n",
    "        # out_features = Size of the output tensor\n",
    "        self.fc1 = nn.Linear(in_features = 64*4*4, out_features = 1024)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight) #Initializing the weights to the 1st linear layer\n",
    "        self.dropout = nn.Dropout(p = 0.2) #Initializing dropout of 0.2\n",
    "        \n",
    "        self.fc2 = nn.Linear(in_features = 1024, out_features = 256)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight) #Initializing the weights to the 2nd linear layer\n",
    "        \n",
    "        self.out = nn.Linear(in_features = 256, out_features = 10)\n",
    "        nn.init.xavier_uniform_(self.out.weight) #Initializing the weights to the output layer\n",
    "\n",
    "    \n",
    "    #Mapping input tensor to the outpur tensor\n",
    "    def forward(self, t):\n",
    "\n",
    "        # 1st Convolutional Layer\n",
    "        t = self.conv1(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 2nd Convolutional Layer\n",
    "        t = self.conv2(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = F.max_pool2d(t, kernel_size = 2, stride = 2)  #Max pooling\n",
    "\n",
    "        # 1st Linear Layer\n",
    "        t = t.reshape(-1, 64*4*4)\n",
    "        t = self.fc1(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "        t = self.dropout(t) #Adding dropout\n",
    "\n",
    "        # 2nd Linear Layer\n",
    "        t = self.fc2(t)\n",
    "        t = F.relu(t)   #relu activation\n",
    "#         t = self.dropout(t)\n",
    "\n",
    "        # Output Layer\n",
    "        t = self.out(t)\n",
    "\n",
    "        return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating an instance of our network class\n",
    "network_relu_dropout3 = Network_relu_dropout3().to(device) #Passing the network model to the GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating arrays which will store the accuracy and loss for training and test data for relu with dropout\n",
    "\n",
    "relu_dropout3_training_accuracy_per_epoch = [] #Accuracy on the training dataset after each epoch\n",
    "relu_dropout3_training_loss_per_epoch = [] #Loss on the training dataset after each epoch\n",
    "relu_dropout3_test_accuracy_per_epoch = [] #Accuracy on the test dataset after each epoch\n",
    "relu_dropout3_test_loss_per_epoch = [] #Loss on the Test dataset after each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training the netwrok and testing it's efficiency on test data after several epochs for the relu network with dropout\n",
    "\n",
    "#Training with learning rate = 0.1\n",
    "epochs = 50\n",
    "for i in range(1,epochs+1):\n",
    "    #Training on training loader\n",
    "    training_accuracy_per_epoch, train_loss_per_epoch= train_set_accuracy(training_loader, network_relu_dropout2, learning_rate =0.1)\n",
    "    #Storing accuracy and loss after every epoch on training data\n",
    "    relu_dropout3_training_accuracy_per_epoch.append(training_accuracy_per_epoch)\n",
    "    relu_dropout3_training_loss_per_epoch.append(train_loss_per_epoch)\n",
    "    \n",
    "    #Testing on test loader\n",
    "    test_accuracy_per_epoch, test_loss_per_epoch= test_set_accuracy(test_loader, network_relu_dropout2)\n",
    "    #Storing accuracy and loss after every epoch on test data\n",
    "    relu_dropout3_test_accuracy_per_epoch.append(test_accuracy_per_epoch)\n",
    "    relu_dropout3_test_loss_per_epoch.append(test_loss_per_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Training Accuracy Obtained after decreasing dropout: 99.175\n",
      "Final Test Accuracy Obtained after decreasing dropout: 90.85\n"
     ]
    }
   ],
   "source": [
    "#Final train and test accuracy obtained after dropout = 0.2\n",
    "print(\"Final Training Accuracy Obtained after decreasing dropout: {}\".format(relu_dropout3_training_accuracy_per_epoch[-1]))\n",
    "print(\"Final Test Accuracy Obtained after decreasing dropout: {}\".format(relu_dropout3_test_accuracy_per_epoch[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}