10.6.2 - Advanced CNN (ResNet_Cifar10).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Lab 10.6.2: Advanced CNN (Resnet with CIFAR10)\r\n",
    "\r\n",
    "Edited By Steve Ive\r\n",
    "\r\n",
    "Reference from\r\n",
    "\r\n",
    "https://github.com/deeplearningzerotoall/PyTorch/blob/master/lab-10_6_2_Advance-CNN(ResNet_cifar10).ipynb"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## imports"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "import torch\r\n",
    "import torch.nn as nn\r\n",
    "import torch.nn.functional as F\r\n",
    "import torch.optim as optim\r\n",
    "\r\n",
    "import torchvision\r\n",
    "import torchvision.datasets as datasets\r\n",
    "import torchvision.transforms as transforms\r\n",
    "\r\n",
    "import visdom"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\r\n",
    "\r\n",
    "torch.manual_seed(1)\r\n",
    "\r\n",
    "if device == 'cuda':\r\n",
    "    torch.cuda.manual_seed_all(1)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "vis = visdom.Visdom()\r\n",
    "vis.close(env = \"main\")"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Setting up a new session...\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Define Value Tracker"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "def value_tracker(value_plot, value, index):\r\n",
    "    '''num, loss_value are Tensor'''\r\n",
    "    vis.line(X = index, Y = value, win = value_plot, update = 'append')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\r\n",
    "\r\n",
    "### How to Calculate mean and std in Normalize"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "transform = transforms.Compose([\r\n",
    "    transforms.ToTensor()\r\n",
    "])\r\n",
    "\r\n",
    "trainset = datasets.CIFAR10(root = 'cifar10/', download = True, transform = transform, train = True)\r\n",
    "\r\n",
    "print(trainset.data.shape)\r\n",
    "\r\n",
    "train_data_mean = trainset.data.mean(axis = (0, 1, 2))\r\n",
    "train_data_std = trainset.data.std(axis = (0, 1, 2))\r\n",
    "\r\n",
    "print(train_data_mean)\r\n",
    "print(train_data_std)\r\n",
    "\r\n",
    "train_data_mean = train_data_mean / 255\r\n",
    "train_data_std = train_data_std / 255\r\n",
    "\r\n",
    "print(train_data_mean)\r\n",
    "print(train_data_std)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Files already downloaded and verified\n",
      "(50000, 32, 32, 3)\n",
      "[125.30691805 122.95039414 113.86538318]\n",
      "[62.99321928 62.08870764 66.70489964]\n",
      "[0.49139968 0.48215841 0.44653091]\n",
      "[0.24703223 0.24348513 0.26158784]\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "transform_train = transforms.Compose([\r\n",
    "    transforms.ToTensor(),\r\n",
    "    transforms.Normalize(train_data_mean, train_data_std)\r\n",
    "])\r\n",
    "\r\n",
    "transform_test = transforms.Compose([\r\n",
    "    transforms.ToTensor(),\r\n",
    "    transforms.Normalize(train_data_mean, train_data_std)\r\n",
    "])\r\n",
    "\r\n",
    "trainset = datasets.CIFAR10(download = True, root = 'cifar10/', transform = transform_train, train = True)\r\n",
    "testset = datasets.CIFAR10(download = True, root = 'cifar10/', transform = transform_test, train = False)\r\n",
    "\r\n",
    "train_loader = torch.utils.data.DataLoader(dataset = trainset, batch_size = 256, shuffle = True, num_workers = 0)\r\n",
    "test_loader = torch.utils.data.DataLoader(dataset = testset, batch_size = 256, shuffle = True, num_workers = 0)\r\n",
    "\r\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horese', 'ship', 'truck')"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Reform the ResNet since the dataset resolution is too small"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "import torchvision.models.resnet as resnet\r\n",
    "from torch import Tensor\r\n",
    "from typing import Type, Any, Callable, Union, List, Optional\r\n",
    "BasicBlock = resnet.BasicBlock\r\n",
    "Bottleneck = resnet.Bottleneck\r\n",
    "conv1x1 = resnet.conv1x1"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "class ResNet(nn.Module):\r\n",
    "    \r\n",
    "    def __init__(\r\n",
    "        self,\r\n",
    "        block: Type[Union[BasicBlock, Bottleneck]],\r\n",
    "        layers: List[int],\r\n",
    "        num_classes: int = 1000,\r\n",
    "        zero_init_residual: bool = False,\r\n",
    "        groups: int = 1,\r\n",
    "        width_per_group: int = 64,\r\n",
    "        replace_stride_with_dilation: Optional[List[bool]] = None,\r\n",
    "        norm_layer: Optional[Callable[..., nn.Module]] = None\r\n",
    "    ) -> None:\r\n",
    "        super(ResNet, self).__init__()\r\n",
    "        if norm_layer is None:\r\n",
    "            norm_layer = nn.BatchNorm2d\r\n",
    "        self._norm_layer = norm_layer\r\n",
    "\r\n",
    "        self.inplanes = 64\r\n",
    "        self.dilation = 1\r\n",
    "        if replace_stride_with_dilation is None:\r\n",
    "            # each element in the tuple indicates if we should replace\r\n",
    "            # the 2x2 stride with a dilated convolution instead\r\n",
    "            replace_stride_with_dilation = [False, False, False]\r\n",
    "        if len(replace_stride_with_dilation) != 3:\r\n",
    "            raise ValueError(\"replace_stride_with_dilation should be None \"\r\n",
    "                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\r\n",
    "        self.groups = groups\r\n",
    "        self.base_width = width_per_group\r\n",
    "        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,\r\n",
    "                               bias=False)\r\n",
    "        self.bn1 = norm_layer(self.inplanes)\r\n",
    "        self.relu = nn.ReLU(inplace=True)\r\n",
    "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\r\n",
    "        self.layer1 = self._make_layer(block, 64, layers[0])\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "        #change strides from 2 to 1 below -----------------------------\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "        self.layer2 = self._make_layer(block, 128, layers[1], stride=1,\r\n",
    "                                       dilate=replace_stride_with_dilation[0])\r\n",
    "        self.layer3 = self._make_layer(block, 256, layers[2], stride=1,\r\n",
    "                                       dilate=replace_stride_with_dilation[1])\r\n",
    "        self.layer4 = self._make_layer(block, 512, layers[3], stride=1,\r\n",
    "                                       dilate=replace_stride_with_dilation[2])\r\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "        \r\n",
    "        self.fc = nn.Linear(512 * block.expansion, num_classes)\r\n",
    "\r\n",
    "        for m in self.modules():\r\n",
    "            if isinstance(m, nn.Conv2d):\r\n",
    "                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\r\n",
    "            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\r\n",
    "                nn.init.constant_(m.weight, 1)\r\n",
    "                nn.init.constant_(m.bias, 0)\r\n",
    "\r\n",
    "        # Zero-initialize the last BN in each residual branch,\r\n",
    "        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\r\n",
    "        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\r\n",
    "        if zero_init_residual:\r\n",
    "            for m in self.modules():\r\n",
    "                if isinstance(m, Bottleneck):\r\n",
    "                    nn.init.constant_(m.bn3.weight, 0)  # type: ignore[arg-type]\r\n",
    "                elif isinstance(m, BasicBlock):\r\n",
    "                    nn.init.constant_(m.bn2.weight, 0)  # type: ignore[arg-type]\r\n",
    "\r\n",
    "    def _make_layer(self, block: Type[Union[BasicBlock, Bottleneck]], planes: int, blocks: int,\r\n",
    "                    stride: int = 1, dilate: bool = False) -> nn.Sequential:\r\n",
    "        norm_layer = self._norm_layer\r\n",
    "        downsample = None\r\n",
    "        previous_dilation = self.dilation\r\n",
    "        if dilate:\r\n",
    "            self.dilation *= stride\r\n",
    "            stride = 1\r\n",
    "        if stride != 1 or self.inplanes != planes * block.expansion:\r\n",
    "            downsample = nn.Sequential(\r\n",
    "                conv1x1(self.inplanes, planes * block.expansion, stride),\r\n",
    "                norm_layer(planes * block.expansion),\r\n",
    "            )\r\n",
    "\r\n",
    "        layers = []\r\n",
    "        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\r\n",
    "                            self.base_width, previous_dilation, norm_layer))\r\n",
    "        self.inplanes = planes * block.expansion\r\n",
    "        for _ in range(1, blocks):\r\n",
    "            layers.append(block(self.inplanes, planes, groups=self.groups,\r\n",
    "                                base_width=self.base_width, dilation=self.dilation,\r\n",
    "                                norm_layer=norm_layer))\r\n",
    "\r\n",
    "        return nn.Sequential(*layers)\r\n",
    "\r\n",
    "    def _forward_impl(self, x: Tensor) -> Tensor:\r\n",
    "        # See note [TorchScript super()]\r\n",
    "        x = self.conv1(x)\r\n",
    "        x = self.bn1(x)\r\n",
    "        x = self.relu(x)\r\n",
    "        x = self.maxpool(x)\r\n",
    "\r\n",
    "        x = self.layer1(x)\r\n",
    "        x = self.layer2(x)\r\n",
    "        x = self.layer3(x)\r\n",
    "        x = self.layer4(x)\r\n",
    "\r\n",
    "        x = self.avgpool(x)\r\n",
    "        x = torch.flatten(x, 1)\r\n",
    "        x = self.fc(x)\r\n",
    "\r\n",
    "        return x\r\n",
    "\r\n",
    "    def forward(self, x: Tensor) -> Tensor:\r\n",
    "        return self._forward_impl(x)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "resnet50 = ResNet(resnet.Bottleneck, [3, 4, 6, 3], 10, True).to(device)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Take a Moment!!!!!!!!!!!!!!!!!\r\n",
    "\r\n",
    "if not normalize the Tensor just like ```torch.Tensor(1, 3, 32, 32).to(device)```\r\n",
    "\r\n",
    "it returns nan to all of classes\r\n",
    "\r\n",
    "**It is important to normalize our data**"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "test = torch.rand(1, 3, 32, 32).to(device)\r\n",
    "out = resnet50(test)\r\n",
    "print(out)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\buddhalight\\envs\\buddhalight\\lib\\site-packages\\torch\\nn\\functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  ..\\c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[ 0.2347, -0.2441,  0.2715,  0.1799,  0.0963, -0.1991,  0.0620, -0.3722,\n",
      "          0.0864,  0.4435]], device='cuda:0', grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "optimizer = optim.SGD(resnet50.parameters(), lr = 0.1, momentum =0.9, weight_decay = 5e-4)\r\n",
    "lr_sche = optim.lr_scheduler.StepLR(optimizer, step_size =10, gamma = 0.5)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Make Plot"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "loss_plt = vis.line(Y=torch.Tensor(1).zero_(), opts = dict(title = \"loss_tracker\", legend = ['loss'], showlegend = True))\r\n",
    "acc_plt = vis.line(Y = torch.Tensor(1).zero_(), opts = dict(title = 'Accuracy', legend = ['Acc'], showlegend = True))"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Define Acc_check Function"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "def acc_check(net, test_loader, epoch, save = 1):\r\n",
    "    correct = 0\r\n",
    "    total = 0\r\n",
    "    with torch.no_grad():\r\n",
    "        for X, Y in test_loader:\r\n",
    "            X = X.to(device)\r\n",
    "            Y = Y.to(device)\r\n",
    "\r\n",
    "            #prediction\r\n",
    "            pred = net(X)\r\n",
    "\r\n",
    "            _, predicted = torch.max(pred, 1)\r\n",
    "\r\n",
    "            total += Y.size(0)\r\n",
    "            correct += (predicted == Y).sum().item()\r\n",
    "\r\n",
    "    acc = (100 * correct / total)\r\n",
    "    print('Accuracy of the network on the 10000 test images: {}'.format(acc))\r\n",
    "\r\n",
    "    if epoch % 10 == 0:\r\n",
    "        if save:\r\n",
    "            torch.save(net.state_dict(), \"./models/resnet_cifar10/model_epoch_{}_acc_{}.pth\".format(epoch, acc))\r\n",
    "\r\n",
    "    return acc"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Training with (acc check + model save)"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "print(len(train_loader))\r\n",
    "epochs = 20\r\n",
    "\r\n",
    "for epoch in range(epochs):\r\n",
    "\r\n",
    "    running_loss = 0.0\r\n",
    "    \r\n",
    "    for i, data in enumerate(train_loader):\r\n",
    "        #get the inputs\r\n",
    "        X, Y = data\r\n",
    "        X = X.to(device)\r\n",
    "        Y = Y.to(device)\r\n",
    "\r\n",
    "        # zero the parameter gradients\r\n",
    "        optimizer.zero_grad()\r\n",
    "\r\n",
    "        # forward + backward + optimize\r\n",
    "        pred = resnet50(X)\r\n",
    "        \r\n",
    "        #cost\r\n",
    "        cost = F.cross_entropy(pred, Y).to(device)\r\n",
    "        \r\n",
    "        #reduce the cost\r\n",
    "        cost.backward()\r\n",
    "        optimizer.step()\r\n",
    "        lr_sche.step()\r\n",
    "\r\n",
    "        #print statistics\r\n",
    "        running_loss += cost.item()\r\n",
    "        if i % 30 == 29:\r\n",
    "            print('Epoch: {} / {}, MiniBatch: {} / {}, Cost: {}'.format(epoch + 1, 20, i, len(train_loader), cost.item()))\r\n",
    "            value_tracker(loss_plt, torch.Tensor([running_loss / 30]), torch.Tensor([i + epoch * len(train_loader)]))\r\n",
    "        running_loss = 0\r\n",
    "\r\n",
    "    #Check accuracy\r\n",
    "    acc = acc_check(resnet50, test_loader, epoch, save = 1)\r\n",
    "    value_tracker(acc_plt, torch.Tensor([acc]), torch.Tensor([epoch]))\r\n",
    "\r\n",
    "\r\n",
    "print('Finshed Learning')\r\n",
    "            "
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "196\n",
      "Epoch: 1 / 20, MiniBatch: 29 / 196, Cost: 1.987879753112793\n",
      "Epoch: 1 / 20, MiniBatch: 59 / 196, Cost: 1.6570311784744263\n",
      "Epoch: 1 / 20, MiniBatch: 89 / 196, Cost: 1.669613003730774\n",
      "Epoch: 1 / 20, MiniBatch: 119 / 196, Cost: 1.70968759059906\n",
      "Epoch: 1 / 20, MiniBatch: 149 / 196, Cost: 1.6382006406784058\n",
      "Epoch: 1 / 20, MiniBatch: 179 / 196, Cost: 1.7769445180892944\n",
      "Accuracy of the network on the 10000 test images: 38.7\n",
      "Epoch: 2 / 20, MiniBatch: 29 / 196, Cost: 1.604959487915039\n",
      "Epoch: 2 / 20, MiniBatch: 59 / 196, Cost: 1.6588995456695557\n",
      "Epoch: 2 / 20, MiniBatch: 89 / 196, Cost: 1.7000317573547363\n",
      "Epoch: 2 / 20, MiniBatch: 119 / 196, Cost: 1.690345048904419\n",
      "Epoch: 2 / 20, MiniBatch: 149 / 196, Cost: 1.5970897674560547\n",
      "Epoch: 2 / 20, MiniBatch: 179 / 196, Cost: 1.5948519706726074\n",
      "Accuracy of the network on the 10000 test images: 38.92\n",
      "Epoch: 3 / 20, MiniBatch: 29 / 196, Cost: 1.669651985168457\n",
      "Epoch: 3 / 20, MiniBatch: 59 / 196, Cost: 1.578109860420227\n",
      "Epoch: 3 / 20, MiniBatch: 89 / 196, Cost: 1.6107028722763062\n",
      "Epoch: 3 / 20, MiniBatch: 119 / 196, Cost: 1.7158043384552002\n",
      "Epoch: 3 / 20, MiniBatch: 149 / 196, Cost: 1.6672979593276978\n",
      "Epoch: 3 / 20, MiniBatch: 179 / 196, Cost: 1.6427626609802246\n",
      "Accuracy of the network on the 10000 test images: 39.14\n",
      "Epoch: 4 / 20, MiniBatch: 29 / 196, Cost: 1.6343843936920166\n",
      "Epoch: 4 / 20, MiniBatch: 59 / 196, Cost: 1.5835413932800293\n",
      "Epoch: 4 / 20, MiniBatch: 89 / 196, Cost: 1.6438624858856201\n",
      "Epoch: 4 / 20, MiniBatch: 119 / 196, Cost: 1.683458685874939\n",
      "Epoch: 4 / 20, MiniBatch: 149 / 196, Cost: 1.724977731704712\n",
      "Epoch: 4 / 20, MiniBatch: 179 / 196, Cost: 1.6418060064315796\n",
      "Accuracy of the network on the 10000 test images: 39.05\n",
      "Epoch: 5 / 20, MiniBatch: 29 / 196, Cost: 1.642008662223816\n",
      "Epoch: 5 / 20, MiniBatch: 59 / 196, Cost: 1.7132502794265747\n",
      "Epoch: 5 / 20, MiniBatch: 89 / 196, Cost: 1.5460219383239746\n",
      "Epoch: 5 / 20, MiniBatch: 119 / 196, Cost: 1.682508111000061\n",
      "Epoch: 5 / 20, MiniBatch: 149 / 196, Cost: 1.6854939460754395\n",
      "Epoch: 5 / 20, MiniBatch: 179 / 196, Cost: 1.7777493000030518\n",
      "Accuracy of the network on the 10000 test images: 39.03\n",
      "Epoch: 6 / 20, MiniBatch: 29 / 196, Cost: 1.6886317729949951\n",
      "Epoch: 6 / 20, MiniBatch: 59 / 196, Cost: 1.680353045463562\n",
      "Epoch: 6 / 20, MiniBatch: 89 / 196, Cost: 1.585144281387329\n",
      "Epoch: 6 / 20, MiniBatch: 119 / 196, Cost: 1.7454557418823242\n",
      "Epoch: 6 / 20, MiniBatch: 149 / 196, Cost: 1.7004505395889282\n",
      "Epoch: 6 / 20, MiniBatch: 179 / 196, Cost: 1.6414577960968018\n",
      "Accuracy of the network on the 10000 test images: 39.06\n",
      "Epoch: 7 / 20, MiniBatch: 29 / 196, Cost: 1.580033540725708\n",
      "Epoch: 7 / 20, MiniBatch: 59 / 196, Cost: 1.627366542816162\n",
      "Epoch: 7 / 20, MiniBatch: 89 / 196, Cost: 1.6823846101760864\n",
      "Epoch: 7 / 20, MiniBatch: 119 / 196, Cost: 1.6452397108078003\n",
      "Epoch: 7 / 20, MiniBatch: 149 / 196, Cost: 1.7099201679229736\n",
      "Epoch: 7 / 20, MiniBatch: 179 / 196, Cost: 1.769403100013733\n",
      "Accuracy of the network on the 10000 test images: 39.13\n",
      "Epoch: 8 / 20, MiniBatch: 29 / 196, Cost: 1.7152318954467773\n",
      "Epoch: 8 / 20, MiniBatch: 59 / 196, Cost: 1.6473793983459473\n",
      "Epoch: 8 / 20, MiniBatch: 89 / 196, Cost: 1.6238384246826172\n",
      "Epoch: 8 / 20, MiniBatch: 119 / 196, Cost: 1.6833553314208984\n",
      "Epoch: 8 / 20, MiniBatch: 149 / 196, Cost: 1.751983642578125\n",
      "Epoch: 8 / 20, MiniBatch: 179 / 196, Cost: 1.64181649684906\n",
      "Accuracy of the network on the 10000 test images: 39.01\n",
      "Epoch: 9 / 20, MiniBatch: 29 / 196, Cost: 1.7322347164154053\n",
      "Epoch: 9 / 20, MiniBatch: 59 / 196, Cost: 1.6911438703536987\n",
      "Epoch: 9 / 20, MiniBatch: 89 / 196, Cost: 1.668939232826233\n",
      "Epoch: 9 / 20, MiniBatch: 119 / 196, Cost: 1.7342000007629395\n",
      "Epoch: 9 / 20, MiniBatch: 149 / 196, Cost: 1.6421308517456055\n",
      "Epoch: 9 / 20, MiniBatch: 179 / 196, Cost: 1.7873035669326782\n",
      "Accuracy of the network on the 10000 test images: 38.88\n",
      "Epoch: 10 / 20, MiniBatch: 29 / 196, Cost: 1.6934001445770264\n",
      "Epoch: 10 / 20, MiniBatch: 59 / 196, Cost: 1.668694257736206\n",
      "Epoch: 10 / 20, MiniBatch: 89 / 196, Cost: 1.6607221364974976\n",
      "Epoch: 10 / 20, MiniBatch: 119 / 196, Cost: 1.6296635866165161\n",
      "Epoch: 10 / 20, MiniBatch: 149 / 196, Cost: 1.706101417541504\n",
      "Epoch: 10 / 20, MiniBatch: 179 / 196, Cost: 1.6491038799285889\n",
      "Accuracy of the network on the 10000 test images: 39.16\n",
      "Epoch: 11 / 20, MiniBatch: 29 / 196, Cost: 1.6102516651153564\n",
      "Epoch: 11 / 20, MiniBatch: 59 / 196, Cost: 1.730950951576233\n",
      "Epoch: 11 / 20, MiniBatch: 89 / 196, Cost: 1.703076958656311\n",
      "Epoch: 11 / 20, MiniBatch: 119 / 196, Cost: 1.657356858253479\n",
      "Epoch: 11 / 20, MiniBatch: 149 / 196, Cost: 1.6531755924224854\n",
      "Epoch: 11 / 20, MiniBatch: 179 / 196, Cost: 1.5814833641052246\n",
      "Accuracy of the network on the 10000 test images: 38.96\n",
      "Epoch: 12 / 20, MiniBatch: 29 / 196, Cost: 1.731263518333435\n",
      "Epoch: 12 / 20, MiniBatch: 59 / 196, Cost: 1.6477997303009033\n",
      "Epoch: 12 / 20, MiniBatch: 89 / 196, Cost: 1.6824933290481567\n",
      "Epoch: 12 / 20, MiniBatch: 119 / 196, Cost: 1.6382399797439575\n",
      "Epoch: 12 / 20, MiniBatch: 149 / 196, Cost: 1.6251916885375977\n",
      "Epoch: 12 / 20, MiniBatch: 179 / 196, Cost: 1.6147524118423462\n",
      "Accuracy of the network on the 10000 test images: 39.16\n",
      "Epoch: 13 / 20, MiniBatch: 29 / 196, Cost: 1.6602740287780762\n",
      "Epoch: 13 / 20, MiniBatch: 59 / 196, Cost: 1.6351022720336914\n",
      "Epoch: 13 / 20, MiniBatch: 89 / 196, Cost: 1.6874055862426758\n",
      "Epoch: 13 / 20, MiniBatch: 119 / 196, Cost: 1.6995447874069214\n",
      "Epoch: 13 / 20, MiniBatch: 149 / 196, Cost: 1.647242546081543\n",
      "Epoch: 13 / 20, MiniBatch: 179 / 196, Cost: 1.6818114519119263\n",
      "Accuracy of the network on the 10000 test images: 38.93\n",
      "Epoch: 14 / 20, MiniBatch: 29 / 196, Cost: 1.717427134513855\n",
      "Epoch: 14 / 20, MiniBatch: 59 / 196, Cost: 1.682494044303894\n",
      "Epoch: 14 / 20, MiniBatch: 89 / 196, Cost: 1.6997138261795044\n",
      "Epoch: 14 / 20, MiniBatch: 119 / 196, Cost: 1.6115663051605225\n",
      "Epoch: 14 / 20, MiniBatch: 149 / 196, Cost: 1.5662286281585693\n",
      "Epoch: 14 / 20, MiniBatch: 179 / 196, Cost: 1.698138952255249\n",
      "Accuracy of the network on the 10000 test images: 38.86\n",
      "Epoch: 15 / 20, MiniBatch: 29 / 196, Cost: 1.6608275175094604\n",
      "Epoch: 15 / 20, MiniBatch: 59 / 196, Cost: 1.6589019298553467\n",
      "Epoch: 15 / 20, MiniBatch: 89 / 196, Cost: 1.7406139373779297\n",
      "Epoch: 15 / 20, MiniBatch: 119 / 196, Cost: 1.6459318399429321\n",
      "Epoch: 15 / 20, MiniBatch: 149 / 196, Cost: 1.7383484840393066\n",
      "Epoch: 15 / 20, MiniBatch: 179 / 196, Cost: 1.6356194019317627\n",
      "Accuracy of the network on the 10000 test images: 38.84\n",
      "Epoch: 16 / 20, MiniBatch: 29 / 196, Cost: 1.741283655166626\n",
      "Epoch: 16 / 20, MiniBatch: 59 / 196, Cost: 1.7021262645721436\n",
      "Epoch: 16 / 20, MiniBatch: 89 / 196, Cost: 1.7051156759262085\n",
      "Epoch: 16 / 20, MiniBatch: 119 / 196, Cost: 1.5408570766448975\n",
      "Epoch: 16 / 20, MiniBatch: 149 / 196, Cost: 1.6500537395477295\n",
      "Epoch: 16 / 20, MiniBatch: 179 / 196, Cost: 1.6973360776901245\n",
      "Accuracy of the network on the 10000 test images: 38.97\n",
      "Epoch: 17 / 20, MiniBatch: 29 / 196, Cost: 1.6869785785675049\n",
      "Epoch: 17 / 20, MiniBatch: 59 / 196, Cost: 1.659433126449585\n",
      "Epoch: 17 / 20, MiniBatch: 89 / 196, Cost: 1.6073474884033203\n",
      "Epoch: 17 / 20, MiniBatch: 119 / 196, Cost: 1.6420623064041138\n",
      "Epoch: 17 / 20, MiniBatch: 149 / 196, Cost: 1.6464067697525024\n",
      "Epoch: 17 / 20, MiniBatch: 179 / 196, Cost: 1.6674926280975342\n",
      "Accuracy of the network on the 10000 test images: 38.97\n",
      "Epoch: 18 / 20, MiniBatch: 29 / 196, Cost: 1.71772038936615\n",
      "Epoch: 18 / 20, MiniBatch: 59 / 196, Cost: 1.6810579299926758\n",
      "Epoch: 18 / 20, MiniBatch: 89 / 196, Cost: 1.6378397941589355\n",
      "Epoch: 18 / 20, MiniBatch: 119 / 196, Cost: 1.6435271501541138\n",
      "Epoch: 18 / 20, MiniBatch: 149 / 196, Cost: 1.635546326637268\n",
      "Epoch: 18 / 20, MiniBatch: 179 / 196, Cost: 1.638332486152649\n",
      "Accuracy of the network on the 10000 test images: 39.04\n",
      "Epoch: 19 / 20, MiniBatch: 29 / 196, Cost: 1.6354948282241821\n",
      "Epoch: 19 / 20, MiniBatch: 59 / 196, Cost: 1.5866830348968506\n",
      "Epoch: 19 / 20, MiniBatch: 89 / 196, Cost: 1.6559324264526367\n",
      "Epoch: 19 / 20, MiniBatch: 119 / 196, Cost: 1.6673970222473145\n",
      "Epoch: 19 / 20, MiniBatch: 149 / 196, Cost: 1.705017328262329\n",
      "Epoch: 19 / 20, MiniBatch: 179 / 196, Cost: 1.6794052124023438\n",
      "Accuracy of the network on the 10000 test images: 38.95\n",
      "Epoch: 20 / 20, MiniBatch: 29 / 196, Cost: 1.6708924770355225\n",
      "Epoch: 20 / 20, MiniBatch: 59 / 196, Cost: 1.7032257318496704\n",
      "Epoch: 20 / 20, MiniBatch: 89 / 196, Cost: 1.633966326713562\n",
      "Epoch: 20 / 20, MiniBatch: 119 / 196, Cost: 1.6499199867248535\n",
      "Epoch: 20 / 20, MiniBatch: 149 / 196, Cost: 1.7093318700790405\n",
      "Epoch: 20 / 20, MiniBatch: 179 / 196, Cost: 1.7326291799545288\n",
      "Accuracy of the network on the 10000 test images: 39.07\n",
      "Finshed Learning\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Model Accuracy Testing"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "source": [
    "correct = 0\r\n",
    "total = 0\r\n",
    "\r\n",
    "with torch.no_grad():\r\n",
    "    for data in test_loader:\r\n",
    "        X, Y = data\r\n",
    "        X = X.to(device)\r\n",
    "        Y = Y.to(device)\r\n",
    "\r\n",
    "        #prediction\r\n",
    "        pred = resnet50(X)\r\n",
    "\r\n",
    "        _, predicted = torch.max(pred, 1)\r\n",
    "\r\n",
    "        total += Y.size(0)\r\n",
    "\r\n",
    "        correct += (predicted == Y).sum().item()\r\n",
    "\r\n",
    "print('Accuracy of the network on the 10000 test images: {}'.format(100 * correct / total))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Accuracy of the network on the 10000 test images: 39.12\n"
     ]
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.8.8",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.8 64-bit ('buddhalight': conda)"
  },
  "interpreter": {
   "hash": "38ed4d61829b01de31b0fe0651719916120d9f7e023a62cbbfea93b7d24a50a0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}