Fully-Convolutional-ResNet-101(FCRN).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "from mpl_toolkits.axes_grid1 import ImageGrid\n",
    "plt.ion()\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "#%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data augmentation and normalization for training \n",
    "# Just normalization for validation\n",
    "\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(224), \n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val':transforms.Compose([\n",
    "        transforms.Resize(320),\n",
    "        transforms.CenterCrop(280),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485,0.456,0.406],[0.229, 0.224, 0.225])\n",
    "    ])    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': Dataset ImageFolder\n",
       "     Number of datapoints: 12454\n",
       "     Root location: /home/mnt/jupyter_dir/imagewoof/train,\n",
       " 'val': Dataset ImageFolder\n",
       "     Number of datapoints: 500\n",
       "     Root location: /home/mnt/jupyter_dir/imagewoof/val}"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#image_dataset Attributes:\n",
    "# classes (list): List of the class names.\n",
    "# class_to_idx (dict): Dict with items (class_name, class_index).\n",
    "# imgs (list): List of (image path, class_index) tuples\n",
    "            \n",
    "#data_dir='/home/mnt/jupyter_dir/hymenoptera_data'\n",
    "data_dir='/home/mnt/jupyter_dir/imagewoof'\n",
    "#data_dir='/home/mnt/jupyter_dir/tiny-imagenet-200'\n",
    "#data_dir='/home/mnt/jupyter_dir/stanford-cars'\n",
    "image_datasets = { x: datasets.ImageFolder(os.path.join(data_dir, x), \n",
    "                        data_transforms[x]) for x in ['train', 'val'] }\n",
    "\n",
    "image_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloaders= {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=16, \n",
    "                                             shuffle=True, num_workers=0)\n",
    "              for x in ['train', 'val'] }\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train','val']}\n",
    "class_names = image_datasets['train'].classes\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 3, 280, 280])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a batch of training data\n",
    "inputs, classes = next(iter(dataloaders['val']))\n",
    "\n",
    "inputs.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot each image using grid axis \n",
    "def imshow(axis, inp):\n",
    "    \"\"\"Denormalize and show\"\"\"\n",
    "    inp = inp.numpy().transpose((1, 2, 0))\n",
    "    mean = np.array([0.485, 0.456, 0.406])\n",
    "    std = np.array([0.229, 0.224, 0.225])\n",
    "    inp = std * inp + mean\n",
    "    axis.imshow((inp * 255).astype(np.uint8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = dataloaders['train']\n",
    "# Get a batch of training data with labels\n",
    "img, label = next(iter(train))\n",
    "print(img.size(), label.size())\n",
    "fig = plt.figure(1, figsize=(16, 4))\n",
    "#make a grid of (2,8)\n",
    "grid = ImageGrid(fig, 111, nrows_ncols=(2, 8), axes_pad=0.05)    \n",
    "for i in range(img.size()[0]):\n",
    "    ax = grid[i]\n",
    "    imshow(ax, img[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, criterion, optimizer, scheduler, num_epochs=25, flag_Load=True):\n",
    "    \n",
    "    since= time.time()\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "    \n",
    "    \n",
    "    \n",
    "    for epoch in range(num_epochs-1):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs-1))\n",
    "        print('-'* 10)\n",
    "        # load the model from disk if flag_Load is True\n",
    "        if flag_Load==True:\n",
    "            print('Loading model from disk')\n",
    "            flag_Load=False\n",
    "            checkpoint = torch.load('/home/mnt/jupyter_dir/saved_model_dog/resnet101_28.pth.tar')\n",
    "            model.load_state_dict(checkpoint['model_state_dict'])\n",
    "            optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "            epoch = checkpoint['epoch']\n",
    "            loss = checkpoint['loss']\n",
    "            best_acc = checkpoint['best_prec1']\n",
    "            \n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train','val']:\n",
    "            \n",
    "            if phase == 'train':\n",
    "                \n",
    "                scheduler.step()\n",
    "                model.train() # set model to training mode\n",
    "            else:\n",
    "                \n",
    "                model.eval() # Set model to evaluate mode\n",
    "            \n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            #Iterate over data\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                \n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                \n",
    "                #zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                #forward\n",
    "                # track history if only in train\n",
    "                \n",
    "                with torch.set_grad_enabled(phase=='train'):\n",
    "                    outputs = model(inputs)\n",
    "                    out = torch.squeeze(outputs, dim=2)\n",
    "                    out2 = torch.squeeze(out, dim=2)\n",
    "                    #print(labels.shape)\n",
    "                    _, preds = torch.max(out2,1)\n",
    "                    loss = criterion(out2, labels)\n",
    "                    \n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                # statistics\n",
    "                running_loss += loss.item()*inputs.size(0)\n",
    "                running_corrects += torch.sum(preds==labels.data)\n",
    "                \n",
    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
    "            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
    "            \n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "            \n",
    "            if phase == 'val' and epoch_acc> best_acc:\n",
    "                #print('input shape for validation', inputs.shape)\n",
    "                best_acc= epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "        \n",
    "            print()\n",
    "            #save number of epochs, model weights, optimization hyper-parameters, loss and best_acc each epoch \n",
    "            torch.save({\n",
    "                'epoch': epoch,\n",
    "                'model_state_dict': model.state_dict(),\n",
    "                'optimizer_state_dict': optimizer.state_dict(),\n",
    "                'loss': loss,\n",
    "                'best_prec1': best_acc}, '/home/mnt/jupyter_dir/saved_model_dog/resnet101_'+str(epoch)+'.pth.tar')\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed//60, time_elapsed % 60))\n",
    "    print('Best val Acc : {:4f}'.format(best_acc))\n",
    "    \n",
    "    #save entire model\n",
    "    torch.save(model,'/home/mnt/jupyter_dir/saved_model_dog/resnet101.pth.tar')\n",
    "    \n",
    "    # load best weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([16, 10, 1, 1])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class ResNet50Bottom(nn.Module):\n",
    "    def __init__(self, original_model):\n",
    "        super(ResNet50Bottom, self).__init__()\n",
    "        self.features = nn.Sequential(*list(original_model.children())[:-2])\n",
    "        self.conv = nn.Conv2d(2048, 10, kernel_size=(7, 7), stride=(1, 1), bias=False)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.conv(x)\n",
    "        return x\n",
    "\n",
    "res50_model = torchvision.models.resnet101(pretrained=True)\n",
    "for param in res50_model.parameters():\n",
    "    param.requires_grad = False\n",
    "res_fcn = ResNet50Bottom(res50_model)\n",
    "inputs, classes = next(iter(dataloaders['train']))\n",
    "res_fcn = res_fcn.to(device)\n",
    "inputs  = inputs.to(device)\n",
    "classes = classes.to(device)\n",
    "outputs = res_fcn(inputs)\n",
    "outputs.data.shape  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer_conv = optim.RMSprop(res_fcn.conv.parameters(), lr=5e-5, momentum=0.9, weight_decay=2e-4)\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=20, gamma=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 510.4882 Acc: 0.8323\n",
      "\n",
      "val Loss: 276.2523 Acc: 0.9220\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 527.1313 Acc: 0.8338\n",
      "\n",
      "val Loss: 388.4131 Acc: 0.9100\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 554.1853 Acc: 0.8376\n",
      "\n",
      "val Loss: 241.1769 Acc: 0.9140\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 526.0953 Acc: 0.8442\n",
      "\n",
      "val Loss: 660.0433 Acc: 0.8880\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 573.8999 Acc: 0.8408\n",
      "\n",
      "val Loss: 269.2233 Acc: 0.9200\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 565.9997 Acc: 0.8393\n",
      "\n",
      "val Loss: 371.0456 Acc: 0.9080\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 425.9030 Acc: 0.8676\n",
      "\n",
      "val Loss: 319.7437 Acc: 0.9260\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 391.7818 Acc: 0.8760\n",
      "\n",
      "val Loss: 239.3853 Acc: 0.9360\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 383.0352 Acc: 0.8716\n",
      "\n",
      "val Loss: 218.2293 Acc: 0.9400\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 363.0457 Acc: 0.8763\n",
      "\n",
      "val Loss: 243.4358 Acc: 0.9360\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 365.0151 Acc: 0.8722\n",
      "\n",
      "val Loss: 220.4575 Acc: 0.9380\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 335.8252 Acc: 0.8785\n",
      "\n",
      "val Loss: 257.7740 Acc: 0.9280\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 337.5035 Acc: 0.8774\n",
      "\n",
      "val Loss: 235.9856 Acc: 0.9360\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 306.8161 Acc: 0.8841\n",
      "\n",
      "val Loss: 230.2557 Acc: 0.9280\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 334.5133 Acc: 0.8797\n",
      "\n",
      "val Loss: 226.3939 Acc: 0.9260\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 327.2673 Acc: 0.8785\n",
      "\n",
      "val Loss: 221.3536 Acc: 0.9320\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 295.3574 Acc: 0.8836\n",
      "\n",
      "val Loss: 218.4079 Acc: 0.9300\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 309.5205 Acc: 0.8802\n",
      "\n",
      "val Loss: 217.6395 Acc: 0.9340\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 297.8553 Acc: 0.8819\n",
      "\n",
      "val Loss: 211.5474 Acc: 0.9400\n",
      "\n",
      "Epoch 19/29\n",
      "----------\n",
      "train Loss: 320.8824 Acc: 0.8779\n",
      "\n",
      "val Loss: 204.8677 Acc: 0.9380\n",
      "\n",
      "Epoch 20/29\n",
      "----------\n",
      "train Loss: 317.9744 Acc: 0.8794\n",
      "\n",
      "val Loss: 208.3683 Acc: 0.9400\n",
      "\n",
      "Epoch 21/29\n",
      "----------\n",
      "train Loss: 319.7024 Acc: 0.8820\n",
      "\n",
      "val Loss: 244.2530 Acc: 0.9220\n",
      "\n",
      "Epoch 22/29\n",
      "----------\n",
      "train Loss: 312.7065 Acc: 0.8808\n",
      "\n",
      "val Loss: 206.8305 Acc: 0.9420\n",
      "\n",
      "Epoch 23/29\n",
      "----------\n",
      "train Loss: 303.7014 Acc: 0.8844\n",
      "\n",
      "val Loss: 189.8910 Acc: 0.9420\n",
      "\n",
      "Epoch 24/29\n",
      "----------\n",
      "train Loss: 314.7653 Acc: 0.8787\n",
      "\n",
      "val Loss: 205.0350 Acc: 0.9380\n",
      "\n",
      "Epoch 25/29\n",
      "----------\n",
      "train Loss: 305.7220 Acc: 0.8791\n",
      "\n",
      "val Loss: 196.6076 Acc: 0.9360\n",
      "\n",
      "Epoch 26/29\n",
      "----------\n",
      "train Loss: 301.0717 Acc: 0.8804\n",
      "\n",
      "val Loss: 220.0507 Acc: 0.9320\n",
      "\n",
      "Epoch 27/29\n",
      "----------\n",
      "train Loss: 316.2844 Acc: 0.8781\n",
      "\n",
      "val Loss: 230.6640 Acc: 0.9300\n",
      "\n",
      "Epoch 28/29\n",
      "----------\n",
      "train Loss: 302.9317 Acc: 0.8792\n",
      "\n",
      "val Loss: 199.0444 Acc: 0.9400\n",
      "\n",
      "Training complete in 185m 6s\n",
      "Best val Acc : 0.942000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type ResNet50Bottom. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "res_conv2 = train_model(res_conv2, criterion, optimizer_conv, exp_lr_scheduler, num_epochs=100, flag_Load=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/99\n",
      "----------\n",
      "train Loss: 4.9282 Acc: 0.7879\n",
      "\n",
      "val Loss: 4.2401 Acc: 0.8860\n",
      "\n",
      "Epoch 1/99\n",
      "----------\n",
      "train Loss: 5.1532 Acc: 0.8165\n",
      "\n",
      "val Loss: 3.0396 Acc: 0.9160\n",
      "\n",
      "Epoch 2/99\n",
      "----------\n",
      "train Loss: 5.5796 Acc: 0.8258\n",
      "\n",
      "val Loss: 2.7954 Acc: 0.9040\n",
      "\n",
      "Epoch 3/99\n",
      "----------\n",
      "train Loss: 5.5537 Acc: 0.8373\n",
      "\n",
      "val Loss: 2.9296 Acc: 0.9280\n",
      "\n",
      "Epoch 4/99\n",
      "----------\n",
      "train Loss: 5.7970 Acc: 0.8356\n",
      "\n",
      "val Loss: 4.6166 Acc: 0.9060\n",
      "\n",
      "Epoch 5/99\n",
      "----------\n",
      "train Loss: 5.8081 Acc: 0.8383\n",
      "\n",
      "val Loss: 3.7867 Acc: 0.9120\n",
      "\n",
      "Epoch 6/99\n",
      "----------\n",
      "train Loss: 5.9344 Acc: 0.8421\n",
      "\n",
      "val Loss: 4.0190 Acc: 0.9220\n",
      "\n",
      "Epoch 7/99\n",
      "----------\n",
      "train Loss: 5.7662 Acc: 0.8465\n",
      "\n",
      "val Loss: 4.9635 Acc: 0.9080\n",
      "\n",
      "Epoch 8/99\n",
      "----------\n",
      "train Loss: 6.1165 Acc: 0.8426\n",
      "\n",
      "val Loss: 3.4955 Acc: 0.9140\n",
      "\n",
      "Epoch 9/99\n",
      "----------\n",
      "train Loss: 5.9119 Acc: 0.8478\n",
      "\n",
      "val Loss: 3.9294 Acc: 0.9180\n",
      "\n",
      "Epoch 10/99\n",
      "----------\n",
      "train Loss: 6.2463 Acc: 0.8444\n",
      "\n",
      "val Loss: 3.0142 Acc: 0.9240\n",
      "\n",
      "Epoch 11/99\n",
      "----------\n",
      "train Loss: 5.8789 Acc: 0.8507\n",
      "\n",
      "val Loss: 2.9462 Acc: 0.9380\n",
      "\n",
      "Epoch 12/99\n",
      "----------\n",
      "train Loss: 5.9721 Acc: 0.8524\n",
      "\n",
      "val Loss: 3.0203 Acc: 0.9140\n",
      "\n",
      "Epoch 13/99\n",
      "----------\n",
      "train Loss: 6.0114 Acc: 0.8520\n",
      "\n",
      "val Loss: 3.7293 Acc: 0.9200\n",
      "\n",
      "Epoch 14/99\n",
      "----------\n",
      "train Loss: 6.0607 Acc: 0.8510\n",
      "\n",
      "val Loss: 3.6186 Acc: 0.9120\n",
      "\n",
      "Epoch 15/99\n",
      "----------\n",
      "train Loss: 6.2970 Acc: 0.8515\n",
      "\n",
      "val Loss: 2.6245 Acc: 0.9120\n",
      "\n",
      "Epoch 16/99\n",
      "----------\n",
      "train Loss: 6.1204 Acc: 0.8494\n",
      "\n",
      "val Loss: 3.8012 Acc: 0.9160\n",
      "\n",
      "Epoch 17/99\n",
      "----------\n",
      "train Loss: 6.0247 Acc: 0.8494\n",
      "\n",
      "val Loss: 3.4594 Acc: 0.9260\n",
      "\n",
      "Epoch 18/99\n",
      "----------\n",
      "train Loss: 6.1101 Acc: 0.8511\n",
      "\n",
      "val Loss: 3.4669 Acc: 0.9280\n",
      "\n",
      "Epoch 19/99\n",
      "----------\n",
      "train Loss: 4.5423 Acc: 0.8712\n",
      "\n",
      "val Loss: 3.1693 Acc: 0.9160\n",
      "\n",
      "Epoch 20/99\n",
      "----------\n",
      "train Loss: 4.6118 Acc: 0.8721\n",
      "\n",
      "val Loss: 3.3782 Acc: 0.9140\n",
      "\n",
      "Epoch 21/99\n",
      "----------\n",
      "train Loss: 4.7895 Acc: 0.8682\n",
      "\n",
      "val Loss: 3.5408 Acc: 0.9140\n",
      "\n",
      "Epoch 22/99\n",
      "----------\n",
      "train Loss: 4.3930 Acc: 0.8674\n",
      "\n",
      "val Loss: 3.8538 Acc: 0.9140\n",
      "\n",
      "Epoch 23/99\n",
      "----------\n",
      "train Loss: 4.1349 Acc: 0.8696\n",
      "\n",
      "val Loss: 3.5773 Acc: 0.9260\n",
      "\n",
      "Epoch 24/99\n",
      "----------\n",
      "train Loss: 4.0655 Acc: 0.8737\n",
      "\n",
      "val Loss: 2.9014 Acc: 0.9320\n",
      "\n",
      "Epoch 25/99\n",
      "----------\n",
      "train Loss: 4.1658 Acc: 0.8718\n",
      "\n",
      "val Loss: 2.8430 Acc: 0.9240\n",
      "\n",
      "Epoch 26/99\n",
      "----------\n",
      "train Loss: 4.0632 Acc: 0.8686\n",
      "\n",
      "val Loss: 2.6605 Acc: 0.9260\n",
      "\n",
      "Epoch 27/99\n",
      "----------\n",
      "train Loss: 3.7292 Acc: 0.8764\n",
      "\n",
      "val Loss: 3.0070 Acc: 0.9180\n",
      "\n",
      "Epoch 28/99\n",
      "----------\n",
      "train Loss: 3.8664 Acc: 0.8661\n",
      "\n",
      "val Loss: 1.9272 Acc: 0.9400\n",
      "\n",
      "Epoch 29/99\n",
      "----------\n",
      "train Loss: 3.9572 Acc: 0.8660\n",
      "\n",
      "val Loss: 2.7859 Acc: 0.9160\n",
      "\n",
      "Epoch 30/99\n",
      "----------\n",
      "train Loss: 3.7773 Acc: 0.8736\n",
      "\n",
      "val Loss: 2.0171 Acc: 0.9240\n",
      "\n",
      "Epoch 31/99\n",
      "----------\n",
      "train Loss: 3.5263 Acc: 0.8733\n",
      "\n",
      "val Loss: 2.8026 Acc: 0.9060\n",
      "\n",
      "Epoch 32/99\n",
      "----------\n",
      "train Loss: 3.6998 Acc: 0.8675\n",
      "\n",
      "val Loss: 3.4107 Acc: 0.9120\n",
      "\n",
      "Epoch 33/99\n",
      "----------\n",
      "train Loss: 3.5684 Acc: 0.8703\n",
      "\n",
      "val Loss: 3.0265 Acc: 0.9120\n",
      "\n",
      "Epoch 34/99\n",
      "----------\n",
      "train Loss: 3.5082 Acc: 0.8733\n",
      "\n",
      "val Loss: 2.6200 Acc: 0.9260\n",
      "\n",
      "Epoch 35/99\n",
      "----------\n",
      "train Loss: 3.4290 Acc: 0.8746\n",
      "\n",
      "val Loss: 2.3496 Acc: 0.9260\n",
      "\n",
      "Epoch 36/99\n",
      "----------\n",
      "train Loss: 3.7416 Acc: 0.8665\n",
      "\n",
      "val Loss: 1.9161 Acc: 0.9240\n",
      "\n",
      "Epoch 37/99\n",
      "----------\n",
      "train Loss: 3.4163 Acc: 0.8725\n",
      "\n",
      "val Loss: 2.1850 Acc: 0.9240\n",
      "\n",
      "Epoch 38/99\n",
      "----------\n",
      "train Loss: 3.5489 Acc: 0.8697\n",
      "\n",
      "val Loss: 2.0076 Acc: 0.9160\n",
      "\n",
      "Epoch 39/99\n",
      "----------\n",
      "train Loss: 2.9585 Acc: 0.8800\n",
      "\n",
      "val Loss: 1.7778 Acc: 0.9260\n",
      "\n",
      "Epoch 40/99\n",
      "----------\n",
      "train Loss: 2.8919 Acc: 0.8774\n",
      "\n",
      "val Loss: 2.2549 Acc: 0.9240\n",
      "\n",
      "Epoch 41/99\n",
      "----------\n",
      "train Loss: 2.9449 Acc: 0.8792\n",
      "\n",
      "val Loss: 1.9164 Acc: 0.9240\n",
      "\n",
      "Epoch 42/99\n",
      "----------\n",
      "train Loss: 2.6506 Acc: 0.8837\n",
      "\n",
      "val Loss: 2.0769 Acc: 0.9240\n",
      "\n",
      "Epoch 43/99\n",
      "----------\n",
      "train Loss: 2.7519 Acc: 0.8775\n",
      "\n",
      "val Loss: 1.9858 Acc: 0.9140\n",
      "\n",
      "Epoch 44/99\n",
      "----------\n",
      "train Loss: 2.5922 Acc: 0.8808\n",
      "\n",
      "val Loss: 1.9995 Acc: 0.9200\n",
      "\n",
      "Epoch 45/99\n",
      "----------\n",
      "train Loss: 2.6076 Acc: 0.8857\n",
      "\n",
      "val Loss: 1.8278 Acc: 0.9140\n",
      "\n",
      "Epoch 46/99\n",
      "----------\n",
      "train Loss: 2.5065 Acc: 0.8792\n",
      "\n",
      "val Loss: 1.7818 Acc: 0.9240\n",
      "\n",
      "Epoch 47/99\n",
      "----------\n",
      "train Loss: 2.4762 Acc: 0.8840\n",
      "\n",
      "val Loss: 1.9082 Acc: 0.9240\n",
      "\n",
      "Epoch 48/99\n",
      "----------\n",
      "train Loss: 2.3823 Acc: 0.8853\n",
      "\n",
      "val Loss: 1.7960 Acc: 0.9240\n",
      "\n",
      "Epoch 49/99\n",
      "----------\n",
      "train Loss: 2.3656 Acc: 0.8867\n",
      "\n",
      "val Loss: 1.6413 Acc: 0.9180\n",
      "\n",
      "Epoch 50/99\n",
      "----------\n",
      "train Loss: 2.2606 Acc: 0.8863\n",
      "\n",
      "val Loss: 1.7517 Acc: 0.9240\n",
      "\n",
      "Epoch 51/99\n",
      "----------\n",
      "train Loss: 2.4567 Acc: 0.8753\n",
      "\n",
      "val Loss: 1.5076 Acc: 0.9240\n",
      "\n",
      "Epoch 52/99\n",
      "----------\n",
      "train Loss: 2.3432 Acc: 0.8809\n",
      "\n",
      "val Loss: 1.2952 Acc: 0.9280\n",
      "\n",
      "Epoch 53/99\n",
      "----------\n",
      "train Loss: 2.1926 Acc: 0.8847\n",
      "\n",
      "val Loss: 1.8144 Acc: 0.9100\n",
      "\n",
      "Epoch 54/99\n",
      "----------\n",
      "train Loss: 2.2857 Acc: 0.8820\n",
      "\n",
      "val Loss: 1.6847 Acc: 0.9220\n",
      "\n",
      "Epoch 55/99\n",
      "----------\n",
      "train Loss: 2.3057 Acc: 0.8777\n",
      "\n",
      "val Loss: 1.9986 Acc: 0.9180\n",
      "\n",
      "Epoch 56/99\n",
      "----------\n",
      "train Loss: 2.2051 Acc: 0.8837\n",
      "\n",
      "val Loss: 1.9891 Acc: 0.9120\n",
      "\n",
      "Epoch 57/99\n",
      "----------\n",
      "train Loss: 2.1683 Acc: 0.8838\n",
      "\n",
      "val Loss: 1.7559 Acc: 0.9180\n",
      "\n",
      "Epoch 58/99\n",
      "----------\n",
      "train Loss: 2.2276 Acc: 0.8813\n",
      "\n",
      "val Loss: 1.2554 Acc: 0.9280\n",
      "\n",
      "Epoch 59/99\n",
      "----------\n",
      "train Loss: 1.9283 Acc: 0.8940\n",
      "\n",
      "val Loss: 1.2209 Acc: 0.9300\n",
      "\n",
      "Epoch 60/99\n",
      "----------\n",
      "train Loss: 1.8136 Acc: 0.8936\n",
      "\n",
      "val Loss: 1.5814 Acc: 0.9220\n",
      "\n",
      "Epoch 61/99\n",
      "----------\n",
      "train Loss: 1.9301 Acc: 0.8891\n",
      "\n",
      "val Loss: 1.2303 Acc: 0.9240\n",
      "\n",
      "Epoch 62/99\n",
      "----------\n",
      "train Loss: 1.9250 Acc: 0.8902\n",
      "\n",
      "val Loss: 1.0788 Acc: 0.9320\n",
      "\n",
      "Epoch 63/99\n",
      "----------\n",
      "train Loss: 1.8117 Acc: 0.8898\n",
      "\n",
      "val Loss: 1.7338 Acc: 0.9060\n",
      "\n",
      "Epoch 64/99\n",
      "----------\n",
      "train Loss: 1.6760 Acc: 0.8966\n",
      "\n",
      "val Loss: 1.4809 Acc: 0.9180\n",
      "\n",
      "Epoch 65/99\n",
      "----------\n",
      "train Loss: 1.7658 Acc: 0.8906\n",
      "\n",
      "val Loss: 1.6910 Acc: 0.9180\n",
      "\n",
      "Epoch 66/99\n",
      "----------\n",
      "train Loss: 1.7397 Acc: 0.8908\n",
      "\n",
      "val Loss: 1.3274 Acc: 0.9240\n",
      "\n",
      "Epoch 67/99\n",
      "----------\n",
      "train Loss: 1.7924 Acc: 0.8927\n",
      "\n",
      "val Loss: 1.4089 Acc: 0.9300\n",
      "\n",
      "Epoch 68/99\n",
      "----------\n",
      "train Loss: 1.6508 Acc: 0.8943\n",
      "\n",
      "val Loss: 1.2875 Acc: 0.9220\n",
      "\n",
      "Epoch 69/99\n",
      "----------\n",
      "train Loss: 1.7285 Acc: 0.8934\n",
      "\n",
      "val Loss: 1.1178 Acc: 0.9300\n",
      "\n",
      "Epoch 70/99\n",
      "----------\n",
      "train Loss: 1.5960 Acc: 0.8979\n",
      "\n",
      "val Loss: 1.1164 Acc: 0.9280\n",
      "\n",
      "Epoch 71/99\n",
      "----------\n",
      "train Loss: 1.7356 Acc: 0.8869\n",
      "\n",
      "val Loss: 1.6940 Acc: 0.9200\n",
      "\n",
      "Epoch 72/99\n",
      "----------\n",
      "train Loss: 1.5974 Acc: 0.8933\n",
      "\n",
      "val Loss: 0.8953 Acc: 0.9300\n",
      "\n",
      "Epoch 73/99\n",
      "----------\n",
      "train Loss: 1.6026 Acc: 0.8936\n",
      "\n",
      "val Loss: 1.4848 Acc: 0.9220\n",
      "\n",
      "Epoch 74/99\n",
      "----------\n",
      "train Loss: 1.6375 Acc: 0.8914\n",
      "\n",
      "val Loss: 0.8909 Acc: 0.9300\n",
      "\n",
      "Epoch 75/99\n",
      "----------\n",
      "train Loss: 1.6737 Acc: 0.8901\n",
      "\n",
      "val Loss: 1.2300 Acc: 0.9120\n",
      "\n",
      "Epoch 76/99\n",
      "----------\n",
      "train Loss: 1.5094 Acc: 0.8959\n",
      "\n",
      "val Loss: 1.1092 Acc: 0.9100\n",
      "\n",
      "Epoch 77/99\n",
      "----------\n",
      "train Loss: 1.5327 Acc: 0.8926\n",
      "\n",
      "val Loss: 1.6582 Acc: 0.9120\n",
      "\n",
      "Epoch 78/99\n",
      "----------\n",
      "train Loss: 1.5194 Acc: 0.8967\n",
      "\n",
      "val Loss: 1.3054 Acc: 0.9180\n",
      "\n",
      "Epoch 79/99\n",
      "----------\n",
      "train Loss: 1.4683 Acc: 0.8970\n",
      "\n",
      "val Loss: 1.2900 Acc: 0.9300\n",
      "\n",
      "Epoch 80/99\n",
      "----------\n",
      "train Loss: 1.4925 Acc: 0.8985\n",
      "\n",
      "val Loss: 1.1694 Acc: 0.9240\n",
      "\n",
      "Epoch 81/99\n",
      "----------\n",
      "train Loss: 1.4779 Acc: 0.8980\n",
      "\n",
      "val Loss: 1.3518 Acc: 0.9200\n",
      "\n",
      "Epoch 82/99\n",
      "----------\n",
      "train Loss: 1.4066 Acc: 0.8979\n",
      "\n",
      "val Loss: 1.0681 Acc: 0.9220\n",
      "\n",
      "Epoch 83/99\n",
      "----------\n",
      "train Loss: 1.3737 Acc: 0.8991\n",
      "\n",
      "val Loss: 1.0603 Acc: 0.9280\n",
      "\n",
      "Epoch 84/99\n",
      "----------\n",
      "train Loss: 1.3911 Acc: 0.8972\n",
      "\n",
      "val Loss: 0.9854 Acc: 0.9240\n",
      "\n",
      "Epoch 85/99\n",
      "----------\n",
      "train Loss: 1.3791 Acc: 0.8977\n",
      "\n",
      "val Loss: 0.9179 Acc: 0.9260\n",
      "\n",
      "Epoch 86/99\n",
      "----------\n",
      "train Loss: 1.3333 Acc: 0.9003\n",
      "\n",
      "val Loss: 0.8058 Acc: 0.9360\n",
      "\n",
      "Epoch 87/99\n",
      "----------\n",
      "train Loss: 1.3317 Acc: 0.8991\n",
      "\n",
      "val Loss: 1.4550 Acc: 0.9120\n",
      "\n",
      "Epoch 88/99\n",
      "----------\n",
      "train Loss: 1.2877 Acc: 0.9012\n",
      "\n",
      "val Loss: 0.9155 Acc: 0.9300\n",
      "\n",
      "Epoch 89/99\n",
      "----------\n",
      "train Loss: 1.4316 Acc: 0.8927\n",
      "\n",
      "val Loss: 1.1290 Acc: 0.9140\n",
      "\n",
      "Epoch 90/99\n",
      "----------\n",
      "train Loss: 1.3540 Acc: 0.8999\n",
      "\n",
      "val Loss: 1.2227 Acc: 0.9120\n",
      "\n",
      "Epoch 91/99\n",
      "----------\n",
      "train Loss: 1.3150 Acc: 0.8985\n",
      "\n",
      "val Loss: 0.8211 Acc: 0.9180\n",
      "\n",
      "Epoch 92/99\n",
      "----------\n",
      "train Loss: 1.2536 Acc: 0.9052\n",
      "\n",
      "val Loss: 0.8736 Acc: 0.9300\n",
      "\n",
      "Epoch 93/99\n",
      "----------\n",
      "train Loss: 1.3146 Acc: 0.8981\n",
      "\n",
      "val Loss: 1.2537 Acc: 0.9160\n",
      "\n",
      "Epoch 94/99\n",
      "----------\n",
      "train Loss: 1.3882 Acc: 0.8989\n",
      "\n",
      "val Loss: 0.7593 Acc: 0.9380\n",
      "\n",
      "Epoch 95/99\n",
      "----------\n",
      "train Loss: 1.3588 Acc: 0.8981\n",
      "\n",
      "val Loss: 0.9507 Acc: 0.9160\n",
      "\n",
      "Epoch 96/99\n",
      "----------\n",
      "train Loss: 1.2504 Acc: 0.9013\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val Loss: 1.2864 Acc: 0.9100\n",
      "\n",
      "Epoch 97/99\n",
      "----------\n",
      "train Loss: 1.2720 Acc: 0.9020\n",
      "\n",
      "val Loss: 1.0633 Acc: 0.9200\n",
      "\n",
      "Epoch 98/99\n",
      "----------\n",
      "train Loss: 1.2892 Acc: 0.8976\n",
      "\n",
      "val Loss: 0.8843 Acc: 0.9360\n",
      "\n",
      "Training complete in 385m 47s\n",
      "Best val Acc : 0.940000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:256: UserWarning: Couldn't retrieve source code for container of type ResNet50Bottom. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "res_fcn= train_model(res_fcn, criterion, optimizer_conv, exp_lr_scheduler, num_epochs=100, flag_Load=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.RMSprop(fcn_model.parameters(), lr=lr, momentum=momentum, weight_decay=w_decay)\n",
    "scheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)  # decay LR by a factor of 0.5 every 30 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_class    = 20\n",
    "\n",
    "batch_size = 6\n",
    "epochs     = 500\n",
    "lr         = 1e-4\n",
    "momentum   = 0\n",
    "w_decay    = 1e-5\n",
    "step_size  = 50\n",
    "gamma      = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparmas:\n",
    "seed = 42\n",
    "workers = 8\n",
    "epochs = 2  # 100\n",
    "crop_size = 512\n",
    "lr = 5e-5\n",
    "weight_decay = 2e-4\n",
    "batch_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.models.resnet import BasicBlock, ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('/home/mnt/jupyter_dir/saved_model_dog/resnet101_28.pth.tar')\n",
    "            model.load_state_dict(checkpoint['model_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[8, 8, 8],\n",
      "         [8, 8, 8],\n",
      "         [8, 8, 8]],\n",
      "\n",
      "        [[9, 9, 9],\n",
      "         [9, 9, 9],\n",
      "         [9, 9, 9]],\n",
      "\n",
      "        [[4, 4, 4],\n",
      "         [4, 4, 4],\n",
      "         [4, 4, 4]],\n",
      "\n",
      "        [[5, 5, 5],\n",
      "         [5, 5, 5],\n",
      "         [5, 5, 5]],\n",
      "\n",
      "        [[6, 6, 6],\n",
      "         [6, 6, 6],\n",
      "         [6, 6, 6]],\n",
      "\n",
      "        [[4, 4, 4],\n",
      "         [0, 1, 1],\n",
      "         [0, 8, 8]],\n",
      "\n",
      "        [[1, 1, 1],\n",
      "         [1, 1, 1],\n",
      "         [1, 1, 1]],\n",
      "\n",
      "        [[4, 4, 4],\n",
      "         [4, 4, 4],\n",
      "         [4, 4, 4]],\n",
      "\n",
      "        [[0, 0, 7],\n",
      "         [0, 0, 5],\n",
      "         [0, 0, 7]],\n",
      "\n",
      "        [[2, 2, 2],\n",
      "         [2, 2, 2],\n",
      "         [0, 2, 2]],\n",
      "\n",
      "        [[5, 5, 5],\n",
      "         [5, 5, 5],\n",
      "         [5, 5, 5]],\n",
      "\n",
      "        [[6, 6, 6],\n",
      "         [6, 6, 6],\n",
      "         [6, 6, 6]],\n",
      "\n",
      "        [[6, 6, 6],\n",
      "         [6, 6, 6],\n",
      "         [6, 6, 6]],\n",
      "\n",
      "        [[6, 6, 6],\n",
      "         [6, 6, 6],\n",
      "         [6, 6, 6]],\n",
      "\n",
      "        [[1, 1, 1],\n",
      "         [1, 1, 1],\n",
      "         [1, 1, 1]],\n",
      "\n",
      "        [[6, 6, 6],\n",
      "         [6, 6, 6],\n",
      "         [6, 6, 6]]], device='cuda:0')\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "The size of tensor a (3) must match the size of tensor b (16) at non-singleton dimension 2",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-163-1ea641c5f0c8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;31m#preds =torch.mean(preds.float(), dim=1,keepdim=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;31m#print(preds)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mrunning_corrects\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrunning_corrects\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdouble\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mdataset_sizes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (3) must match the size of tensor b (16) at non-singleton dimension 2"
     ]
    }
   ],
   "source": [
    "res_fcn.eval()\n",
    "running_corrects = 0.0\n",
    "since = time.time()\n",
    "for inputs, labels in dataloaders['val']:\n",
    "    \n",
    "    inputs = inputs.to(device)\n",
    "    labels = labels.to(device)\n",
    "    outputs = res_fcn(inputs)\n",
    "    out = torch.squeeze(outputs, dim=2)\n",
    "    out2 = torch.squeeze(out, dim=2)\n",
    "    _, preds = torch.max(out2,1)\n",
    "    print(preds)\n",
    "    # statistics\n",
    "    #preds =torch.mean(preds.float(), dim=2,keepdim=True)\n",
    "    #preds =torch.mean(preds.float(), dim=1,keepdim=True)\n",
    "    #print(preds)\n",
    "    running_corrects += torch.sum(preds.long()==labels.data)\n",
    "\n",
    "acc = running_corrects.double() / dataset_sizes['val']\n",
    "print('{} Acc: {:.4f}'.format(\"Testing\", acc))\n",
    "print()\n",
    "time_elapsed = time.time() - since\n",
    "print('Testing complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.random.randint(low = 0,high = 9, size =(16, 3,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[8 8 0]\n",
      " [3 2 0]\n",
      " [8 7 7]]\n",
      "{0: 2, 2: 1, 3: 1, 7: 2, 8: 3}\n",
      "[[3 2 8]\n",
      " [3 1 8]\n",
      " [4 1 5]]\n",
      "{1: 2, 2: 1, 3: 2, 4: 1, 5: 1, 8: 2}\n",
      "[[6 0 7]\n",
      " [3 8 2]\n",
      " [2 6 0]]\n",
      "{0: 2, 2: 2, 3: 1, 6: 2, 7: 1, 8: 1}\n",
      "[[0 3 6]\n",
      " [2 4 1]\n",
      " [3 4 4]]\n",
      "{0: 1, 1: 1, 2: 1, 3: 2, 4: 3, 6: 1}\n",
      "[[8 1 8]\n",
      " [0 4 4]\n",
      " [1 0 5]]\n",
      "{0: 2, 1: 2, 4: 2, 5: 1, 8: 2}\n",
      "[[2 4 1]\n",
      " [1 1 0]\n",
      " [4 1 8]]\n",
      "{0: 1, 1: 4, 2: 1, 4: 2, 8: 1}\n",
      "[[1 1 5]\n",
      " [4 4 2]\n",
      " [8 5 5]]\n",
      "{1: 2, 2: 1, 4: 2, 5: 3, 8: 1}\n",
      "[[8 1 7]\n",
      " [2 5 0]\n",
      " [4 6 8]]\n",
      "{0: 1, 1: 1, 2: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 2}\n",
      "[[5 5 5]\n",
      " [7 8 2]\n",
      " [7 8 7]]\n",
      "{2: 1, 5: 3, 7: 3, 8: 2}\n",
      "[[3 0 2]\n",
      " [5 6 4]\n",
      " [0 5 1]]\n",
      "{0: 2, 1: 1, 2: 1, 3: 1, 4: 1, 5: 2, 6: 1}\n",
      "[[7 5 4]\n",
      " [6 0 1]\n",
      " [8 5 1]]\n",
      "{0: 1, 1: 2, 4: 1, 5: 2, 6: 1, 7: 1, 8: 1}\n",
      "[[6 7 6]\n",
      " [0 8 0]\n",
      " [2 1 4]]\n",
      "{0: 2, 1: 1, 2: 1, 4: 1, 6: 2, 7: 1, 8: 1}\n",
      "[[8 4 1]\n",
      " [3 0 4]\n",
      " [5 1 3]]\n",
      "{0: 1, 1: 2, 3: 2, 4: 2, 5: 1, 8: 1}\n",
      "[[2 6 4]\n",
      " [4 0 6]\n",
      " [2 6 6]]\n",
      "{0: 1, 2: 2, 4: 2, 6: 4}\n",
      "[[1 7 3]\n",
      " [4 0 4]\n",
      " [4 7 8]]\n",
      "{0: 1, 1: 1, 3: 1, 4: 3, 7: 2, 8: 1}\n",
      "[[7 1 4]\n",
      " [7 1 1]\n",
      " [6 2 1]]\n",
      "{1: 4, 2: 1, 4: 1, 6: 1, 7: 2}\n"
     ]
    }
   ],
   "source": [
    "for key in range(a.shape[0]):\n",
    "    print(a[key,:,:])\n",
    "    unique, counts = numpy.unique(a[key,:,:], return_counts=True)\n",
    "    print(dict(zip(unique, counts)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    " unique, counts = numpy.unique(a, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([17, 23, 14, 10, 22, 14, 13, 13, 18])"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unhashable type: 'numpy.ndarray'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-176-ceef492826bc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCounter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mCounter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/collections/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(*args, **kwds)\u001b[0m\n\u001b[1;32m    533\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'expected at most 1 arguments, got %d'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCounter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 535\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    536\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__missing__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/collections/__init__.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(*args, **kwds)\u001b[0m\n\u001b[1;32m    620\u001b[0m                     \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCounter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterable\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# fast path when counter is empty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 622\u001b[0;31m                 \u001b[0m_count_elements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    623\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    624\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unhashable type: 'numpy.ndarray'"
     ]
    }
   ],
   "source": [
    "import collections, numpy\n",
    "collections.Counter(a[0,:,:])\n",
    "Counter({0: 7, 1: 4, 3: 2, 2: 1, 4: 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResNet50Bottom(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace)\n",
       "    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "    (4): Sequential(\n",
       "      (0): Bottleneck(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): Bottleneck(\n",
       "        (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (2): Bottleneck(\n",
       "        (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "    (5): Sequential(\n",
       "      (0): Bottleneck(\n",
       "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): Bottleneck(\n",
       "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (2): Bottleneck(\n",
       "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (3): Bottleneck(\n",
       "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "    (6): Sequential(\n",
       "      (0): Bottleneck(\n",
       "        (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (2): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (3): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (4): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (5): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (6): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (7): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (8): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (9): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (10): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (11): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (12): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (13): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (14): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (15): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (16): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (17): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (18): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (19): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (20): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (21): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (22): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "    (7): Sequential(\n",
       "      (0): Bottleneck(\n",
       "        (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): Bottleneck(\n",
       "        (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (2): Bottleneck(\n",
       "        (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (conv): Conv2d(2048, 10, kernel_size=(7, 7), stride=(1, 1), bias=False)\n",
       ")"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_fcn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}