convnet_tutorial3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mCt-frpFH8oN"
   },
   "source": [
    "The tutorials use PyTorch. You will need to load the following dependencies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hqnl0AKVXIA4"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import PIL\n",
    "import imageio\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import skimage.transform\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "from torchvision import datasets, transforms\n",
    "from IPython import display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CKeYuM-cIXxs"
   },
   "source": [
    "The code below helps visualize PyTorch tensors as images, loads the MNIST training set, and provides training and test functions just like in the previous two tutorials."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZZd_rI8edYIB"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "def show(img):\n",
    "    \"\"\"Show PyTorch tensor img as an image in matplotlib.\"\"\"\n",
    "    npimg = img.cpu().detach().numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)), interpolation='nearest')\n",
    "    plt.grid(False)\n",
    "    plt.gca().axis('off')\n",
    "\n",
    "def display_thumb(img):\n",
    "  display.display(transforms.Resize(128)(img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dzfEE578uSNp"
   },
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the training and test dataset.\n",
    "mnist_train = datasets.MNIST('/tmp/mnist', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = datasets.MNIST('/tmp/mnist', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "# Size of the batches the data loader will produce.\n",
    "batch_size = 64\n",
    "\n",
    "# This creates the dataloaders.\n",
    "train_loader = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10899,
     "status": "ok",
     "timestamp": 1586534977023,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "TgAJ94UgK1VU",
    "outputId": "7aad0856-eaf6-4cb0-a70f-da77c43c4e7e"
   },
   "outputs": [],
   "source": [
    "def train(model, criterion, data_loader, optimizer, num_epochs):\n",
    "    \"\"\"Simple training loop for a PyTorch model.\"\"\" \n",
    "    \n",
    "    # Make sure model is in training mode.\n",
    "    model.train()\n",
    "    \n",
    "    # Move model to the device.\n",
    "    model.to(device)\n",
    "    \n",
    "    # Exponential moving average of the loss.\n",
    "    ema_loss = None\n",
    "    \n",
    "    # Loop over epochs.\n",
    "    for epoch in range(num_epochs):    \n",
    "      \n",
    "    # Loop over data.\n",
    "      for batch_idx, (data, target) in enumerate(data_loader):\n",
    "            \n",
    "          # Forward pass.\n",
    "          output = model(data.to(device))\n",
    "          loss = criterion(output.to(device), target.to(device))\n",
    "          \n",
    "          # Backward pass.\n",
    "          optimizer.zero_grad()\n",
    "          loss.backward()\n",
    "          optimizer.step()\n",
    "          \n",
    "          # NOTE: It is important to call .item() on the loss before summing.\n",
    "          if ema_loss is None:\n",
    "            ema_loss = loss.item()\n",
    "          else:\n",
    "            ema_loss += (loss.item() - ema_loss) * 0.01 \n",
    "          \n",
    "      # Print out progress the end of epoch.\n",
    "      print('Train Epoch: {} \\tLoss: {:.6f}'.format(\n",
    "            epoch, ema_loss),\n",
    "      )\n",
    "              \n",
    "              \n",
    "def test(model, data_loader):\n",
    "    \"\"\"Measures the accuracy of a model on a data set.\"\"\"\n",
    "    \n",
    "    # Make sure the model is in evaluation mode.\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "\n",
    "    # We do not need to maintain intermediate activations while testing.\n",
    "    with torch.no_grad():   \n",
    "        \n",
    "        # Loop over test data.\n",
    "        for data, target in data_loader:\n",
    "          \n",
    "            # Forward pass.\n",
    "            output = model(data.to(device))\n",
    "            \n",
    "            # Get the label corresponding to the highest predicted probability.\n",
    "            pred = output.argmax(dim=1, keepdim=True)\n",
    "            \n",
    "            # Count number of correct predictions.\n",
    "            correct += pred.cpu().eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    # Print test accuracy.\n",
    "    percent = 100. * correct / len(data_loader.dataset)\n",
    "    print(f'Accuracy: {correct}/{len(data_loader.dataset)} ({percent:.0f}%)')\n",
    "    return percent\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ALURWmhbK7yM"
   },
   "source": [
    "## ResNets\n",
    "\n",
    "We are going train a deep neural network using residual connections [2].\n",
    "\n",
    "The network below has a list of `conv_blocks`. Each convolutional block is convolution layer followed by a ReLU with optional pooling.\n",
    "\n",
    "[2] He, Kaiming, et al. \"Identity mappings in deep residual networks.\" European conference on computer vision. Springer, Cham, 2016.\n",
    "\n",
    "**Task**. Implement residual connections within `apply_conv_block`. Residual connection sum the output of a convolutional block with its input. Note that if the output and input have different sizes, you should skip the residual connection for that layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tp6FGIdgmDrG"
   },
   "outputs": [],
   "source": [
    "class DeepConvolutionalNetwork(nn.Module):\n",
    "  \"\"\"Convolutional network that can go arbitrary deep.\"\"\"\n",
    "  \n",
    "  def __init__(self, num_repeated_layers=1, use_residuals=False):\n",
    "    super().__init__()\n",
    "    num_channels = 8\n",
    "    self.use_residuals = use_residuals\n",
    "\n",
    "    conv_blocks = []\n",
    "    # 1x28x28 -> 8x12x12.\n",
    "    conv_blocks.append(nn.Sequential(\n",
    "      nn.Conv2d(1, num_channels, kernel_size=5),\n",
    "      nn.MaxPool2d(2),\n",
    "      nn.ReLU(inplace=True),\n",
    "    ))\n",
    "    for _ in range(num_repeated_layers):\n",
    "      conv_blocks.append(nn.Sequential(\n",
    "        nn.Conv2d(num_channels, num_channels, kernel_size=3, padding=1),          \n",
    "        nn.ReLU(inplace=True),\n",
    "      ))\n",
    "    # 8x12x12 -> 8x4x4.\n",
    "    conv_blocks.append(nn.Sequential(\n",
    "      nn.Conv2d(num_channels, num_channels, kernel_size=5),\n",
    "      nn.MaxPool2d(2),\n",
    "      nn.ReLU(inplace=True),\n",
    "    ))\n",
    "    # It's important to wrap layers into ModuleList so the PyTorch knows that\n",
    "    # the self.conv_blocks has parameters to optimize.\n",
    "    self.conv_blocks = nn.ModuleList(conv_blocks)\n",
    "    self.linear = nn.Linear(num_channels * 4 ** 2, 10)  \n",
    "   \n",
    "  def apply_conv_block(self, conv_block, x):\n",
    "    output = conv_block(x)\n",
    "    if not self.use_residuals:\n",
    "      return output\n",
    "    else:\n",
    "      ###########################################################################\n",
    "      # Implement the residual connection below.                                #\n",
    "      ###########################################################################\n",
    "            \n",
    "\n",
    "\n",
    "  def forward(self, x):\n",
    "    for block in self.conv_blocks:\n",
    "      x = self.apply_conv_block(block, x) \n",
    "    x = self.linear(x.view(x.size(0), -1))\n",
    "    return x\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lRO8MhydMSOS"
   },
   "source": [
    "Let's look at the modules in our deep network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 476
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1259,
     "status": "ok",
     "timestamp": 1586535140016,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "qInniW1-MXCW",
    "outputId": "accfa0fb-b2b8-4244-edee-1ca8ebe97f68"
   },
   "outputs": [],
   "source": [
    "# Print the structure of a (residual) network with 3 convolutional layers.\n",
    "DeepConvolutionalNetwork(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7zLAIfS2Q4fM"
   },
   "source": [
    "Now we are going to train the model with increasing number of layers with and without residual connections. What do you observe?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 323
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 56602,
     "status": "ok",
     "timestamp": 1586535196647,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "6l6YAVuJRGbQ",
    "outputId": "dbc9b867-7679-42bf-9db1-5e3fd619b845"
   },
   "outputs": [],
   "source": [
    "for num_layer in 1, 5:\n",
    "  for use_residual in [False, True]:\n",
    "    print(f'Training a neural network with {num_layer} layers and use_residual={use_residual}')\n",
    "    conv_model = DeepConvolutionalNetwork(num_layer, use_residuals=use_residual)\n",
    "    optimizer = torch.optim.SGD(conv_model.parameters(), lr=1e-1)\n",
    "    train(conv_model, nn.CrossEntropyLoss(), train_loader, optimizer, num_epochs=1)\n",
    "    accuracy = test(conv_model, test_loader)\n",
    "    if use_residual:\n",
    "      assert accuracy > 90, 'Accuracy is very low. Check the implementation of your residual network.'\n",
    "print('Done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CPDc4SCxARBI"
   },
   "source": [
    "## Imagenet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "W0vWX6U_4ge8"
   },
   "source": [
    "Imagenet is the most famous dataset for image classification that is still in use. Real ImageNet dataset is very big (~150Gb). So we will use a smaller version that contains only two classes: bees and ants.\n",
    "\n",
    "First, download the required files and construct the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 72440,
     "status": "ok",
     "timestamp": 1586535214835,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "BxBuHcp70tjX",
    "outputId": "98bfa44f-7892-4ce0-b814-d16cee07c8a9"
   },
   "outputs": [],
   "source": [
    "! rm -rf /tmp/imagenet/unpacked\n",
    "! mkdir -p /tmp/imagenet/unpacked\n",
    "! [[ -f /tmp/imagenet/hymenoptera_data.zip ]] || curl https://download.pytorch.org/tutorial/hymenoptera_data.zip > /tmp/imagenet/hymenoptera_data.zip\n",
    "! cd /tmp/imagenet/unpacked && unzip ../hymenoptera_data.zip > /dev/null\n",
    "imagenet_dataset = datasets.ImageFolder('/tmp/imagenet/unpacked/hymenoptera_data/val')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mCNAGpNF41mW"
   },
   "source": [
    "Explore the images within the dataset. Unlike MNIST or CIFAR the dataset has relatively high-resolution images that can vary in size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 695
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 71453,
     "status": "ok",
     "timestamp": 1586535214836,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "Fx4HoaMP30Ya",
    "outputId": "102fa6b2-07a5-45ba-f608-87af97af1cf7"
   },
   "outputs": [],
   "source": [
    "print('ImageNet classes:', *imagenet_dataset.classes)\n",
    "\n",
    "# Show a random image and the corresponding target.\n",
    "for i in range(3):\n",
    "  img, target = random.choice(imagenet_dataset)\n",
    "  print('Label of image: %d (%s). Original size: %s' % (target, imagenet_dataset.classes[target], img.size))\n",
    "\n",
    "  # Reduce image size by half to fit the images on the page :)\n",
    "  display.display(img.resize((img.size[0] // 2, img.size[1] // 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "x2n_zYU96dmU"
   },
   "source": [
    "A common technique to handle this is to take some segment of the image and resize it to have an image of fixed square size.\n",
    "\n",
    "For training the segment selection is usually randomized.\n",
    "For evaluation a \"center\" crop is used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 867
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 70561,
     "status": "ok",
     "timestamp": 1586535214837,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "VyAClHzg3PKi",
    "outputId": "076d162c-e0f4-4b6a-ac54-477d9710877b"
   },
   "outputs": [],
   "source": [
    "img, _ = random.choice(imagenet_dataset)\n",
    "print('Actual image:')\n",
    "display.display(img)\n",
    "print('Input to the net for training')\n",
    "display.display(transforms.RandomResizedCrop(224)(img))\n",
    "print('Input to the net for eval')\n",
    "display.display(transforms.CenterCrop(224)(transforms.Resize(256)(img)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_pY7eXqw7BmK"
   },
   "source": [
    "If a pre-trained model is used, it's important to match the transformation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 117,
     "referenced_widgets": [
      "cb7a435d8db0472eb3388f5d0d53043f",
      "45200009cb0f40449c05c889854d653c",
      "bdc046fad38343b3aba8fd0fbaffb9c9",
      "7421dc7a0a874ab8835cdb30a73372ee",
      "a18479993a7547d688eed72386289911",
      "3815582c26af4ee4a6031a103a196eff",
      "93f3ec87a84542ada9518c7771066cf3",
      "a8773930a517488dbdbb30ac3bef7971"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 72637,
     "status": "ok",
     "timestamp": 1586535217729,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "rUcHTuWN3Y1S",
    "outputId": "86c54668-e885-48c3-85f5-be04277da0e5"
   },
   "outputs": [],
   "source": [
    "# This is the default transform used in ImageNet models.\n",
    "inference_transform = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "# For visualization purposes we'll create a separate transform that operates in image space.\n",
    "inference_transform_show = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "])\n",
    "\n",
    "# Reload out dataset with this transform.\n",
    "transformed_imagenet_dataset = datasets.ImageFolder(\n",
    "    '/tmp/imagenet/unpacked/hymenoptera_data/val',\n",
    "    transform=inference_transform,\n",
    ")\n",
    "\n",
    "transformed_imagenet_loader = torch.utils.data.DataLoader(transformed_imagenet_dataset, batch_size=16, shuffle=False)\n",
    "\n",
    "# Load a pretrained model.\n",
    "imagenet_resnet = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "# The loaded model is trained to classify image across 1000 classes. We have only two classes so have to take the two correpsonding lines from the softmax.\n",
    "ANT_CLASS_ID = 310\n",
    "BEE_CLASS_ID = 309\n",
    "imagenet_resnet.fc.weight.data = imagenet_resnet.fc.weight.data[[ANT_CLASS_ID, BEE_CLASS_ID]]\n",
    "imagenet_resnet.fc.bias.data = imagenet_resnet.fc.bias.data[[ANT_CLASS_ID, BEE_CLASS_ID]]\n",
    "\n",
    "# If GPU available, transfer model there\n",
    "if device == 'cuda':\n",
    "    imagenet_resnet = imagenet_resnet.cuda()\n",
    "\n",
    "# Check the model accuracy.\n",
    "test(imagenet_resnet, transformed_imagenet_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vusiegGD_OHa"
   },
   "source": [
    "The model quality is pretty high, but there are a few mistakes. Explore misclassified images below. Why do you think the network made a mistake?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 894
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 71691,
     "status": "ok",
     "timestamp": 1586535217730,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "rJtp6Jn582ZD",
    "outputId": "b855564b-56fd-4742-85fb-d0ee85dfafa8"
   },
   "outputs": [],
   "source": [
    "imagenet_resnet.eval()  # make sure the model is in evaluation mode\n",
    "\n",
    "for i in range(len(transformed_imagenet_dataset)):\n",
    "  img, _ = imagenet_dataset[i]\n",
    "  tensor, target = transformed_imagenet_dataset[i]\n",
    "    \n",
    "  if device == 'cuda':\n",
    "    tensor = tensor.cuda()\n",
    "    \n",
    "  _, prediction = imagenet_resnet(tensor.unsqueeze(0)).squeeze(0).cpu().max(-1)\n",
    "  if prediction != target:\n",
    "    print('Img id=%d. Excpected class %s, but predicted class %s.' % (\n",
    "        i,\n",
    "        imagenet_dataset.classes[target],\n",
    "        imagenet_dataset.classes[prediction]))\n",
    "    display_thumb(img)\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4kPeyocHV0vD"
   },
   "source": [
    "**Task** As you saw, the random crop does not always catche the object. Let's understand what it does for a image. For a random image from `imagenet_dataset` sample a few random crops and find the most _bee-like_ and the most _ant-like_ ones. How do they look like?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 499
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 70550,
     "status": "ok",
     "timestamp": 1586535217731,
     "user": {
      "displayName": "Anton Bakhtin",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjq-QU8evTujx-PnMoIketNeW4yB_kgdtnRJ03IxnY=s64",
      "userId": "12331399045734156147"
     },
     "user_tz": 240
    },
    "id": "U9paFPHUKdZA",
    "outputId": "6a03706f-0f96-41ca-8a6f-344184514c11"
   },
   "outputs": [],
   "source": [
    "imagenet_resnet.eval()  # make sure the model is in evaluation mode\n",
    "\n",
    "random_id = 40\n",
    "img, _ = imagenet_dataset[random_id]\n",
    "\n",
    "random_crops = [transforms.RandomResizedCrop(224)(img)\n",
    "                for _ in range(10)]\n",
    "\n",
    "\n",
    "def beeness(image):\n",
    "    \n",
    "  # Convert image to tensor and apply default for the dataset mean-variance normalization.\n",
    "  image = transforms.ToTensor()(image)\n",
    "  image = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(image)\n",
    "  ###########################################################################\n",
    "  # TODO: Sort the random_crops by probability of bee-ness.                 #\n",
    "  # You need to compute logits and convert them to probabilities.           #\n",
    "  ###########################################################################\n",
    "   \n",
    "  return probs[0].item()\n",
    "  \n",
    "random_crops = sorted(random_crops, key=beeness)\n",
    "\n",
    "print('Most ant-like'); display.display(random_crops[0])\n",
    "print('Most bee-like'); display.display(random_crops[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "K_mlTK4LMlse"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "convnet_tutorial3.ipynb",
   "provenance": [
    {
     "file_id": "1Eq_v8eTk4XAxFXhjL6rk26gwfwvpZYBn",
     "timestamp": 1553633151446
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "053499a12c9a4c4ab3c2effba82aba70": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "0542a250bd1a443c89cba5d525ee5e7b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "07dc9f747d104c11a43f01a95ed57902": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_da77a43ce62847e095c45f116b46d575",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5ba9ed21d81a4642a60bf8a25043b231",
      "value": 1
     }
    },
    "1b7ef3e5f8014502929183c185bde554": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0542a250bd1a443c89cba5d525ee5e7b",
      "placeholder": "​",
      "style": "IPY_MODEL_e95daa37066d4e369e3541e1816e07c1",
      "value": " 1654784/? [00:16&lt;00:00, 1174712.52it/s]"
     }
    },
    "2e4528b2db2a497c89907cd6366ad893": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f768eb9193774ea28185768e913147b9",
      "placeholder": "​",
      "style": "IPY_MODEL_9954acb0661f4e438c7baf7e4a43f493",
      "value": " 8192/? [00:00&lt;00:00, 11911.67it/s]"
     }
    },
    "3373c12efcad42459f1d367d75ea3d46": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "34c560af3a2c4560af7d0ff51e3ba192": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3815582c26af4ee4a6031a103a196eff": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3b9341e0dd1942a590a8a75b47c4c02a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3bd8a63651ee4c538747d328c7b1e192": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "45200009cb0f40449c05c889854d653c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4dc4b3a5bc7d4d1896a459836fc7610f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5ba9ed21d81a4642a60bf8a25043b231": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5cc2b2260d394c79abdac992228b0e31": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "63cefcf9944d48cd8896bff26f85d8d0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4dc4b3a5bc7d4d1896a459836fc7610f",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_053499a12c9a4c4ab3c2effba82aba70",
      "value": 1
     }
    },
    "694a02b8ec6340b29c7bd53fd4a91e29": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8a64b78684d54b6bbc4949ac797d2ebe",
       "IPY_MODEL_2e4528b2db2a497c89907cd6366ad893"
      ],
      "layout": "IPY_MODEL_8047845f5cf24c9889c98d227bdbab79"
     }
    },
    "6e22ffc1689840a080a7a0306b6e40aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_af5aba6f88f94c03b40f93feaf5513b4",
      "placeholder": "​",
      "style": "IPY_MODEL_3b9341e0dd1942a590a8a75b47c4c02a",
      "value": " 9920512/? [00:20&lt;00:00, 4620495.53it/s]"
     }
    },
    "71de708bde354564bb46cdbd2254557c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7421dc7a0a874ab8835cdb30a73372ee": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a8773930a517488dbdbb30ac3bef7971",
      "placeholder": "​",
      "style": "IPY_MODEL_93f3ec87a84542ada9518c7771066cf3",
      "value": " 44.7M/44.7M [00:00&lt;00:00, 149MB/s]"
     }
    },
    "76bca89552e94a14a770b01d2ed5a632": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8047845f5cf24c9889c98d227bdbab79": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8a64b78684d54b6bbc4949ac797d2ebe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f723b21b2c314b599f65d06b686693bf",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3bd8a63651ee4c538747d328c7b1e192",
      "value": 1
     }
    },
    "927c1878303a4601a439af11796c860d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_71de708bde354564bb46cdbd2254557c",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c6d63cfbab0d43ef97fd112250844b4b",
      "value": 1
     }
    },
    "93f3ec87a84542ada9518c7771066cf3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9954acb0661f4e438c7baf7e4a43f493": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a18479993a7547d688eed72386289911": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "a42490f444564fd7ac946d852614792f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_07dc9f747d104c11a43f01a95ed57902",
       "IPY_MODEL_6e22ffc1689840a080a7a0306b6e40aa"
      ],
      "layout": "IPY_MODEL_3373c12efcad42459f1d367d75ea3d46"
     }
    },
    "a8773930a517488dbdbb30ac3bef7971": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "af5aba6f88f94c03b40f93feaf5513b4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ba3f6e39cfab49c2bdbf95eb8dcea867": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5cc2b2260d394c79abdac992228b0e31",
      "placeholder": "​",
      "style": "IPY_MODEL_d83b581aeb674ca293e3b67ecdc9059f",
      "value": " 32768/? [00:00&lt;00:00, 52827.20it/s]"
     }
    },
    "bdc046fad38343b3aba8fd0fbaffb9c9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3815582c26af4ee4a6031a103a196eff",
      "max": 46827520,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a18479993a7547d688eed72386289911",
      "value": 46827520
     }
    },
    "c6d63cfbab0d43ef97fd112250844b4b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "cb7a435d8db0472eb3388f5d0d53043f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bdc046fad38343b3aba8fd0fbaffb9c9",
       "IPY_MODEL_7421dc7a0a874ab8835cdb30a73372ee"
      ],
      "layout": "IPY_MODEL_45200009cb0f40449c05c889854d653c"
     }
    },
    "d83b581aeb674ca293e3b67ecdc9059f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "da77a43ce62847e095c45f116b46d575": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e95daa37066d4e369e3541e1816e07c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ef04916378e54e12806b08b0bcfbb8fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_63cefcf9944d48cd8896bff26f85d8d0",
       "IPY_MODEL_1b7ef3e5f8014502929183c185bde554"
      ],
      "layout": "IPY_MODEL_76bca89552e94a14a770b01d2ed5a632"
     }
    },
    "efa04fca6187462393be792a87847ead": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_927c1878303a4601a439af11796c860d",
       "IPY_MODEL_ba3f6e39cfab49c2bdbf95eb8dcea867"
      ],
      "layout": "IPY_MODEL_34c560af3a2c4560af7d0ff51e3ba192"
     }
    },
    "f723b21b2c314b599f65d06b686693bf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f768eb9193774ea28185768e913147b9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}