010_fastbook_resnet_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[K     |████████████████████████████████| 720 kB 12.2 MB/s \n",
      "\u001b[K     |████████████████████████████████| 1.2 MB 31.7 MB/s \n",
      "\u001b[K     |████████████████████████████████| 186 kB 45.4 MB/s \n",
      "\u001b[K     |████████████████████████████████| 46 kB 3.9 MB/s \n",
      "\u001b[K     |████████████████████████████████| 56 kB 3.9 MB/s \n",
      "\u001b[K     |████████████████████████████████| 51 kB 355 kB/s \n",
      "\u001b[?25hMounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "%pip install fastbook -Uqq\n",
    "import fastbook\n",
    "fastbook.setup_book()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install nbdev -qq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST CNN\n",
    "\n",
    "Reference:\n",
    "- [fastai - Lesson 7 Resnet on MNIST](https://nbviewer.org/github/fastai/course-v3/blob/master/nbs/dl1/lesson7-resnet-mnist.ipynb)\n",
    "- https://walkwithfastai.com/MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision.all import *\n",
    "from nbdev.showdoc import show_doc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='15687680' class='' max='15683414' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.03% [15687680/15683414 00:01<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = untar_data(URLs.MNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#2) [Path('/root/.fastai/data/mnist_png/training'),Path('/root/.fastai/data/mnist_png/testing')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fnames = get_image_files(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Path('/root/.fastai/data/mnist_png/training/5/1817.png')"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnames[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA20lEQVR4nGNgGBJAfeG//x1YZfjzjn38+/fvXwYGFjQZXrNEH14Ghgs9DH7omoTbHv39+/fDOiUBDPMC1v38+/fPbH9pTKvknv7++/Z+nTZWd8z8+3eePA7Xq7z764vuOiYYg5WZ4fQfXJLX5zCI4DCUgYFB5M9NWdyy8T9moFuKBDb9zcQtyXX+gws2cQFJBgYGBvPvn+2xSD783crAwMBQ9/o0FslFf/cwMDAwMBz6pYcpafvu7TYHBqes928UsGg1v/P395fff982YHUq35S/f/++S8QqR2UAADOTTFG3AsPuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "PILImageBW mode=L size=28x28"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PILImageBW.create(fnames[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parent_label(fnames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DataBlock approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Setting up Pipeline: PILBase.create\n",
    "Setting up Pipeline: parent_label -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n",
    "\n",
    "Building one sample\n",
    "Pipeline: PILBase.create\n",
    "    starting from\n",
    "      /root/.fastai/data/mnist_png/training/3/13332.png\n",
    "    applying PILBase.create gives\n",
    "      PILImageBW mode=L size=28x28\n",
    "  Pipeline: parent_label -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n",
    "    starting from\n",
    "      /root/.fastai/data/mnist_png/training/3/13332.png\n",
    "    applying parent_label gives\n",
    "      3\n",
    "    applying Categorize -- {'vocab': None, 'sort': True, 'add_na': False} gives\n",
    "      TensorCategory(3)\n",
    "Final sample: (PILImageBW mode=L size=28x28, TensorCategory(3))\n",
    "'''\n",
    "db = DataBlock(blocks=(ImageBlock(cls=PILImageBW), CategoryBlock),\n",
    "               get_items=get_image_files,\n",
    "               get_y=parent_label, \n",
    "               splitter=GrandparentSplitter(train_name='training', valid_name='testing'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting items from /root/.fastai/data/mnist_png\n",
      "Found 70000 items\n",
      "2 datasets of sizes 60000,10000\n",
      "Setting up Pipeline: PILBase.create\n",
      "Setting up Pipeline: parent_label -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n"
     ]
    }
   ],
   "source": [
    "dset = db.datasets(path, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PILImageBW mode=L size=28x28, TensorCategory(5))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dset.train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f940a48f850>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEQAAABUCAYAAAA7xZEpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAJb0lEQVR4nO2bW28bZRqAn288tseHcSapExy7pInT4hwQNGoDUinLigsqDlJ7x4q9QuIGaa/4Bay00l5ysfsDuOEHVLAKQgg1VFVRqSiKSVJPD4E0pibY8Sm2x/bYsxet3Wha2oiND6z8SJYiO/H3fo/e+Q7v90VYlsWAB0i9DqDfGAixMRBiYyDExkCIjYEQGwMhNvpCiBDighDCEELs3n8lehVLXwi5z98sy/Lff8V6FUQ/CekL+knIP4UQaSHEJSHEn3sVhOiHvYwQ4kVgDagBfwH+DRy3LOtW12PpByF2hBCfA/+xLOtf3W67nx6ZvViA6EXDPRcihNCEEGeEEIoQQhZC/BX4E/B5L+KRe9GoDSfwD2AGaADXgXOWZem9CKYvx5Be0vNHpt8YCLExEGJjIMTGk2aZ/+cR95HrnEGG2BgIsTEQYmMgxMZAiI2BEBsDITYGQmz0w/b/f8I0TZrNJuVyGcMw2N3dxTRNACRJQtM03G43fr8fh8PxxO/7QwuxLAvDMCiVSiwtLbG2tsbS0hLJZBLLsvD5fLz77rvMzs5y9uxZfD7fE7/zDyPEsqy2gFqtRqlUolqtsr29TTab5fvvv+fOnTvcvXuXXC6HZVlUq1V0XceyLN588819tfOkAlHf7GVM08Q0TRKJBBsbGywvL7O5ucn6+jrZbJZisUitVqPRaNDqkxACWZaZnJxkeXmZUCi09ysfuZfpuwxpZUK9XqfRaFAul6lUKuRyOYrFIteuXWNzc5ObN29y584dMpkMpVIJl8uFx+NhcnISVVUBaDQapFIpIpHIvsYP6EMhzWaTRqNBLpcjm83yzTffEI/HuXr1Krqutx+VZrPZlidJEhMTExw+fJi3336baDQKgGEYLC8vI8syLpdrX+33XEirY+Vymd3dXfL5PNlslmQySSqVYn19nc3NTba2tigWizQaDQCCwSBer5dnn32WkZERIpEIw8PDPP/88wwPDwP3MkQIgRBi30J6PoaYpkm9XieRSHDx4kWuXr3KlStXSKfTFAqFdhY0m00APB4PbrebF198kaeffpr33nuPqakpFEXB4XDgcrmQpAfLq1b/9r53n/4YQ2q1GoZhUC6XKRQKZLNZfv31V3744QeuX7+Oruvs7OxQLpcxTZNQKMShQ4cIhUIMDQ0xPDyM3+/n2LFjjI+PEw6H8Xq9OBwOJElCkiSEeNDXvT/vh64LKZfL3Lp1i3g8zuXLl1ldXeXatWuYptl+HPbOEgsLC8zPz7O4uMiRI0cIh8OoqoqiKDidzvbvHRQdF9JsNjFNk2w2i67rJJNJEokEW1tb6LpOKpVCCIHb7UaSJJ566inC4TChUAhN01hYWGhnQyAQQFVVnE4nDofjQEW06LiQRqOBYRhcvnyZDz74gGKxSD6fb48LiqLg8Xjwer24XC5eeeUVFhYWOHHiBBMTEwwNDaEoSkc6/yg6LqRSqZBIJFhdXaVQKGAYBs1mk2AwyOHDh4nFYkxPT+Pz+fD7/Rw9epRQKMTo6Ch+vx9ZlrsmA7ogZGdnh08//ZR4PN5eUgMcOXKEl19+mTNnzrC4uIiiKLjd7q52/lF0TEij0aBer5NOp1lfX2djYwPLsojFYpw6dYrZ2VleeOEFJiYmUBSl65nwW3RUSLlc5qeffuLixYuUSiUAFhcXef/994lEIoyNjbUXTv1CRwtErYFz74bL5/MxOjqK1+vtKxEtOpYhdhktIX6/n5GREZxOZ18K6ViGyLKM3+8nGo3y6quv8swzzwD3luq1Wq29FO83OibE4XDg8XgYHx/n5MmTzMzMIITANE2q1Wp7VdpvdLzIrGkar732GnNzcwghiMfjfPLJJ6yurmIYRrv+2S90fB3i8XiIxWLcvHmzLSSdTuNwOJidnb0XhNzzKkQbx4cffvi4zx/74X5p7UAdDgeqqnLjxg00TUPTNDweD4FAADjYTdo++PsjY+10q5IkIcsy4+PjnDt3jmPHjpHP59F1nQsXLnD37l366eJf13K1Vd3KZDJcuXKFer3OF198gaqqaJrWrnf0mq6d3CmKQjAYZHp6mqNHj2JZFt999x3xeJzr16+Ty+W6Fcpj6coYshePx8PMzAyyLHP79m1M02RjYwOfz8fY2BgOh6Nd+OkwvRlD7KiqytzcHIuLi0xMTJBOp/nyyy9ZWVlhe3sbwzB6OqYcWIZYltUuA9brdSzL+s2zkFYVPBKJUKvVWFlZoVAooOs6Pp+PQ4cO4XA4Oj0dPzJDDqzFZrNJvV5v71/2lvnsRV8hBCMjI5w8eZL19XWEECQSCXRdJxqNEovFcLvdKIpyUOHtmwPLkO3tbT766COWlpY4f/48q6urSJJEs9lE07SH1hiSJLXLhk6nE5fLRTqdRpZltra2mJqaYmxs7Hd1ap90NkOKxSKfffYZv/zyC9vb2xw/fhyfz4fT6WRqaurhhmUZWZYJh8MsLCxQLBZZW1vj9u3b7O7u8tZbb2FZVtd3xAc2qCqKwokTJ4jFYgghyGQyJBIJfv7558f+XTgc5uzZs5w+fZrp6WlM02RtbY0bN26Qz+epVqsHFeK+ODAhTqeT8fFxRkdHAahWq2QyGXZ2dtoH161jS3hQL3G5XGiaRjgcRtM0TNMkl8uRz+epVCpdLxMc2CPj9/t5/fXXGRoa4quvvsI0TeLxOJIkUS6XmZ+fZ35+HlVVUVW1faJfKpUoFotcunSpvQNuVeBbZzXd5MCEtMaDyclJgsFgu6O6riPLMqVSiWAw2J6BCoUCqVSKXC5HKpUimUxSKBRwuVyoqorH4+lJ4fnADrtb024+n2dlZYVvv/2Wjz/+GMMwMAwDVVWJRCKoqkogEKBcLpPNZjEMg0qlQqFQYHd3l5deeonZ2Vneeecd5ubm2ue1HaCzh92SJOF2uwkEAszMzGAYBhMTEySTSTKZDIVCgR9//LG9NG8t4FoEAgGGhoaIRCLMzc2haVpP6iQHfh2idZZrGAbpdBpd1zl//jyJRIKvv/76XqNCtAvPLpcLt9vNG2+8wenTpzl16hTRaBRFUfZ9p+N30p3rEJIk4XK5kGW5fSYbi8VoNBrcunXroX2Ky+XC6/USjUZ57rnnCIfD7YJRL+johZnWXbHWzcFKpfJwAPeX8j6fD7fbjSzL3drtPjJDen6DqIcM/qNqPwyE2BgIsTEQYuNJ027/nUZ3mEGG2BgIsTEQYmMgxMZAiI2BEBv/BTBJHqnXR6xAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 72x72 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dset.show(dset.train[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transforms\n",
    "- Ensure our images are all the same size\n",
    "- Make sure our output are the tensor our models are wanting\n",
    "- Give some image augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToTensor: Converts to Tensor\n",
    "# CropPad & RandomCrop : Resizing Transforms\n",
    "# Applied on CPU via after_item\n",
    "tfms=[ToTensor(), CropPad(size=34, pad_mode=PadMode.Zeros), RandomCrop(size=28)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IntToFloatTensor : Converts to a float\n",
    "# Normalize: Normalizes data\n",
    "gpu_tfms=[IntToFloatTensor(), Normalize()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = dset.dataloaders(bs=128, after_item=tfms, after_batch=gpu_tfms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIHCAYAAADpfeRCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZQV1dX38X0YlKFpEEEGIyCD8CIiQRF5Hxx5cCYLQWOMCGoMykNIHABlKeAUNRKMBMQgBAdQDA4oCAujIEo0IkiUgCBoVJAhTMrcLdL1/tH6vMZ9Gureut1Vdff3s5ZrJb++de+WdWx+FKdPuSAIBAAA2FIp7gEAAEDFowAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGUQAAADCIApAF51wz59wc59yXzrlNzrlxzrkqcc8FZMo518o5V+Scmxr3LECmnHP/xzk33zm3wzn3sXPu4rhnShMKQHbGi8hmEWkkIh1E5AwR+Z9YJwKy87CILI57CCBT3/6h6yUReVlE6opIfxGZ6pw7LtbBUoQCkJ1jRWR6EARFQRBsEpG5InJ8zDMBGXHO/UxEvhKReXHPAmShjYg0FpE/BEFwIAiC+SLylohcGe9Y6UEByM5DIvIz51wN59zRInK+lJYAIBWcc4UicpeI3BT3LEAOORFpF/cQaUEByM6bUvon/p0i8oWILBGRF2OdCMjM3SLy5yAIvoh7ECBLH0npX8UOcc5Vdc6dI6V/HVsj3rHSgwKQIedcJSn90/4LIlJTROqJyBEi8rs45wLCcs51EJH/FpE/xD0LkK0gCPaLSE8RuVBENonIzSIyXUr/UIYQHE8DzIxzrp6IbBGROkEQ7Pg26yki9wRBwK0nJJ5z7gYR+a2I7Po2KhCRyiKyMgiCjrENBkTknHtbRJ4IgmBC3LOkAXcAMhQEwVYR+VREBjjnqjjn6ohIPxFZFu9kQGiPikgLKf0Jlg4i8icRmS0i58Y5FJAp51x751y1b/djDZbSn8x6POaxUoMCkJ1eInKelN4J+FhE9ovIjbFOBIQUBMHeIAg2ffePiOwWkaIgCLbEPRuQoStFZKOU7gXoJiLdgyAojnek9OCvAAAAMIg7AAAAGEQBAADAIAoAAAAGUQAAADCIAgAAgEGHeoQtPyKAKFzcAwhrGNEkYQ2LsI4RjXcdcwcAAACDKAAAABhEAQAAwCAKAAAABlEAAAAwiAIAAIBBFAAAAAyiAAAAYNChDgJCBfv8889V9sgjj6hs0aJFKluwYIHK6tSpo7Knn35aZeeff37ICQEA+YA7AAAAGEQBAADAIAoAAAAGUQAAADAo9ZsAS0pKVFapUny9Jgj0Q7sWLlyosqVLl3qvv+OOO1Tm+3esWbOmyu655x6VnXbaaSrr0KGD97MBAHZwBwAAAIMoAAAAGEQBAADAIAoAAAAGUQAAADAosT8FUFRUpLK1a9eqrF69eiqrW7duucz0Q+vWrVPZrFmzVHbbbbepbMeOHd739P10QEXs2n/33Xe9ecuWLVVWUb++AIDywx0AAAAMogAAAGAQBQAAAIMoAAAAGJTYTYBbtmxR2TPPPKOyESNG5PyzfRsQb7rpJpU98cQTKvMdBVxYWKiyrVu3ej+7du3aYUYMzTfPww8/rLIbbrjBe/0nn3yiMjYBAkD6cQcAAACDKAAAABhEAQAAwCAKAAAABjnfJrHvOegX88HixYtV1rNnT5Vt2rRJZb5fu169eqnsj3/8o8oaN24cdsTQdu/erbLx48er7NZbb1VZnTp1vO/50Ucfqax+/fphR3JhX1iOErWGfScrdu/eXWWPPPJIRYwT2qpVq1TWpk2bGCapcElYwyIJW8dR7N+/35v7NlUPHDhQZZdddpnKnnzyyeiD5TfvOuYOAAAABlEAAAAwiAIAAIBBFAAAAAxK7EmAUezZs0dljz32mPe1t99+u8p27dqlsuOOO05lkyZNUtkpp5yisqpVq3o/Owrf5rxrrrlGZX//+99V5tuAuHLlSu/n1KpVK4vpUJZKlXTndi4p+8xK7du3T2XXXXedyjZu3Oi9fvXq1TmfCcl34MABla1Zs0ZlXbt29V7/9ddfq6xhw4Yqu+eee7KYDj7cAQAAwCAKAAAABlEAAAAwiAIAAIBBqd8EuH79epWdf/75KluxYkXo9/Q9+te3WTDXj+4ti28jX//+/VX24YcfquzCCy9U2fTp01VWvXr1LKdDWXynR3755ZcxTJIZ34mSCxcuVFmTJk0qYhykxN133x0qK+vU0dGjR6vsl7/8pcp8J7Bu27ZNZa+99prK3nvvPZXdeOON3nkaNWrkzfMJdwAAADCIAgAAgEEUAAAADKIAAABgUOo3AfpO7Vu+fLnKyjptzfcYycsvv1xlvhPcoti8ebM3923Qu/nmm1VWt25dlT399NMqu+SSS1RWuXLlMCMionr16qmssLBQZb5TKkeOHKmyBg0a5GYwICLfSaQTJkwIde28efO8ue+01RdffFFlL7/8ssrKOuk1jOLiYm8+atQolR122GFZf04ScQcAAACDKAAAABhEAQAAwCAKAAAABqV+E2BYvtOjRPyP7831hr/FixerrHPnzqGv79Wrl8oefvhhlbFJLFl8JziuXbtWZb7HqJaUlJTLTEAu+E74K2tj8w/deeed3vyLL75Q2dKlS1XWo0cPlf3pT39Sme977IgRI1Q2btw47zy+DdSnnXaa97VpxR0AAAAMogAAAGAQBQAAAIMoAAAAGEQBAADAoNT/FMDRRx+tsilTpqisb9++3uvbtm2rsj59+qjMt3u0ShX9y3f//ferzLdD1XckrIjIoEGDVHbXXXeprKyjjZEce/fuVZlvx3/z5s1VVrNmzXKZCciFoUOHquzVV19V2datW1VW1k9ZXX311SqbNm2aylq2bBlmRK/GjRuHfu3rr7+uMn4KAAAApB4FAAAAgygAAAAYRAEAAMCg1G8CrFWrlsouv/xylfk2X4mI/PrXv1bZk08+GSrzHS/s25zn2yy4YMEC7zwdOnTw5ki27du3q2zw4MGhrm3VqpXK4nzu+MKFC2P7bKRD+/btVfbhhx+qrKioSGUNGzb0vmflypWjD/Y9xcXFKvv0009Vduyxx3qv9/3ekG+4AwAAgEEUAAAADKIAAABgEAUAAACDUr8J0Md30lRZJwH6Nuj5TgIMy7cxcP78+Spjs19+8W2cW7FiRahrV65cqTLfBqZq1aplPlgW8u20M1SMI488Mu4R/sPu3btV9te//lVlvtNkRcreOJ5PuAMAAIBBFAAAAAyiAAAAYBAFAAAAg/JyE6BvI96iRYu8r/3Vr36lMt9pfnXq1FHZrl27VObbODJx4kSV/dd//Zd3HqST7xS0sC644AKV1a5dO8o4FaJ3795xjwBE1rFjR2+etE2N5YE7AAAAGEQBAADAIAoAAAAGUQAAADDI+TbMfc9Bv5gE+/btU9mwYcNUNnbsWO/11atXV9mf//xnlV166aUqmzt3rsp+8pOfqKxq1aoq882dh/RuyopXIWu4QYMGKtuyZUuoay+++GKV+darb62KiBxxxBGhPics39y+f7+jjjrKe/2aNWtU5ntsd0okYQ2LpOB7cdJs27ZNZb41u3jxYu/1ZW0OTCnvOuYOAAAABlEAAAAwiAIAAIBBFAAAAAxK1UmARUVFKvOdRubbnFfWBirfhqVGjRqFmsd3glthYaHKfI+l3Lp1q/c969WrF+qzkT9mzJihsrfeektlvrUlItK+fXuV+U649D12uHv37ip79NFHvZ/zQ5s3b/bmPXv2VNm8efNCvSeQK/fff7/KSkpKVOZ7JLwV3AEAAMAgCgAAAAZRAAAAMIgCAACAQana/dC/f3+VvfLKKyrr0qWLyp5++mnve4bd8OezY8cOldWvX19lvlP/2OyXX+677z6VDRgwQGX79+8P9X6+DXZlbbr7+OOPVfbCCy+E+pwofCdciohceeWV5f7ZwPf5TrT1fd+tVIk/834fvxoAABhEAQAAwCAKAAAABlEAAAAwiAIAAIBBzrd78ntiewb1unXrVNaiRQuVtWrVSmWLFi1SWUFBQaR59u7dq7I777xTZaNGjVLZz3/+c5VNnTo10jwpkYRnqce2hpcvX66y4cOHq8y3u//9999XmW9Xs4h/B7Rz+pe+Ro0aKmvevLnKfHP7NGnSxJt/9tlnoa5PiSSsYZEY13Ea+I5br127tsp8v4csWbLE+55lHb2dUt51zB0AAAAMogAAAGAQBQAAAIMoAAAAGJTYo4C/+eabUJlvY5PviN5MNgH6rh80aJDKfBv5rrjiCpVNmjQp9Gcjf7Rr105lM2bMCHWtbyPerl27Is1Tp04dlR177LEqu/jii1XmO3IbSArf92wf39HvebbZLyPcAQAAwCAKAAAABlEAAAAwiAIAAIBBid0E2KxZM5XdcsstKvvd736nMt/pZm3btvV+zsaNG1Xm21BSXFyssnPOOUdlDzzwgMoOP/xw72cDZfFtIKwovv/2gCSbPHlyqNddc8015TxJunAHAAAAgygAAAAYRAEAAMAgCgAAAAYldhOg73Gmt912m8ratGmjsg8++EBlY8aMiTTP5ZdfrrJx48ap7Igjjoj0OUBasNaRNq1bt457hEThDgAAAAZRAAAAMIgCAACAQRQAAAAMSuwmQB/fI3379esX6toHH3ww1+MAeemCCy5Q2YQJE1TWsWPHihgHQDnhDgAAAAZRAAAAMIgCAACAQRQAAAAMStUmQADlb86cOaFeN2/evHKeBMhey5YtVcbG1f/EHQAAAAyiAAAAYBAFAAAAgygAAAAYxCZAAP+hcuXKcY8ARHb44Yer7LDDDothkuTiDgAAAAZRAAAAMIgCAACAQRQAAAAMogAAAGAQPwUA4D+MHDlSZePHj49hEgDliTsAAAAYRAEAAMAgCgAAAAZRAAAAMMgFQXCwrx/0i8AhuLgHENYwoknCGhZhHSMa7zrmDgAAAAZRAAAAMIgCAACAQRQAAAAMogAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGHeokQAAAkIe4AwAAgEEUAAAADKIAAABgEAUAAACDKAAAABhEAQAAwCAKAAAABlEAAAAwiAIAAIBBFAAAAAyiAAAAYBAFAAAAgygAAAAYRAEAAMAgCkAWnHNTnXMbnXM7nXOrnXPXxj0TkAnWMPKBc66Zc26Oc+5L59wm59w451yVuOdKCxcEQdwzpI5z7ngR+TgIgmLnXBsRWSAiFwZB8F68kwHhsIaRD5xzc0Rks4hcLyJ1RORVEZkYBMEfYx0sJbgDkIUgCFYEQVD83f/99p8WMY4EZIQ1jDxxrIhMD4KgKAiCTSIyV0SOj3mm1KAAZMk5N945t1dEVonIRhGZE/NIQEZYw8gDD4nIz5xzNZxzR4vI+VJaAhACBSBLQRD8j4jUEpHTROQFESk++BVAsrCGkQfelNI/8e8UkS9EZImIvBjrRClCAYggCIIDQRD8TUR+JCID4p4HyBRrGGnlnKskpX/af0FEaopIPRE5QkR+F+dcaUIByI0qwt+fIt1Yw0ibuiLSRETGBUFQHATBNhF5TEQuiHes9KAAZMg5d5Rz7mfOuQLnXGXn3LkicrmIzIt7NiAM1jDyQRAEW0XkUxEZ4Jyr4pyrIyL9RGRZvJOlBz8GmCHnXH0ReU5ETpTSAvW5iPwxCIKJsQ4GhMQaRr5wznWQ0o2AJ4rIARGZLyKDgiD4d6yDpQQFAAAAg/grAAAADKIAAABgEAUAAACDKAAAABh0qKcmsUMQUbi4BxDWMKJJwhoWYR0jGu865g4AAAAGUQAAADCIAgAAgEEUAAAADKIAAABgEAUAAACDKAAAABhEAQAAwCAKAAAABlEAAAAwiAIAAIBBFAAAAAyiAAAAYBAFAAAAgygAAAAYRAEAAMAgCgAAAAZRAAAAMIgCAACAQRQAAAAMogAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGUQAAADCIAgAAgEEUAAAADKIAAABgEAUAAACDqsQ9AAAAubZ582aVNW3aVGXnnHOO9/opU6aorLCwMPpgCcIdAAAADKIAAABgEAUAAACDKAAAABjEJkAgD02bNk1l11xzjcoGDhwY6v0ef/xxlZ155pne1w4YMEBl3bp1C/U5QK7MmTNHZV9//bXKZs2a5b3et75ff/11ldWuXTvz4RKCOwAAABhEAQAAwCAKAAAABlEAAAAwiE2AQB6qVauWyqpVq6Yy55zKCgoKVOY7AW3u3Lnez/ZtqmrWrJnKBg0apDLfpkTfjMChbNu2LdTrRo4c6c3HjBmjsiuuuEJlM2fOVFmlSun4s3U6pgQAADlFAQAAwCAKAAAABlEAAAAwyAVBcLCvH/SLSbV+/XqVrVq1KoZJMlezZk2VnXrqqTFMkhNJ2L2VyjUcVXFxscp2796tsiOPPDLrz9iwYYM3f+aZZ1Q2ePDgUO85b948lZ111lmZDZZbSVjDIkbXcRSjR49W2dChQ1V24MAB7/XXXXedyiZNmqQy339X1atXDzNiRfKuY+4AAABgEAUAAACDKAAAABhEAQAAwKBEbALcuXOnykpKSlT2wQcfqGzs2LEqW7Fihco+/vjj0PP4PjvXJzv5PkNEpEaNGio799xzVXbSSSepbNiwYdEHy60kbKBi81QCPP/88yq79NJLVdahQweVLV26tFxmCikJa1iEdZyx3r17q2zGjBkqe/fdd73X+zb3nX322aE++6qrrlJZ2BMtW7du7c19GxgzwCZAAABQigIAAIBBFAAAAAyiAAAAYBAFAAAAg6qU1xv7jvQUEVm9erXK7rjjDpVt375dZRWxOz9uRUVFKnvppZdU9uGHH6qsTp06KhswYEBuBgMiqFy5cqjXde7cuZwngRXHHXecynw78ffu3eu93necb+3atVXm+ym2KVOmqOyEE05Q2cknn6yy66+/3jtPeciv3z0BAEAoFAAAAAyiAAAAYBAFAAAAg3JyFLBvs8XGjRu9r/VtcgurPDYBduzYUWVNmjRR2eLFi1V2yimnhPoM31GT69at874215sa9+/fn9P3y1ASjlHlCNUKtn79epUNHDhQZTNnzlTZ559/rrJjjjkmN4NlJwlrWIR1nLHRo0erzHecblnfx4uLi1XmO46+fv36KvNtNizr98QKwlHAAACgFAUAAACDKAAAABhEAQAAwKCcnAT4ySefqKyiTuhr0KCByqZOnRr6+vbt26usbt26KvP9O7Zo0SLUZwwZMkRlDz74YKhrM3Hrrbfm/D2BsqxatcqbX3vttSp7++23VTZo0CCVxbzhDwYtWrTIm/tO/Xv99ddVVqWK/m20d+/e0QerANwBAADAIAoAAAAGUQAAADCIAgAAgEE5OQmwR48eKpszZ07oIX70ox+p7Kc//anKRo0aFfo9K4LvxDPf4x03b96sMt+phiLhN0/26tVLZc8++2yoaytQEk5R4wS1DPlOj1ywYIHKhg8f7r3+vffeU9nNN9+sMt9jwKtVq3boAStWEtawCOs4Yxs2bFCZb+O279Q+EZFly5aprGXLltEHiwcnAQIAgFIUAAAADKIAAABgEAUAAACDcnIS4LRp01SWyaMPa9asqbLGjRtHmimKr7/+WmV33323yp577jmVbd26VWWZnIoY9rW+DVTAwRw4cEBl06dPV5lvba1Zs0Zl55xzjvdzli5dqrITTjghxIRAdnxre9iwYSrzfW8/6qijvO+Z4g1/oXEHAAAAgygAAAAYRAEAAMAgCgAAAAblZBNgQUGBylq1apWLty53ixcvVtmECRNU9sQTT1TEOHLkkUeqzLcpK+yjiIHv3HjjjSobN25cqGvHjBmjsoEDB3pfW1GPAge+8/LLL6ssk8fCW8V/qQAAGEQBAADAIAoAAAAGUQAAADAoJ48DToNPP/3Um5900kkq27Fjh8pyvbGpZ8+e3tx34mCbNm1y+tkVKAmPUs2bNZyJTZs2qax58+YqKyoqUtmdd96psptuukllvhM881AS1rCI0XXsU1xcrLK2bduqrF69eir78ssvVbZr1y7v52Rymm0K8DhgAABQigIAAIBBFAAAAAyiAAAAYBAFAAAAg8z8FMBHH33kzdu1a6eykpISleX6pwD279+f0/dLqCTsoM6bNZyJrVu3qmzIkCEqe/XVV1W2YcMGlVWpok8NHzFihPezfc9hr1y5sve1KZCENSxidB37vPLKKyo7//zzVfaXv/xFZb///e9VVq1aNe/nvPHGG1lMl1j8FAAAAChFAQAAwCAKAAAABlEAAAAwyMwmwC1btnjzs88+W2XLly9XWa43AZa1KbFly5Y5/ZyYJWEDVd6s4fLgO1Z11qxZKrv55ptVtm7dOu97Pvfccyrr1atXFtMlQhLWsAjr+H8NGDBAZTNmzFDZv/71L5W1aNFCZd27d/d+zpNPPpnFdInFJkAAAFCKAgAAgEEUAAAADKIAAABgkD7eK0/Vr1/fm/ft21dlkydPVplzeg/Ftm3bVLZ9+/ZQ8/g2H4qIXHjhhSq77777VFZYWKiyXG9URP47/PDDVXbJJZeobMGCBSobP3689z2LiooizwWIiKxcuVJlEyZMUNnIkSNVVqNGjVCfccIJJ2Q+WJ7gdwwAAAyiAAAAYBAFAAAAgygAAAAYZGYTYFl8j0j1ZT6rV69W2fz581Xmeyzlm2++6X3PSZMmhcqmTZumMt/mLSAXmjZtGvcIMGjPnj2hXtetWzeVffbZZyrznQh71llnZTxXvuAOAAAABlEAAAAwiAIAAIBBFAAAAAwyvwkwiuOOOy5U1q9fP5UVFBRE+uyrr75aZV999ZXKrr322kifA3u++eYblfk2nZa1hjt16pTzmYDv+B5h36hRI5X5HktdUlJSLjOlFXcAAAAwiAIAAIBBFAAAAAyiAAAAYBCbACtA9erVc/6evkeuDhgwQGVsAsTB+DZFDR06VGXvv/++ymbOnOl9z1atWkUfDCiD79Hsa9euVdnu3btDXXv88cfnZrAU4g4AAAAGUQAAADCIAgAAgEEUAAAADGITYEjLli1Tme/Rkn369FHZ5s2bVcaJVDgY3ybPvXv3el9bt27dUNevWrVKZRdffLHKPv/8c5X16NFDZaeffrp3HiBXateuHep1U6dOVdns2bNVdv3116usPDZppwV3AAAAMIgCAACAQRQAAAAMogAAAGAQBQAAAIOc79nK33PQLyaBbzf9zp07VeY7KlJE5K677lKZ79fkb3/7m8q2b98eZkSvsn4KoFKlcJ2sa9euKnv00UdVFvOxrPrczYqX+DXsc+DAAZWVtet+48aNKmvbtq3KfLuifc4880yVzZo1S2UFBQWh3i/lkrCGRVK6jsvDGWecoTLf92efpUuXquzEE0+MPFMKeNcxdwAAADCIAgAAgEEUAAAADKIAAABgUOo3AX711Vcqq1+/fqT39G3QC7s5L8pnZPI5zzzzjMp69+4daaZykIQNVIlfw2Hdc8893nzEiBGhru/UqZPKRo0apTLfBtNcr/8UScIaFsmjdYxYsAkQAACUogAAAGAQBQAAAIMoAAAAGJT6TYC+zXQPPPCAyoYPHx7pPaNsgurYsaPK+vbt631tnz59Qr1nrVq1VJbAjVpJ2ECV+DWMREvCGhZhHSMaNgECAIBSFAAAAAyiAAAAYBAFAAAAgw61CRAAAOQh7gAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGUQAAADCIAgAAgEEUAAAADKIAAABgEAUAAACDKAAAABhEAQAAwCAKAAAABlEAAAAwiAKQIefc7h/8c8A5NzbuuYBMOOd+5Zxb4pwrds49Hvc8QLacc62cc0XOualxz5I2VeIeIG2CICj47n875wpEZJOIPBvfREBWNojIPSJyrohUj3kWIIqHRWRx3EOkEXcAouktIptFZGHcgwCZCILghSAIXhSRbXHPAmTLOfczEflKRObFPUsaUQCi6SciTwZBEMQ9CABY4pwrFJG7ROSmuGdJKwpAlpxzTUXkDBF5Iu5ZAMCgu0Xkz0EQfBH3IGnFHoDsXSkifwuC4NO4BwEAS5xzHUTkv0Xkx3HPkmYUgOz1FZH74x4CAAw6U0Saicha55yISIGIVHbOtQ2CoGOMc6UKBSALzrn/KyJHC7v/kVLOuSpS+t9/ZSn9xllNRL4JguCbeCcDQnlURJ753v8fLKWFYEAs06QUewCy009EXgiCYFfcgwBZul1E9onIrSLS59v/fXusEwEhBUGwNwiCTd/9IyK7RaQoCIItcc+WJo4N7AAA2MMdAAAADKIAAABgEAUAAACDKAAAABh0qB8DZIcgonBxDyCsYUSThDUswjpGNN51zB0AAAAMogAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGUQAAADCIAgAAgEEUAAAADKIAAABgEAUAAACDKAAAABhEAQAAwCAKAAAABlEAAAAwiAIAAIBBFAAAAAyiAAAAYBAFAAAAgygAAAAYRAEAAMAgCgAAAAZRAAAAMIgCAACAQRQAAAAMogAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGVYl7AAAA3nrrLZVNnz491LX33nuvymrWrBl5pnzHHQAAAAyiAAAAYBAFAAAAgygAAAAY5IIgONjXD/pF60pKSlS2e/fuSO85fvz4SNf/0LBhw1RWqVL43nfGGWeobP78+WEvd6E/qPywhnNg2bJlKrvuuuu8r92zZ0+o61MiCWtYJI/Wse/7pohI//79VTZ58uRQ77lp0yaVHXXUUZkNlt+865g7AAAAGEQBAADAIAoAAAAGUQAAADAosScBLl++XGVRNsgNHjzYm8+ePVtlK1euDPWevs1OTz31VGaD/YBvU6Zz2e9D8m34y+T9mjZtmvVnI3889NBDKlu0aFHo69955x2VnXrqqZFmQjrt3bvXm4fd8Fe3bl2VHXbYYSobOnSoyoqLi1U2ZsyYUJ+bj7gDAACAQRQAAAAMogAAAGAQBQAAAIMSuwnwxBNPVFmUzXATJkzw5lHeM9cb9kRErr/++kjX/5DvEZu+DZZl6d27dy7HQQr4TlV77bXXIr1nYWFhpOuRP7CM7OYAAAeuSURBVF5++eVI199www0qW7t2rcp8m/t8pxD26dPH+zmdOnXKYrp04Q4AAAAGUQAAADCIAgAAgEEUAAAADErs44C3bNmissceeyzUtffff7/KduzY4X1tkyZNVDZgwIBQn+P7tTvmmGNUdtFFF4V6P5Hcb5bybeJ76aWXVNa+fXvv9W+//bbKqlWrFvbjk/Ao1bx5jGpF8a1/3ybaRo0aea/3ra+TTz45+mDxSMIaFknpOvZtujv66KO9r/33v/+tsq5du6ps3rx5KnvzzTdV1r179zAjyplnnunNM3jseRrwOGAAAFCKAgAAgEEUAAAADKIAAABgEAUAAACDEnsUcP369VXme76zT9jX5ZvVq1erbMaMGSrzHVc8cuRI73tmsOMfKbRhwwaVTZkyJdS1v/nNb7x5inf8I8d8Pynl2+1fltq1a6usatWqkWb6ob59++b0/dKEOwAAABhEAQAAwCAKAAAABlEAAAAwKLGbAHFw+/btU9mQIUNU5tvw59tY07lz59wMhlQ599xzVbZ3716V+TblXnbZZeUyE1CR6tSpE/cIseEOAAAABlEAAAAwiAIAAIBBFAAAAAxiE2BKvf/++yqbPXt2qGsnTpyosoYNG0aeCcnme2b6ihUrQl07fPhwlTVt2jTyTMhvy5cvj3R9rVq1cjQJfLgDAACAQRQAAAAMogAAAGAQBQAAAIPYBJhS77zzTqjXFRYWqqxLly65HgcJs3PnTpVdeumloa71PQK6R48ekWeCPbNmzYp0/eDBg3M0CXy4AwAAgEEUAAAADKIAAABgEAUAAACD2ASYAhs3blTZQw89pLIgCFTWr18/lTVu3Dg3gyGxxowZo7ItW7aEuvYXv/iFyjj1D4eya9culd13332hrz/vvPNU1rp160gz4eC4AwAAgEEUAAAADKIAAABgEAUAAACD2ASYMAcOHFDZDTfcoLL169erzDmnsttvvz03gyGxduzYobLRo0eHurZz584q+8Mf/hB5Jtjz6quvqmzfvn2hrx8yZIjKatasGWmmMN577z1vfswxx6jsiCOOUFnz5s1zPlNF4Q4AAAAGUQAAADCIAgAAgEEUAAAADHK+0+O+56BfRO6tWrVKZe3atQt1bfv27VW2dOnSyDNFoHclVry8X8Nz5sxR2UUXXRTqWt/66NChQ+SZ8kgS1rBIwtbxl19+qbLTTz9dZStWrFBZ9erVve+5bNkylbVo0SLUPL7HVc+ePTvUtZno3bu3yp599tmcf0458K5j7gAAAGAQBQAAAIMoAAAAGEQBAADAIAoAAAAGcRRwwnTp0iXU63w7/hcsWJDjaZAkRUVF3nzkyJGhrvftvmbHP7JxxRVXqMy349+nuLjYm/t+iiCszZs3Z32tZdwBAADAIAoAAAAGUQAAADCIAgAAgEFsAqwA+/fvV9m1117rfa3v2e7VqlVT2b333quywsLCLKZDWkyfPt2bl/U88x8aPXp0LseBEWPHjlXZ3Llzs36/kpISb75x48as3xPZ4Q4AAAAGUQAAADCIAgAAgEEUAAAADGITYAVYsmSJyp566inva53Tj22+8sorVXbeeedFHwyp8tVXX4V+bbt27VTWv3//XI4DI/7xj3/E9tlBEKjM9z0yLN+1zZo18762b9++Krvwwguz/uwk4g4AAAAGUQAAADCIAgAAgEEUAAAADHK+TRbfc9AvQtu3b5/K+vTpo7KXXnrJe73vka3Lli1T2bHHHpvFdBUu+906uZPKNbxmzRqVdevWzfvaL774QmVXXXWVyiZPnhx5LoOSsIZFYlzHn3zyicp8jyP3fe/r2rWryn77299Gmuedd95R2S233BLq2oKCApXt3Lkz0jwp4V3H3AEAAMAgCgAAAAZRAAAAMIgCAACAQZwEmGOvvPKKysra8OezcOFClaVkwx9yaOLEiSrzbfYri+/0SCAbLVq0UNk///lPle3Zs0dlvhMpo5zkJyLy/PPPR7oe/x93AAAAMIgCAACAQRQAAAAMogAAAGAQmwAjKCkpUdmMGTNU5jttsbCw0PueHTp0iD4YUq+4uDj0a30brXwnsAG50rx58wr5nPXr16vs8ccfr5DPtoA7AAAAGEQBAADAIAoAAAAGUQAAADCITYARrFu3TmVPPfWUynwnXz388MPlMhPyw9ixY0O/tlOnTiqrWrVqLscBYtGgQQOV/fjHP1bZG2+8Eer9zjjjjMgz5RPuAAAAYBAFAAAAgygAAAAYRAEAAMAgCgAAAAbxUwARTJs2LdTrLrnkEpVddtlluR4HKfXmm2+qzHd8dFnPUe/SpUvOZwKSoEoV/VvUeeedpzLfTwHccccdKhs2bFhO5soX3AEAAMAgCgAAAAZRAAAAMIgCAACAQc632eh7DvpF65YsWaKy+fPnq6xv374qa9iwYbnMlDD+XWsVK/FreM2aNSpr3bq1yrp16+a9fubMmSqrXr169MEgkow1LJKCdYxE865j7gAAAGAQBQAAAIMoAAAAGEQBAADAIDYBojwlYQMVaxhRJGENi7COEQ2bAAEAQCkKAAAABlEAAAAwiAIAAIBBh9oECAAA8hB3AAAAMIgCAACAQRQAAAAMogAAAGAQBQAAAIMoAAAAGPT/AIkUnnFV5wxqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x648 with 9 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xb, yb = dls.one_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([128, 1, 28, 28]), torch.Size([128]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb.shape, yb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dls.c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic CNN with batchnorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here we can see our ni is equivalent to the depth of the filter, and nf is equivalent to how many filters we will be using. (Fun fact this always has to be divisible by the size of our image).\n",
    "def conv(ni, nf): return nn.Conv2d(ni, nf, kernel_size=3, stride=2, padding=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bn(nf): return nn.BatchNorm2d(nf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ReLU(): return nn.ReLU(inplace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Flatten(full=False)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Flatten `x` to a single dimension, e.g. at end of a model. `full` for rank-1 tensor\n",
    "Flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(\n",
    "    conv(1, 8), # 14\n",
    "    bn(8),\n",
    "    ReLU(),\n",
    "    conv(8, 16), # 7\n",
    "    bn(16),\n",
    "    ReLU(), \n",
    "    conv(16, 32), # 4\n",
    "    bn(32),\n",
    "    ReLU(),\n",
    "    conv(32, 16), # 2\n",
    "    bn(16),\n",
    "    ReLU(),\n",
    "    conv(16, 10), # 1\n",
    "    bn(10),\n",
    "    Flatten() # remove(1, 1) grid   \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls, model, loss_func=CrossEntropyLossFlat(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Sequential (Input shape: 128)\n",
       "============================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "============================================================================\n",
       "                     128 x 8 x 14 x 14   \n",
       "Conv2d                                    80         True      \n",
       "BatchNorm2d                               16         True      \n",
       "ReLU                                                           \n",
       "____________________________________________________________________________\n",
       "                     128 x 16 x 7 x 7    \n",
       "Conv2d                                    1168       True      \n",
       "BatchNorm2d                               32         True      \n",
       "ReLU                                                           \n",
       "____________________________________________________________________________\n",
       "                     128 x 32 x 4 x 4    \n",
       "Conv2d                                    4640       True      \n",
       "BatchNorm2d                               64         True      \n",
       "ReLU                                                           \n",
       "____________________________________________________________________________\n",
       "                     128 x 16 x 2 x 2    \n",
       "Conv2d                                    4624       True      \n",
       "BatchNorm2d                               32         True      \n",
       "ReLU                                                           \n",
       "____________________________________________________________________________\n",
       "                     128 x 10 x 1 x 1    \n",
       "Conv2d                                    1450       True      \n",
       "BatchNorm2d                               20         True      \n",
       "____________________________________________________________________________\n",
       "                     []                  \n",
       "Flatten                                                        \n",
       "____________________________________________________________________________\n",
       "\n",
       "Total params: 12,126\n",
       "Total trainable params: 12,126\n",
       "Total non-trainable params: 0\n",
       "\n",
       "Optimizer used: <function Adam at 0x7f9410ffdcb0>\n",
       "Loss function: FlattenedLoss of CrossEntropyLoss()\n",
       "\n",
       "Callbacks:\n",
       "  - TrainEvalCallback\n",
       "  - Recorder\n",
       "  - ProgressCallback"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 10])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb = xb.cuda()\n",
    "model(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(valley=0.0063095735386013985)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xV9f3H8dcng4RMRsIOhr1HJLioqJW6RbQCKtVqadVKq1Zra7U/R+uqtbgVsFbUotQirrq1iANFArJB9ggzrBCyx/f3x70gjUlISG7Ovbnv5+NxH+See+6971wg75z1/ZpzDhERCV8RXgcQERFvqQhERMKcikBEJMypCEREwpyKQEQkzKkIRETCXJTXAeoqJSXFpaenex1DRCSkzJ8/f5dzLrWqx0KuCNLT08nKyvI6hohISDGzjdU9pl1DIiJhTkUgIhLmVAQiImEu5I4RiIhUp7S0lOzsbIqKiryO4pnY2Fg6depEdHR0rZ+jIhCRJiM7O5vExETS09MxM6/jNDrnHLt37yY7O5suXbrU+nnaNSQiTUZRURGtW7cOyxIAMDNat25d5y0ibRFUoai0nJy8YnbsL2J/USmJsdEkN48mJSGGVvHNvI4nIjUI1xI46Gi+fxUBsHRLLh8u38GyrftZvjWXrblVt2mEwSOXZDByUIdGTigiTVVCQgIHDhxgw4YNnHfeeSxdurTRMwSsCMwsDXgBaAs4YIpz7tFK6yQD/wQ6+7M85Jx7LlCZqrJ0Sy4/fnoOJeUVdE2JJzO9FT3aJNA2KZY2STEkN4/mQHEZ+wpKmfzpWu59ezkj+rQhrpk6VCTkLX4FPv4T5GZDcic4/Q4YOMbrVI0ukD/NyoCbnXMLzCwRmG9mHzrnlh+2zgRguXPufDNLBb41s2nOuZIA5jpkT34J17w4n1bxzXhjwjDaJMXWuH775FgunvQlz3y6nhtG9KhynaLScnYdKKZTy7hARBaRhrL4FXjreigt9N3P3ey7D/Uqg1tvvZW0tDQmTJgAwF133UVUVBSzZs1i7969lJaWcs8993DBBRdU+xrl5eXceuutfPLJJxQXFzNhwgSuueYarrjiCi666CJGjRoFwLhx4xgzZkyNr1UbATtY7Jzb5pxb4P86D1gBdKy8GpBovp1aCcAefAUScGXlFfz65QXkHChm0k+GHLEEADLTW3F2/3ZM/nQtO/d/t/uovMLx+epd/Pbfixh6z0cMf3AWHy3fEcj4IlJfH//puxI4qLTQt7wexo4dyyuvvHLo/iuvvMJPf/pTXnvtNRYsWMCsWbO4+eabqWma4GeffZbk5GTmzZvHvHnzeOaZZ1i/fj3jx49n6tSpAOTm5jJnzhzOPffceuWFRjpryMzSgQxgbqWHngD6AFuBJcANzrmKKp5/tZllmVlWTk5Og2T66wff8sWa3dwzqj+D0lrU+nm3nt2b0vIKJn64CuccHyzbzo8mzuYnz87l/aXbOat/O/p2SOLXL3/Dos37GiSriARAbnbdltdSRkYGO3fuZOvWrSxatIiWLVvSrl07brvtNgYOHMiIESPYsmULO3ZU/8viBx98wAsvvMDgwYM5/vjj2b17N6tXr+aUU05h9erV5OTk8PLLL/PjH/+YqKj679gJ+I5uM0sAXgVudM7tr/TwmcBC4IdAN+BDM/us8nrOuSnAFIDMzMzqa7SWtuwrZPLsdVx6XGfGZKbV6bnHtI7nihPTee6L9Xy7I49vNu2jW2o8j1+awY/6tiU2OpKdeUVc9NQcxj8/j9euG0ZaK+0mEgk6yZ18u4OqWl5Po0ePZsaMGWzfvp2xY8cybdo0cnJymD9/PtHR0aSnp9d4iqdzjscff5wzzzzze49dccUV/POf/2T69Ok891zDHFIN6BaBmUXjK4FpzrmZVaxyFTDT+awB1gO9A5kJYP7GvQCMO77zUT3/1z/sTou4ZmzeU8A9o/rz/o3DOX9QB2KjIwFokxjL1KuGUlru+OlzX5NbUNpg2UWkgZx+B0Q3/99l0c19y+tp7NixTJ8+nRkzZjB69Ghyc3Np06YN0dHRzJo1i40bqx0IFIAzzzyTp59+mtJS38+OVatWkZ+fD8CVV17JI488AkDfvn3rnRUCe9aQAc8CK5xzE6tZbRNwOvCZmbUFegHrApXpoG827SU2OoJe7RKP6vkt4prx4W+G07xZZLVnD3Vvk8iUy4cw7u9zufPNpTxySUZ9IgNQUlZBUVk5SbG1v3RcRKpx8IBwAM4a6tevH3l5eXTs2JH27dszbtw4zj//fAYMGEBmZia9e9f8++7Pf/5zNmzYwLHHHotzjtTUVF5//XUA2rZtS58+fQ4dMG4IVtMBi3q9sNkPgM/w7fs/uN//NnyniuKcm2RmHYCpQHvAgAecc/+s6XUzMzNdfecjGPXkFzSLjOCVa0+s1+vUxiMfreKRj1Yz6SdDOKt/u6N+na/W7eY3/1rIttwiOiTH0rNdIsd2bskVJx5Dizhd5CYCsGLFCvr06eN1jIAqKChgwIABLFiwgOTk5CrXqepzMLP5zrnMqtYP2BaBc+5zfD/ca1pnK3BGoDJUpbisnOVb93PVsPRGeb8Jp3XnoxU7uP21JQxNb0nrhJg6Pb+svIJHP17NE7PWkN46nt+e0ZM1Ow+wcnseD3+0imc+XcfPftCF8Sd3qXJLYdPugkPXSEREfP+vwznH4uxc3lm6jU4tmnNS9xS6psRjZuwrKGHl9jzKKxxDjml5aNeXiHjjo48+Yvz48fzmN7+ptgSORthdFbVs635KyivI6Fz7M4XqIzoygr+NHsz5j3/OH19fylPjjq31JeA79hcxYdoCsjbuZfSQTtw1sh/xMd/9lX27PY+HP1zFox+v5h9frGd4z1R+0D2FzGNaMm/DXl5dkH3oeEhCTBT9OybRq20irf1DZRSXVTBjfjYrtu0nwqDCv3HYLikWM9h22BXWzaIiOC69FcO6p5DRuQUDOib/TxYRCbwRI0Yc8fjC0Qi7/8nfbPKd0pnRuWWjvWevdon85kc9+ct7K/n581mktYojNTGGwWktOKlb1QNkLdi0l2tfnM+B4jIevWQwFwyufAmG73UnXT6EpVtymTpnA5+tzuHtxdsOPd6jTQK3nt2b1vHNWJydy+ItucxcsIW84u8u1ejXIYl7RvVn5OAO7DlQwpy1u/lq3W4iDPq0T6JXu0Scg8/X7OLz1bv4y3srAd9wGz3aJNKjbQLdUhPomhpPcWkFG/fks3F3AbsOFFNUWkFRaTkRZvRom0Dvdkn0bpdI59ZxdGzR/IhbGPnFZcz6didz1+2hXXIsvdom0rNtIh1axBIVqfESRRpKGBbBXjokx9K2FheQNaSrh3dl0558vl6/h6/X7zn0w7h/xyR+eUr3Q8cPducX8+HyHdz95nLaJcfy4vjjj3hQu3/HZB4aPQjnHGtzDpC1YS99OyQxoGPyoZIZfdhpssVl5ewrKKWkrOJ/Tm1Nio0mPSWey6o4m+q03m0A39XYizbvY+HmfSzO3sei7H28vWQbBw81RUUYHVs2p21iLImxUaQmxlBaXkHWhr28sXDr/7xm26QYWjRvRmyzSOKiI4lrFklcTBQJMZHsPlDC7FU5FJdVENcskoKS8kPPM4OUhBjaJsXQPrk5aS3jSGvVnLZJsURHRhAVaUSYkV9cRl5RKXlFZZRVOCqcwzkoKCkjt7CU/YVllFVUkOQfVDA5LppWcc1oFd+M1gkx9OuQpN1hIcg5F9YDzx3Ncd+AHSwOlPoeLB72wH8ZnNaCJ8cd24Cp6i6/uIz/LN7KpNnrWL8rn4SYKApKyg7tnjm5RwqPX5oREgeCi0rL2bi7gObRkTX+tp5bUMqqnXls3lPA5j2FZO8tIK+ojMLScgpLyskvKSO/uIwDxeXEREUwok8bzhnQnsz0VuSXlLF6xwHW7Mxj674iduwvYvv+IrbuK2TznkIKS8urfM+qREYYSbFRJDePJioygtzCUnILfcV4uNjoCE7qlsJpvVI5tVcbXQ8SAtavX09iYmLYDkV9cD6CvLy8781HUNPB4rAqgp37izjuvo/547l9+PnJXRs42dEpr3C8t3Q7X63bTcu4aFITY+jQojmn9mpDZBUHd+X7nHPszi8hJ6+YsnJHaUUFFRWOhNgoEmOjSYiJollkBGa+rQnf19//bAtLytlTUMKeAyVs31/EF2t2MevbnWzcXQBAl5R4hvdI4az+7Tmha6uw/EET7DRDWfUzlKkI/N5ftp1rXpzPq788kSHHtGrgZNIUOedYtyufT1fl8OmqHL5ct5ui0gpO6taa353Vm8F1GJ5ExEuenD4ajL7ZtI/oSKNfh4Y77UqaNjOjW6rvgPhVw7pQVFrOy19v4on/rmHUk19wVr923HJWL7qlJngdVeSohVkR7KVvex0AlKMXGx3JVcO6MDozjWc/W8+UT9fy4YodjMlM48YRPRr9JASRhhA25+CVlVewODu3UU8blaYrISaKG0b0YPbvTuPyE45hxvzNnPLXWTz1yRpKy783gK5IUAubIvh2Rx6FpeWNdiGZhIeUhBjuGtmPj246hVN6pvLge99y/uOfs1BDkEsICZsiWLPzAGaQkaYtAml4x7SOZ/LlmUy+fAj7Ckq58KkveODdlZRp60BCQFidNZRXVEpCTJRO+5OAyisq5b53VvDy15s5sWtrHr8sg5Q6jjEl0tBqOmsobLYIABJjo1UCEnCJsdHcf9FAHho9iAWb9nLeY58fGvNJJBiFVRGINKaLh3Ri5nUnER1ljJ40h/vfWUFRHa6AFmksKgKRAOrXIZm3rz+ZsUPTmPzpOs5+9DPmbdjjdSyR/6EiEAmwJP+uomk/P57S8grGTv6S5+ds8DqWyCEqApFGMqx7Cu/fOJzT+7TlzjeXcfdbyyivCK2TNaRpUhGINKL4mCgm/WQIPxvWhee+2MA1/jknRLykIhBpZJERxh3n9+Xukf3478odnPfYZyzSBWjiIRWBiEd+elI6068+kZKyCn789BwmzV5LhXYViQdUBCIeOq5LK969YThn9GvLA++u5FcvL/jeBDkigaYiEPFYclw0T152LLef04d3lmznmhezdL2BNCoVgUgQMDN+Mbwr9104gE9W5fCzqfPI10FkaSQqApEgctnxnZk4ZhBfrdvNz6bOo7hMWwYSeCoCkSBzYUYnHh47mLnr9/C7GYsJtYEhJfSE1QxlIqHigsEdyd5byF/f/5bOreK4+YxeXkeSJkxFIBKkrju1G5v3FPD4f9eQ1jKOMUPTvI4kTVTAdg2ZWZqZzTKz5Wa2zMxuqGa9U81soX+d2YHKIxJqzIw/j+rPyT1SuHXmYqbN3eh1JGmiAnmMoAy42TnXFzgBmGBmfQ9fwcxaAE8BI51z/YDRAcwjEnKiIyOY9JMhnNIzldtfW8pD73+rYwbS4AJWBM65bc65Bf6v84AVQMdKq10GzHTObfKvtzNQeURCVXxMFM9ckcmlx6XxxKw13PzvRZRqCkxpQI1yjMDM0oEMYG6lh3oC0Wb2CZAIPOqce6GK518NXA3QuXPnQEYVCUpRkRHcd+EA2ic3Z+KHq4g048GLB2rGPWkQAS8CM0sAXgVudM7tr+L9hwCnA82BL83sK+fcqsNXcs5NAaaAb87iQGcWCUZmxvWn96CswvHYx6vp2LI5N47o6XUsaQICWgRmFo2vBKY552ZWsUo2sNs5lw/km9mnwCBgVRXrigjwmxE92LqvkEc+Wk2H5OY6m0jqLZBnDRnwLLDCOTexmtXeAH5gZlFmFgccj+9YgohUw8y4/6IBnNwjhT+8toRPvtWhNamfQJ41NAy4HPih//TQhWZ2jplda2bXAjjnVgDvAYuBr4G/O+eWBjCTSJMQHRnBU+OOpVfbRK6btkDzGUi9WKidipaZmemysrK8jiESFHbuL+Kip+dQWFLOq788ifSUeK8jSZAys/nOucyqHtNYQyIhrE1SLC/87DgqnOOKf3xNTl6x15EkBKkIREJc19QE/nHlUHLyirlq6teaA1nqTEUg0gRkdG7Jk+MyWLEtj2tezNLw1VInKgKRJuKHvdvylx8P5Is1u7nplUWa/1hqTaOPijQhFw/pxO4Dxdz/7kpSE2K48/y+uvpYjkhFINLEXD28Kzvzinn28/V0TY3nihPTvY4kQU67hkSaGDPjtnP6cHrvNtz91nK+WLPL60gS5FQEIk1QZITxyCWD6ZYaz3XTFrB+V77XkSSIqQhEmqjE2Gj+fsVQIgzGPz+P3IJSryNJkFIRiDRhnVvH8fRPhpC9p5Dxz8+jsESnlcr3qQhEmrgTurbmkUsGM3/TXl6Y/CDu4X5wVwt4uD8sfsXreBIEdNaQSBg4Z0B7/jl0ExmLJmJW4luYuxneut739cAx3oUTz2mLQCRMDNv4JHEHS+Cg0kL4+E/eBJKgoSIQCRe52XVbLmFDRSASLpI71W25hA0VgUi4OP0OiG7+P4uKLQZ3+h0eBZJgoSIQCRcDx8D5j0FyGmDkxbbnluLxTM0b6nUy8ZjOGhIJJwPHHDpDKME5Cl7I4t63V9C7XRIndmvtcTjxirYIRMKUmfHw2MGkp8Tzy2nz2bS7wOtI4hEVgUgY8w1DkYlz8PMX5pFXpGEowpGKQCTMpafE89S4Y1mbk8+N0xdSVl7hdSRpZCoCEWFY9xTuGtmPj1fu5JYZizW7WZjRwWIRAeDyE44ht6CEhz5YRVyzSO4Z1V+zm4UJFYGIHDLhtO4cKC5n0uy1JMREcevZvVUGYUBFICKHmBm/P6sX+cVlTP50HV1T4xk7tLPXsSTAdIxARP6HmXHXyH4M696aO95YxvKt+72OJAEWsCIwszQzm2Vmy81smZndUMO6Q82szMwuDlQeEam9yAjjkbEZJDePZsJLC3RaaRMXyC2CMuBm51xf4ARggpn1rbySmUUCfwE+CGAWEamj1MQYHr80g017Crj11SU4pzOJmqqAFYFzbptzboH/6zxgBdCxilV/DbwK7AxUFhE5Osd3bc1vz+jF20u28dwXG7yOIwHSKMcIzCwdyADmVlreEbgQeLoxcohI3V0zvCsj+rTlvndWMHfdbq/jSAAEvAjMLAHfb/w3OucqH3V6BPi9c67GSxnN7GozyzKzrJycnEBFFZEqREQYE8cOonOrOCa89A3bc4u8jiQNLKBFYGbR+EpgmnNuZhWrZALTzWwDcDHwlJmNqrySc26Kcy7TOZeZmpoayMgiUoWk2GgmXz6EwpIyfjltPsVl5V5HkgYUyLOGDHgWWOGcm1jVOs65Ls65dOdcOjADuM4593qgMonI0evRNpGHRg/im037uP21pTp43IQE8oKyYcDlwBIzW+hfdhvQGcA5NymA7y0iAXD2gPbccHoPHv14NSkJMdx6dm+vI0kDCFgROOc+B2p9bbpz7spAZRGRhnPjiB7syS9h0uy1tIqP5urh3byOJPWkISZEpE4OXnm8t6CE+95ZScu4ZozOTPM6ltSDikBE6iwywpg4ZjC5haXc9toSerVLZGCnFl7HkqOksYZE5Kg0i4rg8UszSE2I4VcvfcN+DUMRslQEInLUWsQ14/HLMtiyr5A/zNQwFKFKRSAi9TLkmFbcfEZP3l68jZe/3ux1HDkKKgIRqbdrh3fj5B4p3P3WMlZs07DVoUZFICL1FhFhPDx2MEnNo/n1y99QUFLmdSSpAxWBiDSIlIQYHhk7mLU5B7jrzWVex5E6UBGISIMZ1j2FCad255WsbN5YuMXrOFJLKgIRaVA3juhB5jEtuW3mEjbsyvc6jtRCrYrAzOLNLML/dU8zG+kfWVRE5H9ERUbw6KUZREVGcMP0bygtr3GUeQkCtd0i+BSI9U8k8wG+weSmBiqUiIS2ji2ac9+FA1iUncsjH63yOo4cQW2LwJxzBcBFwFPOudFAv8DFEpFQd+7A9owe0omnPlmrmc2CXK2LwMxOBMYBb/uXRQYmkog0FXeN7McxreL4zb8WklugISiCVW2L4EbgD8BrzrllZtYVmBW4WCLSFMTHRPHoJRnszCvm9teXeB1HqlGrInDOzXbOjXTO/cV/0HiXc+76AGcTkSZgUFoLbji9B/9ZvE2nlAap2p419JKZJZlZPLAUWG5mtwQ2mog0Fb88tRsZnVvwf68vZVtuoddxpJLa7hrq65zbD4wC3gW64DtzSETkiKIiI5g4ZjCl5Y7fzVhMRYVGKQ0mtS2CaP91A6OAN51zpYD+JkWk1rqkxHP7uX34bPUuXvhyg9dx5DC1LYLJwAYgHvjUzI4BNMSgiNTJuOM7c2qvVO5/dyWrd+R5HUf8anuw+DHnXEfn3DnOZyNwWoCziUgTY2Y8ePFA4mOiuH76QorLyr2OJNT+YHGymU00syz/7W/4tg5EROqkTWIsD/54ICu27edvH+iq42BQ211D/wDygDH+237guUCFEpGmbUTftow7vjPPfLaOOWt2eR0n7NW2CLo55+50zq3z3+4GugYymIg0bX88ty9dUuK56ZVF7Cso8TpOWKttERSa2Q8O3jGzYYBOBhaRo9a8WSSPXZLB7vxiTXzvsdoWwbXAk2a2wcw2AE8A1wQslYiEhf4dk7n5jF68u3Q7/87K9jpO2KrtWUOLnHODgIHAQOdcBvDDgCYTkbBw9cldObFra+56axnrNZGNJ+o0Q5lzbr//CmOAm2pa18zSzGyWmS03s2VmdkMV64wzs8VmtsTM5pjZoLrkEZHQFxFhTBw7iOjICG7URDaeqM9UlXaEx8uAm51zfYETgAlm1rfSOuuBU5xzA4A/A1PqkUdEQlT75OY8cJFvIhudUtr46lMENR7Zcc5tc84t8H+dB6wAOlZaZ45zbq//7ldAp3rkEZEQdvaA9lx6XGcmzV7L7FU5XscJKzUWgZnlmdn+Km55QIfavomZpQMZwNwaVhuPb0A7EQlTd57fl15tE7npXwvZub/I6zhho8YicM4lOueSqrglOueiavMGZpYAvArceNjxhcrrnIavCH5fzeNXH7yqOSdHvymINFWx0ZE8cVkG+SVl3PivhZRrlNJGUZ9dQ0fkH7H0VWCac25mNesMBP4OXOCcq3JiU+fcFOdcpnMuMzU1NXCBRcRzPdom8qeR/ZmzdjdPf7LG6zhhIWBFYGYGPAuscM5NrGadzsBM4HLnnI4QiQgAozM7MXJQBx7+aDULNu098hOkXgK5RTAM3+Q1PzSzhf7bOWZ2rZld61/nDqA18JT/8awA5hGREGFm3HNhf9onx3LD9G/IK9LE94FkoXZZd2ZmpsvKUl+IhIP5G/cwetKXXDC4Iw+PHex1nJBmZvOdc5lVPRbQYwQiIvUx5JhW3HB6T177ZguvfaMhKAJFRSAiQW3Cad0Ymt6SO95YxvZcnVIaCCoCEQlqUZER/PXiQZSWV3DbaxqlNBBUBCIS9NJT4vndmb3578qdzFywxes4TY6KQERCwpUnpTM0vSV3v7VMVx03MBWBiISEiAjjwYsHUVymXUQNTUUgIiGjS0o8t5zZi49W7OSfczd5HafJUBGISEj52bAunNorlT//ZznLt1Y5fJnUkYpAREJKRITxt9GDaBkXza9eWkB+cZnXkUKeikBEQk7rhBgeGZvBht35/N8bS72OE/JUBCISkk7s1prrT+/BzAVbeGfJNq/jhDQVgYiErF+d1p1+HZK4+61lHNAuoqOmIhCRkBUVGcG9Fw5gZ14xEzXX8VFTEYhISBuc1oJxx3dm6pz1LN2S63WckKQiEJGQd8uZvWkV34zbX1+q6S2PgopAREJecvNo/nhuXxZt3se0uRu9jhNyVAQi0iRcMLgDw3umcv87K9mwK9/rOCFFRSAiTYKZ8ZcfDyAq0vjtvxdpF1EdqAhEpMlon9ycu0f2I2vjXp79fJ3XcUKGikBEmpQLMzpyRt+2PPT+KlbtyPM6TkhQEYhIk2Jm3HfRABJio7hFu4hqRUUgIk1OSkIMd5zXl0XZubz0tYarPhIVgYg0SRcM7sBJ3Vrz4Hsryckr9jpOUFMRiEiTZGb86YL+FJWWc987K7yOE9RUBCLSZHVvk8A1w7vx2jdbmLN2l9dxgpaKQESatF/9sDtprZrzx9eXUlRa7nWcoKQiEJEmLTY6kntHDWBdTj4Pf6gRSqsSsCIwszQzm2Vmy81smZndUMU6ZmaPmdkaM1tsZscGKo+IhK/hPVO59LjOTPlsHVkb9ngdJ+gEcougDLjZOdcXOAGYYGZ9K61zNtDDf7saeDqAeUQkjN1+bh86tmjOb/+9iIISTWJzuIAVgXNum3Nugf/rPGAF0LHSahcALzifr4AWZtY+UJlEJHwlxETx14sHsWF3AQ++963XcYJKoxwjMLN0IAOYW+mhjsDmw+5n8/2ywMyuNrMsM8vKyckJVEwRaeJO7NaaK09KZ+qcDdpFdJiAF4GZJQCvAjc65/YfzWs456Y45zKdc5mpqakNG1BEwsrvzupF++RY7nhjmYaf8AtoEZhZNL4SmOacm1nFKluAtMPud/IvExEJiLhmUdx2Th+Wb9vP9HkafgICe9aQAc8CK5xzE6tZ7U3gCv/ZQycAuc65bYHKJCICcN7A9pzQtRUPvf8t+wpKvI7juUBuEQwDLgd+aGYL/bdzzOxaM7vWv847wDpgDfAMcF0A84iIAL7hJ+4a2Y/cwlL+9oGuLYgK1As75z4H7AjrOGBCoDKIiFSnd7skLj/hGF78aiOXHteZvh2SvI7kGV1ZLCJh66Yf9aJFXDPufmsZvt9Lw5OKQETCVnJcNDef0ZO56/fwzpLtXsfxjIpARMLaJUM706d9Eve9syJsB6VTEYhIWIuMMO48vy9b9hUyeXZ4TnivIhCRsHdC19acO6A9T89ew9Z9hV7HaXQqAhER4A/n9MY5eODdlV5HaXQqAhERoFPLOK4e3pU3F21l4eZ9XsdpVCoCERG/a07pRkpCDPe+vTysTidVEYiI+CXERHHTj3oyb8Ne3l8WPqeTqghERA4zJrMTPdok8MC7Kykpq/A6TqNQEYiIHCYqMoLbzu3Dht0F/POrjV7HaRQqAhGRSk7tmcoPuqfw6Mer2XWg2Os4AaciEBGpxMx3kVlBSRn3vr3C6zgBpyIQEX9QR+sAAAuJSURBVKlCj7aJXHtKN177Zgufr97ldZyAUhGIiFRjwmndSW8dxx9fX9KkxyFSEYiIVCM2OpJ7LxzAht0FPDlrjddxAkZFICJSg2HdU7gooyOTZq9lzc48r+MEhIpAROQIbj+3D3HNovjj60ub5BXHKgIRkSNonRDD787qxVfr9vD6wi1ex2lwKgIRkVq4dGhnBqW14N63V5BbUOp1nAalIhARqYWICOPeUf3Zk1/CXz9oWkNVqwhERGqpf8dkrjgxnWlzN7Fg016v4zQYFYGISB3cdEZPOiQ359cvfdNkdhGpCERE6iApNponLstgZ14RN/97YZM4i0hFICJSRxmdW3LbOX34aMVOnvks9Ce8j/I6gIhIKLrypHS+Xr+Hv7z3LW2TYjmrfztioiK9jnVUVAQiIkfBzPjLxQNZ/dQcbpi+kPhmkZzWuw1jh6Zxco9Ur+PVScB2DZnZP8xsp5ktrebxZDN7y8wWmdkyM7sqUFlERAIhKTaad64/malXDWXk4I58tW4Plz/7NZNnrw2pYweBPEYwFTirhscnAMudc4OAU4G/mVmzAOYREWlwzaIiOLVXG+6/aACf//40zh3YnvvfXcmdby6jvCI0yiBgu4acc5+aWXpNqwCJZmZAArAHKAtUHhGRQIuNjuTxSzLo2KI5Uz5dx9Z9Rdx/0QBSE2O8jlYjL88aegLoA2wFlgA3OOeqnCnazK42sywzy8rJyWnMjCIidRIRYdx2Th/uHtmP2at2ctpDnzB59lqKy4J3PgMvi+BMYCHQARgMPGFmSVWt6Jyb4pzLdM5lpqaG1kEYEQlPPz0pnfdvHM7xXVpx/7srOfPhT1m1IziHsfayCK4CZjqfNcB6oLeHeUREGlTX1ASevXIoz//sOA4UlzP++XnszS/xOtb3eFkEm4DTAcysLdALCP0rM0REKjmlZypTrhjCjtxifvXyAsrKq9wL7plAnj76MvAl0MvMss1svJlda2bX+lf5M3CSmS0BPgZ+75xr2jNEi0jYOrZzS+65sD9frNnN/e8G1+ilgTxr6NIjPL4VOCNQ7y8iEmzGZKaxYtt+nv18Pb3aJTImM83rSICuLBYRaVS3n9OHNTsPcNvMJbRLimV4T+9PgNGgcyIijSgqMoKnxh1L9zYJXDdtAcu25nodSUUgItLYEmOjmXrVcSTGRnHVc/PYsq/Q0zwqAhERD7RLjmXqVcdRWFrO5X+fy7Zc78pARSAi4pFe7RJ57sqh7MwrZszkL9m8p8CTHCoCEREPZaa34qVfHE9eURmjJ33J2pwDjZ5BRSAi4rGBnVow/eoTKKuoYNSTX3DXm8tYnL2v0YaytlAaMxsgMzPTZWVleR1DRKTBrd+Vz0MffMuHy3dQUlZB9zYJjBrcgZGDOtK5dVy9XtvM5jvnMqt8TEUgIhJccgtLeWfJNmYuyGbehr0ADE5rwS9O7sq5A9sf1WvWVAS6oExEJMgkN4/m0uM6c+lxndmyr5C3Fm3ljYVbA3ZmkbYIRERCRHmFIzLCjuq5NW0R6GCxiEiIONoSOBIVgYhImFMRiIiEORWBiEiYUxGIiIQ5FYGISJhTEYiIhDkVgYhImAu5C8rMLAfY6L+bDOTW8HXlZdHArjq+5eGvUZvHKi+rbcaDf6bUMWNj5Tu4TJ9hcOULhYzBnq8+GWtaFmyf4THOuarnxXTOhewNmFLT15WXAVn1eY/aPFZ5WW0zHvZnnTI2Vj59hsGZLxQyBnu++mQ8Qtag+gxruoX6rqG3jvB1dY8f7XvU5rHKy2qbMdjzHem9aqLP8MjvU5MjPS/YMwZ7vuoer03GIy2ri0B/htUKuV1D9WFmWa6asTaCRbBnDPZ8EPwZgz0fBH/GYM8HoZHxoFDfIqirKV4HqIVgzxjs+SD4MwZ7Pgj+jMGeD0IjIxBmWwQiIvJ94bZFICIilagIRETCnIpARCTMqQj8zOxkM5tkZn83szle56mKmUWY2b1m9riZ/dTrPJWZ2alm9pn/czzV6zxVMbN4M8sys/O8zlIVM+vj//xmmNkvvc5TFTMbZWbPmNm/zOwMr/NUZmZdzexZM5vhdZaD/P/unvd/buO8zlNZkygCM/uHme00s6WVlp9lZt+a2Rozu7Wm13DOfeacuxb4D/B8MGYELgA6AaVAdhDmc8ABIDZI8wH8HnilIbM1ZEbn3Ar/v8MxwLAgzfi6c+4XwLXA2CDMt845N74hc1WljlkvAmb4P7eRgc5WZ3W58i1Yb8Bw4Fhg6WHLIoG1QFegGbAI6AsMwPfD/vBbm8Oe9wqQGIwZgVuBa/zPnRGE+SL8z2sLTAvCfD8CLgGuBM4Lxr9j/3NGAu8ClwVrRv/z/gYcG8T5GvT/SD2z/gEY7F/npUDmOppbFE2Ac+5TM0uvtPg4YI1zbh2AmU0HLnDO3Q9UuVvAzDoDuc65vGDMaGbZQIn/bnmw5TvMXiAm2PL5d1fF4/uPWWhm7zjnKoIpo/913gTeNLO3gZcaKl9DZTQzAx4A3nXOLQi2fI2lLlnxbSF3AhYShHtimkQRVKMjsPmw+9nA8Ud4znjguYAl+r66ZpwJPG5mJwOfBjKYX53ymdlFwJlAC+CJwEYD6pjPOXc7gJldCexqyBKoQV0/w1Px7UaIAd4JaLLv1PXf4a+BEUCymXV3zk0KZDjq/hm2Bu4FMszsD/7CaCzVZX0MeMLMzuXoh6AImKZcBHXmnLvT6ww1cc4V4CuroOScm4mvrIKac26q1xmq45z7BPjE4xg1cs49hu8HW1Byzu3Gd/wiaDjn8oGrvM5RnaDbRGlAW4C0w+538i8LJsGeUfnqTxnrL9jzHS6Ush7SlItgHtDDzLqYWTN8Bwnf9DhTZcGeUfnqTxnrL9jzHS6Usn7H66PVDXT0/mVgG9+dVjnev/wcYBW+o/i3K6PyKWNwZwz2fKGa9Ug3DTonIhLmmvKuIRERqQUVgYhImFMRiIiEORWBiEiYUxGIiIQ5FYGISJhTEUiTYGYHGvn9GmTOCvPN4ZBrZgvNbKWZPVSL54wys74N8f4ioCIQqZKZ1TgOl3PupAZ8u8+cc4OBDOA8MzvSPASj8I2gKtIgVATSZJlZNzN7z8zmm2/mtN7+5eeb2Vwz+8bMPjKztv7ld5nZi2b2BfCi//4/zOwTM1tnZtcf9toH/H+e6n98hv83+mn+YZoxs3P8y+ab2WNm9p+a8jrnCvENU9zR//xfmNk8M1tkZq+aWZyZnYRvvoK/+rciulX3fYrUlopAmrIpwK+dc0OA3wJP+Zd/DpzgnMsApgO/O+w5fYERzrlL/fd74xta+zjgTjOLruJ9MoAb/c/tCgwzs1hgMnC2//1TjxTWzFoCPfhuiPGZzrmhzrlBwAp8QxjMwTd2zS3OucHOubU1fJ8itaJhqKVJMrME4CTg3/5f0OG7yXI6Af8ys/b4ZpFaf9hT3/T/Zn7Q2865YqDYzHbim32t8jScXzvnsv3vuxBIxzdl5zrn3MHXfhm4upq4J5vZInwl8Ihzbrt/eX8zuwff/A4JwPt1/D5FakVFIE1VBLDPv++9sseBic65N/0Twdx12GP5ldYtPuzrcqr+P1ObdWrymXPuPDPrAnxlZq845xYCU4FRzrlF/sl0Tq3iuTV9nyK1ol1D0iQ55/YD681sNPimVzSzQf6Hk/lujPifBijCt0DXw6YyPOIk7/6thweA3/sXJQLb/Lujxh22ap7/sSN9nyK1oiKQpiLOzLIPu92E74fneP9ul2X45o4F3xbAv81sPrArEGH8u5euA97zv08ekFuLp04ChvsL5P+AucAXwMrD1pkO3OI/2N2N6r9PkVrRMNQiAWJmCc65A/6ziJ4EVjvnHvY6l0hl2iIQCZxf+A8eL8O3O2qyx3lEqqQtAhGRMKctAhGRMKciEBEJcyoCEZEwpyIQEQlzKgIRkTCnIhARCXP/D969BMSh6tSpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.240978</td>\n",
       "      <td>0.122962</td>\n",
       "      <td>0.969600</td>\n",
       "      <td>02:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.110139</td>\n",
       "      <td>0.055658</td>\n",
       "      <td>0.985200</td>\n",
       "      <td>02:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.078175</td>\n",
       "      <td>0.044224</td>\n",
       "      <td>0.988600</td>\n",
       "      <td>02:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, lr_max=6e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Refactor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": "<h2 id=\"ConvLayer\" class=\"doc_header\"><code>class</code> <code>ConvLayer</code><a href=\"https://github.com/fastai/fastai/tree/master/fastai/layers.py#L237\" class=\"source_link\" style=\"float:right\">[source]</a></h2>\n\n> <code>ConvLayer</code>(**`ni`**, **`nf`**, **`ks`**=*`3`*, **`stride`**=*`1`*, **`padding`**=*`None`*, **`bias`**=*`None`*, **`ndim`**=*`2`*, **`norm_type`**=*`<NormType.Batch: 1>`*, **`bn_1st`**=*`True`*, **`act_cls`**=*`ReLU`*, **`transpose`**=*`False`*, **`init`**=*`'auto'`*, **`xtra`**=*`None`*, **`bias_std`**=*`0.01`*, **`dilation`**:`Union`\\[`int`, `Tuple`\\[`int`, `int`\\]\\]=*`1`*, **`groups`**:`int`=*`1`*, **`padding_mode`**:`str`=*`'zeros'`*, **`device`**=*`None`*, **`dtype`**=*`None`*) :: `Sequential`\n\nCreate a sequence of convolutional (`ni` to `nf`), ReLU (if `use_activ`) and `norm_type` layers.",
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ConvLayer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a sequence of convolutional (ni to nf), ReLU and norm_type layers.\n",
    "def conv2(ni, nf): return ConvLayer(ni, nf, stride=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(\n",
    "    conv2(1, 8),\n",
    "    conv2(8, 16),\n",
    "    conv2(16, 32),\n",
    "    conv2(32, 16),\n",
    "    conv2(16, 10),\n",
    "    Flatten()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls, net,loss_func=CrossEntropyLossFlat(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(valley=0.010964781977236271)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3yV9d3/8dcnm0wghBVGCBtFhogoTmy1tVpHFa3crqpota623rWtv+rdvRxdDm7rqquuWuu+bVWcKMgeskeYCSMDyP78/jiHSjEJCeTKdZLzfj4e50FyjXPeOUDe55pfc3dERCR+JYQdQEREwqUiEBGJcyoCEZE4pyIQEYlzKgIRkTinIhARiXNJYQdoqW7dunlBQUHYMURE2pVZs2aVuHteQ/PaXREUFBQwc+bMsGOIiLQrZramsXnaNSQiEudUBCIicU5FICIS59rdMQIRkcbU1NRQVFREZWVl2FFCk5aWRp8+fUhOTm72OioCEekwioqKyMrKoqCgADMLO06bc3e2bt1KUVERAwYMaPZ62jUkIh1GZWUlubm5cVkCAGZGbm5ui7eItEUggauvd7bvqmZTWSUlFdXU1tXjDg7065rOoO6ZJCa07n9cd4/bXwbxLt7/3g/k51cRtBOLNpTxxEdr2Vi6m5xOKXRJT6ZLRgpdo4/UpAQWbyxnXtEOFm4oo6Kqlrp6p67eSU9JpGdOGt2z0ijITefIwlzGF3QlJ735+xAbU1/vbNtVzZayKnbX1OEeec2NpZXMXrud2et2sGRjOdV19Y0+R2ZqEof1yeHQ/BwG5mVQmJfJwLxMumakNLqOu7OyZCcL1peyZFM5SzeVs3bbLsoqayjbXUtlbR25Gan0ykmje1YqFVW1bC6rZHNZFV3Skxk/oCsTCnMZ278L/XPTSU1KPOj3QuRAZGZmUlFRwerVqznttNNYsGBBm2dQEcQwd+fl+Zt44L1VzFqzndSkBArzMlm0oYwdu2vYVV33uXX6dU1nZJ8cuqQnk5SQQIIZFVU1bCqromj7LqYvK+b+d1dhBgPzMslMTSI50UhKSCAxwUhIMBIMdlXXsWNXNdt31QDQu3Mn+nTuRJeMZIrLq9hUVsXm0kpKKqqorW94cKP0lERG9enMxUf3p3fnTvTMTiMvK5XkxEiuendWFFcwe+0OZq/bzkPvrf6PwuiWmcqQHpkM6h7JmZacSFKisXhjOR+u3EpxeRUASQnGwLxMCvMy6NwphexOSaQmJVJSUcXG0ko2lFaSlZrEofk5nDQ8jU2llby7vITn52wAIMGgT5d0+uem0z0rjW6ZKeRmppCekkRqUgJpyYn06dKJEb2zVRgdzbyn4J8/htIiyOkDJ/0IDpscdqo2F1gRmFlf4BGgB5G9ANPc/Xf7LHMC8HdgVXTSc+7+46AytSe7q+v44fPzee6T9RTkpnPLV4ZzzuF96Jz+2afkypo6tu2sZtvOanZW1TK4R1aTn6L3rDN33Q4+XLmN+etLqaqto7bOqa2vp6rWqXeodyctOZEB3TIYm56CO2wo3c3ijWVs31VNt8xUeuakMbh7N7pnpUYe2WmkpySSYEZigtE1I4UhPbL2u8tnVN/OnD22DwB19c767btZUVLB8s0VLN1cztItFfxt9np2VddRFy2cHtmpTByYy4TCXMb068KAbhmkJLXscJe7s6I4skWxsmQnq0p2smbrTlZsqaCkorrBLZiUxASG985mfEEXvjC8B4f370JSog6ztVvznoJ/XAc1uyPfl66LfA8HVQY333wzffv25ZprrgHgtttuIykpiTfffJPt27dTU1PDT3/6U84444xGn6Ouro6bb76Zt956i6qqKq655hquvPJKLrroIs4++2zOPPNMAKZMmcLkyZObfK7msKCGqjSzXkAvd//EzLKAWcCZ7r5or2VOAL7r7qc193nHjRvnHf0WE6tLdnLVo7P4dHM51580mGsnDW71fejtUW1dPVW19aSnJAa6H9jdKa+qpbK6jqraenbX1LFiSwVz1u1g9rodzFm7g+q6erqkJ3P8kDxG9e3MyPwchvfKJiNVG9lhWrx4McOHD2/ewnceGvnlv6+cvnDjge+emT17NjfccANvv/02ACNGjOC1114jJyeH7OxsSkpKmDBhAsuWLcPMGtw1NG3aNLZs2cItt9xCVVUVEydO5Omnn2bt2rXceeedPP/885SWljJ69GiWLVtGUtJ//rtr6H0ws1nuPq6hzIH9q3X3jcDG6NflZrYYyAcWNbliyNydsspastOSDvqXjbszt6iUrRVVVNfWU11XT4/sNIb3zCYnPZmq2jreXVbCy/M3MWfddmqj+/RLKqpITUrkwUuO4ISh3VvpJ2v/khIT2uQTuJmRnZZMdtpnx1CG9MjiyyN7AVBRVcv0pcW8sWgz05d9tovJDAbkZjCsVxbDembTKyeN7E6R58nLSqV/bjrJ2oKIHaVFLZveTGPGjGHLli1s2LCB4uJiunTpQs+ePbnxxhuZPn06CQkJrF+/ns2bN9OzZ88Gn+P1119n3rx5PPPMM5FIpaUsW7aMk08+mauvvpri4mKeffZZvva1r32uBA5Em3x8MbMCYAwwo4HZR5nZXGADka2DhW2RqSGVNXVc8chM3llWQkZKIn27ptO7cyc6pSSSmpRASmICJRVVbNhRyZbySr4yshc/Ov2QBj+t19c7P35xEQ+9v7rB1+qdk0Z5ZS3lVZHSmVCYS6eURBITjIyUJKYeV0jfrukB/8RyIDJTkzh1ZC9OHdkLd2dLeRUL1peyYH0ZizeWsWhDGa8s2MS+G9vJiUZBbgZDe2ZxeP8ujOvfleG9srR7KSw5fRrZIuhz0E997rnn8swzz7Bp0ybOO+88HnvsMYqLi5k1axbJyckUFBQ0eYqnu/OHP/yBU0455XPzLrroIh599FGefPJJHnzwwYPOCm1QBGaWCTwL3ODuZfvM/gTo7+4VZnYq8DwwuIHnmApMBejXr18gOWvr6rnuidm8s6yEqccVUl1bT9H2XWwsrWR3TR1VNZHdEt0yU+iVk0bvzp14+IM1lFfW8utzDvuP/8zVtfXc9Mxc/j5nA5ccXcBZY/JJTU4gKSGB9Tsi+9qXbCwjLTmRUw7tycSB3Vq8j1tig5nRIzuNHtlpnDS8x7+n76quZWtFNaW7ayjbXcPG0kqWF1ewbHPk4PiL8zYCkJGSyJh+XTiioCtHDIj8qa2GNnLSj/7zGAFAcqfI9IN03nnnccUVV1BSUsLbb7/NU089Rffu3UlOTubNN99kzZpGbwQKwCmnnMI999zDpEmTSE5OZunSpeTn55ORkcEll1zC+PHj6dmzJyNGjDjorBBwEZhZMpESeMzdn9t3/t7F4O4vm9ndZtbN3Uv2WW4aMA0ixwhaO2d9vXPzc/N5fdFm/uerh3Dx0QXNWu+P/1rGb19fSlVdPXedN5raOmdlSQW/fvVT3l5azH9/aSjfPH7gf+xiGtQ9k+OHNHhLcOlA0lOSSO+aRN9G5m/YsZuZa7Yzc/U2Pl69nbv+uRT3yMHzu6eMJb9zpzbNG5f2HBAO4KyhQw45hPLycvLz8+nVqxdTpkzh9NNPZ+TIkYwbN45hw4Y1uf7ll1/O6tWrGTt2LO5OXl4ezz//PAA9evRg+PDh/z5g3BqCPFhswMPANne/oZFlegKb3d3NbDzwDJEthEZDtcbB4rLKGs6/70OqauvIzUwFh49Wb+OGLwzmhi8MadFzTZu+gp+/vIScTsmU7o6caplg8POzRnL++GC2XqTjKd1dw/8t2sxtLywkOdH43fljOE4fGFqsRQeL26ldu3YxcuRIPvnkE3JychpcJmYOFgMTgQuB+WY2JzrtB0A/AHe/FzgH+KaZ1QK7gfObKoHW8vfZ61m0sYwvDO9OWWUt23ZVc92kQVx/0uf2Su3X1OMGkpeVyjvLShiQG7kY6tD8bPrnZgSQXDqqnE7JnHN4H8b268w3H/2Eix/8iGtPHMQ1kwbp2gX5tzfeeIPLLruMG2+8sdESOBCBbREE5WC3CNydL//uHZITE/jHtce0YjKR1rGrupZbnl/Ac5+sZ2BeBr84+zDGD+gadqx2IR62CJqjpVsEcXdUam5R5JYEX9duG4lR6SlJ3DF5NA9eegRVtfVMvu8Dvvv0XJZvqQg7mnRQcVcET8xYS3pKIl8d3TvsKCJNOnFod16/8TiuPK6QF+Zu4At3vM1FD3zEW59uob1tybeleH9vDuTnj6siKKus4YW5GzhjdG8ydQWotAPpKUl8/9ThfHDzJL7zxSEs2VjGJQ9+zDce+piNpbv3/wRxJi0tja1bt8ZtGewZjyAtLa1F68XVb8O/z9nA7po67RaSdic3M5VrTxrMlccP5NEP1/Cb1z7l5Dumc8tpw5k8rm/c33p5jz59+lBUVERxcXHYUUKzZ4SyloibInB3Hp+xlkN6ZzMyv/WOtou0pZSkBL5xzABOGt6d7z07j+89O59XF2ziN+eOoltmatjxQpecnNyikbkkIm52Dc0rKmXxxjK+Pr6fPj1Ju9c/N4PHL5/AbaeP4L0VW/nSXdN589MtYceSdipuimB3TR1HFHThDB0klg4iIcG4ZOIA/vGtY+iWmcqlD37Mz19eTH0j40OINCburiMQ6Ygqa+r46UuLePTDtXzlsF7cfu4o0pJ1IZp8Jqwri0WkjaQlJ/KTMw6lf9cMfvbyYorLq/jfC8e1ynCk0vHFza4hkY7OzLjiuEJ+d/5oZq/dztn3vMeC9aVhx5J2QEUg0sGcMTqfR75xJOWVtZx193v86c3l1DYw9KbIHioCkQ7oqIG5vHbDcZx8SE9+89qnTL7vA12AJo1SEYh0UF0yUvjTBWP53fmjWbq5gjP/pF1F0jAVgUgHd8bofJ6+6igSzJh83wf8c/HmsCNJjFERiMSB4b2yef6aiRTmZXDFIzP52+yDG6BdOhYVgUic6JGdxlNXHsWEwlxuenoeby+N3/vxyH9SEYjEkfSUJO678HCG9Mjim4/OYl7RjrAjSQxQEYjEmay0ZB76xhF0zUjh0gc/ZnXJzrAjSchUBCJxqHtWGo98YzwOTLl/Buu27Qo7koQosCIws75m9qaZLTKzhWZ2fRPLHmFmtWZ2TlB5ROQ/FeZl8sg3xlNRVcv50z5k7VaVQbwKcougFviOu48AJgDXmNmIfRcys0TgV8DrAWYRkQYcmp/DY5cfyc7qWr7+vyqDeBVYEbj7Rnf/JPp1ObAYyG9g0WuBZwHdTF0kBHuXwXnTPmD5loqwI0kba5NjBGZWAIwBZuwzPR84C7inLXKISMMO6Z3DE1dMoKaungfu+RXVvx0Bt3WGOw+FeU+FHU8CFvhtqM0sk8gn/hvcvWyf2XcB33P3+qZGDTOzqcBUgH79NN6wSBCG98rm5RM3kfV/95FSURWZWLoO/nFd5OvDJocXTgIV6MA0ZpYMvAi85u53NDB/FbCnAboBu4Cp7v58Y8+pgWlEAnTnoZFf/vvK6Qs3Lmj7PNJqQhmYxiIf8f8MLG6oBADcfcBeyz8EvNhUCYhIwEobufVEY9OlQwhy19BE4EJgvpnNiU77AdAPwN3vDfC1ReRA5PRpcIugMr0XaSHEkbYRWBG4+7t8ttunOctfElQWEWmmk34UOSZQ89nYBZWk8oPyszlneQlHD+oWYjgJiq4sFpHPHDYZTv995JgABjl9qf3KXSzoejJXPTpLp5Z2UIEeLA6CDhaLtL1123Zx1t3vkZ6SxPPXTKRrRkrYkaSFmjpYrC0CEdmvvl3TmXbRODaXVTL1kZlU1daFHUlakYpARJplbL8u3D55FDPXbOe2FxaGHUdakYpARJrttMN6c9XxA3nio3W89anuCtNRqAhEpEVu/OJghvTI5OZn51O6uybsONIKVAQi0iKpSYn89txRFFdU8ZMXF4UdR1qBikBEWuywPp25+oSBPDOriH8u3hx2HDlIKgIROSDXThrMsJ5Z/OBv86moqg07jhwEFYGIHJCUpAR+cfZINpdV8ft/Lgs7jhwEFYGIHLAx/bpw3ri+PPDuKpZtLg87jhwgFYGIHJT//tJQ0lMSufWFhbS3OxVIhIpARA5KbmYqN50ylPdXbOWl+RvDjiMHQEUgIgftgiP7c0jvbH764mLKK3VtQXujIhCRg5aYYPz0zEPZUl7J//xD1xa0NyoCEWkVY/p14ZoTB/HMrCJeXaBdRO2JikBEWs11Jw3m0Pxsvv/cfLaUV4YdR5pJRSAirSY5MYE7J49mV3UdNz87X2cRtRMqAhFpVYN7ZHHzl4fxryVbeHTG2rDjSDOoCESk1V18VAHHD8njJy8uYsmmsrDjyH4EVgRm1tfM3jSzRWa20Myub2CZM8xsnpnNMbOZZnZMUHlEpO0kJBi3Tx5FTqdkvvX4bHZXa0SzWBbkFkEt8B13HwFMAK4xsxH7LPNPYJS7jwa+AdwfYB4RaUPdMlO5c/JoVhRX8OMXNaJZLAusCNx9o7t/Ev26HFgM5O+zTIV/djQpA9CRJZEO5JjB3f49otnLuuo4ZrXJMQIzKwDGADMamHeWmS0BXiKyVdDQ+lOju45mFhcXBxlVRFrZt784hFF9cvh/zy9gx67qsONIAwIvAjPLBJ4FbnD3zx01cve/ufsw4EzgJw09h7tPc/dx7j4uLy8v2MAi0qqSExP45dcOo3R3DT9/eXHYcaQBgRaBmSUTKYHH3P25ppZ19+lAoZl1CzKTiLS94b2yufzYQp6aWcQHK7aGHUf2EeRZQwb8GVjs7nc0ssyg6HKY2VggFdC/EpEO6PqTBtOvazo//Nt8Kmt0FlEsCXKLYCJwITApenroHDM71cyuMrOrost8DVhgZnOAPwHnuS5FFOmQOqUk8rOzDmVlyU7+9ObysOPIXpKCemJ3fxew/SzzK+BXQWUQkdhy7OA8zh6Tz91vreALw3swqm/nsCMJurJYRNrYracfQvesVG58ao4uNIsRKgIRaVM56cncfu4oVhbv5Bev6CyiWKAiEJE2d/Sgblx2zAAe+WANb326Jew4cU9FICKhuOmUoQzunslNz8xjc5nGLgiTikBEQpGWnMgfLhjDrqpaLnv4Y3ZV14YdKW6pCEQkNMN6ZvPHC8ayaEMZ1z0xh7p6nT0eBhWBiITqxGHdue2rh/DG4s387CUdPA5DYNcRiIg010VHFbCqZCcPvLeK4b2yOHdc37AjxRVtEYhITLjlKyM4qjCXH/19Icu3VIQdJ66oCEQkJiQmGHedP5pOKYl86/FPdD+iNqQiEJGY0SM7jd+eexhLNpXzC92yus2oCEQkpkwa1oPLjxnAwx+s4Y1Fm8OOExdUBCISc/77S8MY1jOLH7+4iKpa7SIKmopARGJOSlICP/zKcNZu28VfPlgTdpwOT0UgIjHp2MF5HD8kjz/8a7nGOg6YikBEYtYPTh1OeWUNf/yXBrIJkopARGLW0J5ZTB7Xl4c/WM3arbvCjtNhqQhEJKZ9+4tDSEpI4Jev6nTSoKgIRCSmdc9O4+oTBvLy/E28t7wk7DgdUmBFYGZ9zexNM1tkZgvN7PoGlpliZvPMbL6ZvW9mo4LKIyLt1xXHFdKvazq3vrCQ6tr6sON0OEFuEdQC33H3EcAE4BozG7HPMquA4919JPATYFqAeUSknUpLTuTW00ewfEsFD7+/Ouw4HU5gReDuG939k+jX5cBiIH+fZd539+3Rbz8E+gSVR0Tat5OG9+CkYd25642lGtGslbXJMQIzKwDGADOaWOwy4JW2yCMi7dOPTh9BTb3zc92HqFU1qwjMLMPMEqJfDzGzr5pZcjPXzQSeBW5w97JGljmRSBF8r5H5U81sppnNLC4ubs7LikgH1D83gyuPK+TvczYwv6g07DgdRnO3CKYDaWaWD7wOXAg8tL+VomXxLPCYuz/XyDKHAfcDZ7j71oaWcfdp7j7O3cfl5eU1M7KIdERTjyskp1Myd72xNOwoHUZzi8DcfRdwNnC3u58LHNLkCmYG/BlY7O53NLJMP+A54EJ319+qiOxXVloyU48r5J9LtjBn3Y6w43QIzS4CMzsKmAK8FJ2WuJ91JhLZcphkZnOij1PN7Cozuyq6zI+AXODu6PyZLf0BRCT+XHx0AV3StVXQWpo7ZvENwPeBv7n7QjMrBN5sagV3fxew/SxzOXB5MzOIiACQmZrE1OMG8qtXlzBrzXYO798l7EjtWrO2CNz9bXf/qrv/KnrQuMTdrws4m4hIoy46qj+5GSnaKmgFzT1r6HEzyzazDGABsMjMbgo2mohI4zJSk7jq+IG8s6yEmau3hR2nXWvuMYIR0VM/zyRyrv8AIvv/RURCM2VCP7qkJ3Pv2yvDjtKuNbcIkqOngp4JvODuNYAHF0tEZP/SU5K46KgC3li8mWWby8OO0241twjuA1YDGcB0M+sPNHhxmIhIW7r46ALSkhOYNl1bBQequQeLf+/u+e5+qkesAU4MOJuIyH51zUjhvHF9eX7OejaW7g47TrvU3IPFOWZ2x57bPJjZ7US2DkREQnf5sYXUOzz43uqwo7RLzd019ABQDkyOPsqAB4MKJSLSEn27pvOVkb14fMZaSnfXhB2n3WluEQx091vdfWX08T9AYZDBRERa4srjC6moquXRD9eEHaXdaW4R7DazY/Z8Y2YTAe2ME5GYcUjvHCYN687976xkZ1Vt2HHaleYWwVXAn8xstZmtBv4IXBlYKhGRA3DtpEFs31WjrYIWau5ZQ3PdfRRwGHCYu48BJgWaTESkhcb068JxQ/KYNn0lu6q1VdBcLRqhzN3L9hpc5tsB5BEROSjXnzSIrTureXzG2rCjtBsHM1Rlk3cWFREJw+H9uzJxUC73vr2Sypq6sOO0CwdTBLrFhIjEpOsmDaakokpbBc3UZBGYWbmZlTXwKAd6t1FGEZEWObIwlwmFXbnn7RU6VtAMTRaBu2e5e3YDjyx3b+6gNiIibe67Jw+luLyKh95fHXaUmHcwu4ZERGLWuIKuTBrWnXvfWqGrjfdDRSAiHdZ3Tx5KWWUt06avCDtKTFMRiEiHNaJ3Nl8d1ZsH3l1NcXlV2HFiVmBFYGZ9zexNM1tkZgvN7PoGlhlmZh+YWZWZfTeoLCISv2784hCq6+r505vLw44Ss4LcIqgFvuPuI4AJwDVmNmKfZbYB1wG/DTCHiMSxAd0ymDyuL4/NWMO6bbvCjhOTAisCd9/o7p9Evy4HFgP5+yyzxd0/BnQkR0QCc/1JgzEz7npjWdhRYlKbHCMwswJgDDDjANefumdQnOLi4taMJiJxoGdOGhcf1Z+/zS7S2MYNCLwIzCwTeBa4Ya/7FLWIu09z93HuPi4vL691A4pIXPjmCYNIT0ni9teXhh0l5gRaBGaWTKQEHnP354J8LRGRpnTNSOGKYwt5deEm5q7bEXacmBLkWUMG/BlY7O53BPU6IiLNddmxA8jNSOE3r30adpSYEuQWwUTgQmCSmc2JPk41s6vM7CoAM+tpZkVEbml9i5kVmVl2gJlEJI5lpiZx9YmDeHd5CTNWbg07TswI7H5B7v4u+7lVtbtvAvoElUFEZF9TjuzHPW8t5563V3BkYW7YcWKCriwWkbiSlpzIpRMH8NanxSzacEDnr3Q4KgIRiTv/NaE/malJ3PO27kEEKgIRiUM5nZKZMqEfL83bwJqtO8OOEzoVgYjEpcsmDiApMYFp01eGHSV0KgIRiUvds9M45/A+PD2riC3llWHHCZWKQETi1tRjC6mpq+cvH6wJO0qoVAQiErcKumVwwpA8/vrxOmrr6sOOExoVgYjEtQuO7M+W8ir+uWRL2FFCoyIQkbh24tA8eman8fiMtWFHCY2KQETiWlJiApOP6Mv0ZcVxO3CNikBE4t75R/TFgCc/js+tAhWBiMS93p07ceLQ7jw1s4iaODxorCIQEQEuOLIfxeVVvLFoc9hR2pyKQEQEOGFod3rnpPFIHF5ToCIQEQESE4xLJhbwwcqtzCuKrxHMVAQiIlFfH9+PrNQk7ouz+w+pCEREorLSkpkyoT+vzN8YV3clVRGIiOzl0okFJCUkcP87q8KO0mZUBCIie+mRncZZY/J5auY6tlZUhR2nTQRWBGbW18zeNLNFZrbQzK5vYBkzs9+b2XIzm2dmY4PKIyLSXFccV0hVbT0Px8kZREFuEdQC33H3EcAE4BozG7HPMl8GBkcfU4F7AswjItIsg7pn8sURPXjkg9VU1tSFHSdwgRWBu29090+iX5cDi4H8fRY7A3jEIz4EOptZr6AyiYg016UTC9ixq4YX5m4IO0rg2uQYgZkVAGOAGfvMygfW7fV9EZ8vC8xsqpnNNLOZxcXFQcUUEfm3owpzGdw9k0c+WI27hx0nUIEXgZllAs8CN7h72YE8h7tPc/dx7j4uLy+vdQOKiDTAzLjoqP4sWF/G7HUd+wKzQIvAzJKJlMBj7v5cA4usB/ru9X2f6DQRkdCdNbYPmalJHX4oyyDPGjLgz8Bid7+jkcVeAC6Knj00ASh1941BZRIRaYnM1CTOObwPL83bSEkHPpU0yC2CicCFwCQzmxN9nGpmV5nZVdFlXgZWAsuB/wWuDjCPiEiL/deE/lTX1fPXj9ftf+F2KimoJ3b3dwHbzzIOXBNUBhGRgzWoeybHDOrGox+u4crjCklK7HjX4Xa8n0hEpJVdeFR/NpZW8q8OOsC9ikBEZD9OGtY9MsD9Rx1zKEsVgYjIfuwZ4P7tpR1zgHsVgYhIM3TkAe5VBCIizdCRB7hXEYiINFNHHeBeRSAi0kx7BrjvaAeNVQQiIs2UmGCcP74f7ywrYXVJxxnKUkUgItIC5x3Rl6QE46H3V4cdpdWoCEREWqBHdhpnjsnnyY/XdpihLFUEIiItdNXxkaEsO8pWgYpARKSFBnXP4uQRPXj4/dWUV9aEHeegqQhERA7A1ScMoqyylic6wBlEKgIRkQMwqm9nJg7K5f53VlFV274HuFcRiIgcoKtPGMSW8iqenlkUdpSDoiIQETlARw/M5YiCLvz61SWs37E77DgHTEUgInKAzIzfnjuKunrnxr/Ooa7ew450QFQEIiIHoX9uBv9zxqF8tGob9769Iuw4B0RFICJykL42Np/TDilPTZkAAAtJSURBVOvFnf+3lLnrdoQdp8UCKwIze8DMtpjZgkbmdzGzv5nZPDP7yMwODSqLiEiQzIyfnTWSHtlpXPfkbMra2bUFQW4RPAR8qYn5PwDmuPthwEXA7wLMIiISqJxOyfzu/NEUbd/NTU/Pxb39HC8IrAjcfTqwrYlFRgD/ii67BCgwsx5B5RERCdq4gq58/8vDeG3hZv787qqw4zRbmMcI5gJnA5jZeKA/0CfEPCIiB+2yYwZwyiE9+OUrS5i5uqnPwrEjzCL4JdDZzOYA1wKzgQYvzzOzqWY208xmFhcXt2VGEZEWMTN+fc4o8rt04luPz6a4PPbvUBpaEbh7mbtf6u6jiRwjyANWNrLsNHcf5+7j8vLy2jSniEhL5XRK5u4pY9mxu5qrH5tFdW1sj3EcWhGYWWczS4l+ezkw3d3LwsojItKaDumdw2/OGcXHq7dz2z8Whh2nSUlBPbGZPQGcAHQzsyLgViAZwN3vBYYDD5uZAwuBy4LKIiIShtNH9WbhhjLufXsFI3pl818T+ocdqUGBFYG7f30/8z8AhgT1+iIiseCmU4ayZFMZt72wkL5d0zl+SOzt3taVxSIiAUpMMH53/hiG9Mjiiodn8trCTWFH+hwVgYhIwHI6JfPEFRMY0Tubqx/7hL/PWR92pP+gIhARaQM56ck8evmRjOvfhRv+OofnPomdMQxUBCIibSQzNYmHLh3PUYW5fO/ZeXwcIxecqQhERNpQp5RE7p4ylvzOnbjqL7NiYkAbFYGISBvrnJ7C/RcfQXVtPZc/PJNd1bWh5lERiIiEYFD3TH5/wRg+3VTGlX+ZxZbyytCyqAhEREJy4tDu/PyskcxYtY0v3jGd5z4pCuX21SoCEZEQnT++H69cfyyDumfy7afmcvnDM9lZ1ba7ilQEIiIhG5iXyVNXHsX/O20Eb366hcse/pjd1Q3ejDkQKgIRkRiQmGBcdswA7pg8mhmrtjH1LzOprGmbMlARiIjEkDPH5PPrrx3GO8tK+OajbXMLaxWBiEiMOXdcX35+1kje/LSY2//v08BfT0UgIhKDLjiyH+cf0Zdp01cGPuSlikBEJEbdctoI8jt34jtPzw30TCIVgYhIjMpMTeK3545i7bZd/OKVxYG9jopARCSGTSjM5RsTB/Doh2uZvrQ4kNdQEYiIxLibThnK0B5ZLN4YzLDugQ1VKSIirSMtOZG/f2siacmJgTy/tghERNqBoEoAAiwCM3vAzLaY2YJG5ueY2T/MbK6ZLTSzS4PKIiIijQtyi+Ah4EtNzL8GWOTuo4ATgNvNLCXAPCIi0oDAisDdpwNNXQXhQJaZGZAZXTbc0RlEROJQmMcI/ggMBzYA84Hr3b3Bm2qY2VQzm2lmM4uLgzl9SkQkXoVZBKcAc4DewGjgj2aW3dCC7j7N3ce5+7i8vLy2zCgi0uGFWQSXAs95xHJgFTAsxDwiInEpzCJYC5wEYGY9gKHAyhDziIjEJQtqfEwze4LI2UDdgM3ArUAygLvfa2a9iZxZ1Asw4Jfu/mgznrcYWBP9NgcobeLrfaclAyUt/FH2fo7mzNt3WnMz7vmzWwsztlW+PdP0HsZWvvaQMdbzHUzGpqbF2nvY390b3rfu7u32AUxr6ut9pwEzD+Y1mjNv32nNzbjXny3K2Fb59B7GZr72kDHW8x1Mxv1kjan3sKlHe7+y+B/7+bqx+Qf6Gs2Zt++05maM9Xz7e62m6D3c/+s0ZX/rxXrGWM/X2PzmZNzftJYI+j1sVGC7hmKRmc1093Fh52hKrGeM9XwQ+xljPR/EfsZYzwftI+Me7X2LoKWmhR2gGWI9Y6zng9jPGOv5IPYzxno+aB8ZgTjbIhARkc+Lty0CERHZh4pARCTOqQhEROKciiDKzI41s3vN7H4zez/sPA0xswQz+5mZ/cHMLg47z77M7AQzeyf6Pp4Qdp6GmFlG9AaGp4WdpSFmNjz6/j1jZt8MO09DzOxMM/tfM/urmZ0cdp59mVmhmf3ZzJ4JO8se0X93D0fftylh59lXhyiCxgbBMbMvmdmnZrbczG5u6jnc/R13vwp4EXg4FjMCZwB9gBqgKAbzOVABpMVoPoDvAU+1ZrbWzOjui6P/DicDE2M04/PufgVwFXBeDOZb6e6XtWauhrQw69nAM9H37atBZ2uxllz5FqsP4DhgLLBgr2mJwAqgEEgB5gIjgJFEftnv/ei+13pPAVmxmBG4Gbgyuu4zMZgvIbpeD+CxGMz3ReB84BLgtFj8O46u81XgFeCCWM0YXe92YGwM52vV/yMHmfX7wOjoMo8HmetAHh1i8Hp3n25mBftMHg8sd/eVAGb2JHCGu/8CaHC3gJn1A0rdvTwWM5pZEVAd/bYu1vLtZTuQGmv5orurMoj8x9xtZi97I2NghJUx+jwvAC+Y2UvA462Vr7UyRgeT+iXwirt/Emv52kpLshLZQu5D5Nb7MbcnpkMUQSPygXV7fV8EHLmfdS4DHgws0ee1NONzwB/M7FhgepDBolqUz8zOJjLORGciAw8FrUX53P2HAGZ2CVDSmiXQhJa+hycQ2Y2QCrwcaLLPtPTf4bXAF4AcMxvk7vcGGY6Wv4e5wM+AMWb2/WhhtJXGsv6eyJgrX+HAb0ERmI5cBC3m7reGnaEp7r6LSFnFJHd/jkhZxTR3fyjsDI1x97eAt0KO0SR3/z2RX2wxyd23Ejl+ETPcfSeRMVhiUsxtorSi9UDfvb7vE50WS2I9o/IdPGU8eLGeb2/tKeu/deQi+BgYbGYDzCyFyEHCF0LOtK9Yz6h8B08ZD16s59tbe8r6mbCPVrfS0fsngI18dlrlZdHppwJLiRzF/6EyKp8yxnbGWM/XXrPu76GbzomIxLmOvGtIRESaQUUgIhLnVAQiInFORSAiEudUBCIicU5FICIS51QE0iGYWUUbv16rjFlhkTEcSs1sjpktMbPfNmOdM81sRGu8vgioCEQaZGZN3ofL3Y9uxZd7x91HA2OA08xsf+MQnEnkDqoirUJFIB2WmQ00s1fNbJZFRk4bFp1+upnNMLPZZvaGmfWITr/NzP5iZu8Bf4l+/4CZvWVmK83sur2euyL65wnR+c9EP9E/Fr1NM2Z2anTaLDP7vZm92FRed99N5DbF+dH1rzCzj81srpk9a2bpZnY0kfEKfhPdihjY2M8p0lwqAunIpgHXuvvhwHeBu6PT3wUmuPsY4Engv/daZwTwBXf/evT7YURurT0euNXMkht4nTHADdF1C4GJZpYG3Ad8Ofr6efsLa2ZdgMF8dovx59z9CHcfBSwmcguD94ncu+Ymdx/t7iua+DlFmkW3oZYOycwygaOBp6Mf0OGzwXL6AH81s15ERpFatdeqL0Q/me/xkrtXAVVmtoXI6Gv7DsP5kbsXRV93DlBAZMjOle6+57mfAKY2EvdYM5tLpATucvdN0emHmtlPiYzvkAm81sKfU6RZVATSUSUAO6L73vf1B+AOd38hOhDMbXvN27nPslV7fV1Hw/9nmrNMU95x99PMbADwoZk95e5zgIeAM919bnQwnRMaWLepn1OkWbRrSDokdy8DVpnZuRAZXtHMRkVn5/DZPeIvDijCp0DhXkMZ7neQ9+jWwy+B70UnZQEbo7ujpuy1aHl03v5+TpFmURFIR5FuZkV7Pb5N5JfnZdHdLguJjB0LkS2Ap81sFlASRJjo7qWrgVejr1MOlDZj1XuB46IF8v+AGcB7wJK9lnkSuCl6sHsgjf+cIs2i21CLBMTMMt29InoW0Z+AZe5+Z9i5RPalLQKR4FwRPXi8kMjuqPtCziPSIG0RiIjEOW0RiIjEORWBiEicUxGIiMQ5FYGISJxTEYiIxDkVgYhInPv/o9z57K+JjM0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.244206</td>\n",
       "      <td>0.146792</td>\n",
       "      <td>0.961500</td>\n",
       "      <td>02:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.118846</td>\n",
       "      <td>0.066031</td>\n",
       "      <td>0.980400</td>\n",
       "      <td>02:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.087754</td>\n",
       "      <td>0.046837</td>\n",
       "      <td>0.985900</td>\n",
       "      <td>02:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, lr_max=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet-ish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": "<h2 id=\"ConvLayer\" class=\"doc_header\"><code>class</code> <code>ConvLayer</code><a href=\"https://github.com/fastai/fastai/tree/master/fastai/layers.py#L237\" class=\"source_link\" style=\"float:right\">[source]</a></h2>\n\n> <code>ConvLayer</code>(**`ni`**, **`nf`**, **`ks`**=*`3`*, **`stride`**=*`1`*, **`padding`**=*`None`*, **`bias`**=*`None`*, **`ndim`**=*`2`*, **`norm_type`**=*`<NormType.Batch: 1>`*, **`bn_1st`**=*`True`*, **`act_cls`**=*`ReLU`*, **`transpose`**=*`False`*, **`init`**=*`'auto'`*, **`xtra`**=*`None`*, **`bias_std`**=*`0.01`*, **`dilation`**:`Union`\\[`int`, `Tuple`\\[`int`, `int`\\]\\]=*`1`*, **`groups`**:`int`=*`1`*, **`padding_mode`**:`str`=*`'zeros'`*, **`device`**=*`None`*, **`dtype`**=*`None`*) :: `Sequential`\n\nCreate a sequence of convolutional (`ni` to `nf`), ReLU (if `use_activ`) and `norm_type` layers.",
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ConvLayer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fastai.layers.ConvLayer"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ConvLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResBlock(Module):\n",
    "  def __init__(self, nf):\n",
    "    self.conv1 = ConvLayer(nf, nf)\n",
    "    self.conv2 = ConvLayer(nf, nf)\n",
    "\n",
    "  def forward(self, x): return self.conv2(self.conv1(x)) + x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(\n",
    "    conv2(1, 8),\n",
    "    ResBlock(8),\n",
    "    conv2(8, 16),\n",
    "    ResBlock(16),\n",
    "    conv2(16, 32),\n",
    "    ResBlock(32),\n",
    "    conv2(32, 16),\n",
    "    ResBlock(16),\n",
    "    conv2(16, 10),\n",
    "    Flatten()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): ConvLayer(\n",
       "    (0): Conv2d(1, 8, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (1): ResBlock(\n",
       "    (conv1): ConvLayer(\n",
       "      (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "    (conv2): ConvLayer(\n",
       "      (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (2): ConvLayer(\n",
       "    (0): Conv2d(8, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (3): ResBlock(\n",
       "    (conv1): ConvLayer(\n",
       "      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "    (conv2): ConvLayer(\n",
       "      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (4): ConvLayer(\n",
       "    (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (5): ResBlock(\n",
       "    (conv1): ConvLayer(\n",
       "      (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "    (conv2): ConvLayer(\n",
       "      (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (6): ConvLayer(\n",
       "    (0): Conv2d(32, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (7): ResBlock(\n",
       "    (conv1): ConvLayer(\n",
       "      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "    (conv2): ConvLayer(\n",
       "      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): ReLU()\n",
       "    )\n",
       "  )\n",
       "  (8): ConvLayer(\n",
       "    (0): Conv2d(16, 10, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (9): Flatten(full=False)\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_and_res(ni, nf): return nn.Sequential(conv2(ni, nf), ResBlock(nf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(\n",
    "    conv_and_res(1, 8),\n",
    "    conv_and_res(8, 16),\n",
    "    conv_and_res(16, 32),\n",
    "    conv_and_res(32, 16),\n",
    "    conv2(16, 10),\n",
    "    Flatten()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls, net, loss_func=CrossEntropyLossFlat(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.182298</td>\n",
       "      <td>0.399562</td>\n",
       "      <td>0.869300</td>\n",
       "      <td>02:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.091580</td>\n",
       "      <td>0.057553</td>\n",
       "      <td>0.983600</td>\n",
       "      <td>02:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.044026</td>\n",
       "      <td>0.028930</td>\n",
       "      <td>0.991000</td>\n",
       "      <td>02:13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, lr_max=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Resnet = x + conv2(conv1(x))\n",
    "\n",
    "DenseNet: cat(x.orig, conv2(conv1(x)) - Works well for small datasets and for segmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}