notebooks/04_Neural_Net_Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'fastai.models'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-6a6cf2ba1ce7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfastai\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimports\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mfastai\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtorch_imports\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfastai\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/envs/fastai/lib/python3.6/site-packages/fastai/torch_imports.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorchvision\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdensenet121\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdensenet161\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdensenet169\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdensenet201\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresnext_50_32x4d\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mresnext_50_32x4d\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresnext_101_32x4d\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mresnext_101_32x4d\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresnext_101_64x4d\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mresnext_101_64x4d\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'fastai.models'"
     ]
    }
   ],
   "source": [
    "from fastai.imports import *\n",
    "from fastai.torch_imports import *\n",
    "from fastai.io import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'data/mnsit'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below code will make directories, recursively in given path. If `exist_ok = True`, it will not raise exception error if tarket directory already exists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.makedirs(path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now I have directory where I will store data, let's download the data from source URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "URL='http://deeplearning.net/data/mnist/'\n",
    "FILENAME='mnist.pkl.gz' # see this is .gz .pkl file --> pickle file, use pickle load to load\n",
    "\n",
    "def load_mnsit(file):\n",
    "    return pickle.load(gzip.open(file), encoding='latin-1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is `fastai` function which will get file from URL and save in local repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'get_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-3166d1bfd182>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mURL\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mFILENAME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mFILENAME\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx_valid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_valid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_mnsit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mFILENAME\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'get_data' is not defined"
     ]
    }
   ],
   "source": [
    "get_data(URL+FILENAME, path + FILENAME)\n",
    "((x,y), (x_valid, y_valid), _) = load_mnsit(path + FILENAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's look at data. Understand dimensions, size of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_valid.shape # 28x28 pixel size and 10k images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_img = x_valid.reshape(-1,28,28)\n",
    "x_img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 8, 6, ..., 5, 6, 8])"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADl1JREFUeJzt3X3IXHV6xvHr8mUxMaLRVE1iNLtP\nhL5htAapKEVd3NqtEFdwMWBJoyVrWaGrVSpBUBTBlu5qK1SJGMzirltN3FVWRcXa+gZifGGNGzer\n4saYJ09Qi4mou43e/eM5WR7jzG8mM2fmTHJ/P/AwM+eeM+dmyJVzzvzOzM8RIQD57Nd0AwCaQfiB\npAg/kBThB5Ii/EBShB9IivADSRF+tGT7btvjtrfb3mj775ruCfUyF/mgFdt/IumNiPit7T+U9N+S\n/joiXmy2M9SFPT9aiojXIuK3ux5Wf2MNtoSaEX60Zfs/bH8s6XVJ45Iebrgl1IjDfhTZ3l/SqZLO\nkPTPEfF/zXaEurDnR1FEfBYRz0g6RtLfN90P6kP40a0DxDn/PoXw40tsH2n7QtszbO9v+y8lLZH0\nX033hvpwzo8vsf0HktZIWqjJHcRvJP17RNzRaGOoFeEHkuKwH0iK8ANJEX4gKcIPJHXAMDdmm08X\ngQGLCHfzvL72/LbPsf0r22/Yvrqf1wIwXD0P9VXXfG+UdLakzZJekLQkIn5ZWIc9PzBgw9jzn6LJ\n73u/FRG/k/QTSYv7eD0AQ9RP+OdKemfK483Vsi+wvdz2Otvr+tgWgJr184Ffq0OLLx3WR8RKSSsl\nDvuBUdLPnn+zpHlTHh8jaUt/7QAYln7C/4Kk421/1fZXJF0o6cF62gIwaD0f9kfETtuXSXpU0v6S\nVkXEa7V1BmCghvqtPs75gcEbykU+APZehB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK\n8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kNdQputGbhQsXFuuXX35529rY\n2Fhx3enTpxfrK1asKNYPPfTQYv2RRx5pW9uxY0dxXQwWe34gKcIPJEX4gaQIP5AU4QeSIvxAUoQf\nSIpZekfAjBkzivVNmzYV64cddlid7dTq3XffbVsrXZ8gSWvWrKm7nRS6naW3r4t8bL8taYekzyTt\njIhF/bwegOGp4wq/MyPivRpeB8AQcc4PJNVv+EPSY7ZftL281RNsL7e9zva6PrcFoEb9HvafFhFb\nbB8p6XHbr0fEU1OfEBErJa2U+MAPGCV97fkjYkt1u03STyWdUkdTAAav5/DbPtj2IbvuS/qGpPV1\nNQZgsHoe57f9NU3u7aXJ04cfR8SNHdbhsL+FQw45pFh/+OGHi/X333+/be3ll18urnvSSScV68cd\nd1yxPm/evGJ92rRpbWsTExPFdU899dRivdP6WQ18nD8i3pJU/pUJACOLoT4gKcIPJEX4gaQIP5AU\n4QeS4iu96MusWbOK9auuuqqnmiQtW7asWF+9enWxnlW3Q33s+YGkCD+QFOEHkiL8QFKEH0iK8ANJ\nEX4gKaboRl/ee6/8263PPvts21qncf5OXzdmnL8/7PmBpAg/kBThB5Ii/EBShB9IivADSRF+ICnG\n+dGXmTNnFusrVqzo+bXnzJnT87rojD0/kBThB5Ii/EBShB9IivADSRF+ICnCDyTF7/ajaOHC8kTM\n9913X7G+YMGCtrWNGzcW1z377LOL9XfeeadYz6q23+23vcr2Ntvrpyw73Pbjtn9d3Zav9AAwcro5\n7L9L0jm7Lbta0hMRcbykJ6rHAPYiHcMfEU9J+mC3xYsl7foNpdWSzqu5LwAD1uu1/UdFxLgkRcS4\n7SPbPdH2cknLe9wOgAEZ+Bd7ImKlpJUSH/gBo6TXob4J27MlqbrdVl9LAIah1/A/KGlpdX+ppAfq\naQfAsHQc57d9j6QzJM2SNCHpWkk/k3SvpGMlbZJ0QUTs/qFgq9fisH/ELF26tFi//vrri/V58+YV\n65988knb2rnnnltc98knnyzW0Vq34/wdz/kjYkmb0tf3qCMAI4XLe4GkCD+QFOEHkiL8QFKEH0iK\nn+7eB8yYMaNt7corryyue8011xTr++1X3j988EF5hPf0009vW3v99deL62Kw2PMDSRF+ICnCDyRF\n+IGkCD+QFOEHkiL8QFKM8+8D7rrrrra1888/v6/XXrNmTbF+yy23FOuM5Y8u9vxAUoQfSIrwA0kR\nfiApwg8kRfiBpAg/kBTj/PuAsbGxgb32bbfdVqw/99xzA9s2Bos9P5AU4QeSIvxAUoQfSIrwA0kR\nfiApwg8kxTj/PuCxxx5rW1u4cOHAXlvqfB3ATTfd1La2ZcuWnnpCPTru+W2vsr3N9vopy66z/a7t\nV6q/bw62TQB16+aw/y5J57RYfnNEnFj9PVxvWwAGrWP4I+IpSeU5mQDsdfr5wO8y27+oTgtmtnuS\n7eW219le18e2ANSs1/DfJmlM0omSxiV9v90TI2JlRCyKiEU9bgvAAPQU/oiYiIjPIuJzSXdIOqXe\ntgAMWk/htz17ysNvSVrf7rkARpMjovwE+x5JZ0iaJWlC0rXV4xMlhaS3JX0nIsY7bswubww9mTZt\nWtva3XffXVz35JNPLtaPPfbYnnraZevWrW1ry5YtK6776KOP9rXtrCLC3Tyv40U+EbGkxeI797gj\nACOFy3uBpAg/kBThB5Ii/EBShB9IquNQX60bY6hv6A466KBi/YADygM+27dvr7OdL/j000+L9Suu\nuKJYv/322+tsZ5/R7VAfe34gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIpxfhSdcMIJxfrNN99crJ95\n5pk9b3vTpk3F+vz583t+7X0Z4/wAigg/kBThB5Ii/EBShB9IivADSRF+ICnG+UfA9OnTi/WPP/54\nSJ3suZkz287UJklatWpV29rixYv72vbcuXOL9fHxjr8mv09inB9AEeEHkiL8QFKEH0iK8ANJEX4g\nKcIPJNVxll7b8yT9UNLRkj6XtDIi/s324ZL+U9J8TU7T/e2I+N/Btbr3GhsbK9afeeaZYv2hhx4q\n1tevX9+21mms+5JLLinWDzzwwGK901j7ggULivWSN998s1jPOo5fl272/Dsl/WNE/JGkP5f0Xdt/\nLOlqSU9ExPGSnqgeA9hLdAx/RIxHxEvV/R2SNkiaK2mxpNXV01ZLOm9QTQKo3x6d89ueL+kkSc9L\nOioixqXJ/yAkHVl3cwAGp+M5/y62Z0haK+l7EbHd7uryYdleLml5b+0BGJSu9vy2D9Rk8H8UEfdX\niydsz67qsyVta7VuRKyMiEURsaiOhgHUo2P4PbmLv1PShoj4wZTSg5KWVveXSnqg/vYADEo3h/2n\nSfobSa/afqVatkLSTZLutX2JpE2SLhhMi3u/Cy4ovzVHH310sX7xxRfX2c4e6XR6189Xwj/66KNi\n/dJLL+35tdFZx/BHxDOS2v0L+Hq97QAYFq7wA5Ii/EBShB9IivADSRF+ICnCDyTV9eW96N0RRxzR\ndAsDs3bt2mL9hhtuaFvbtq3lRaG/t3Xr1p56QnfY8wNJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUkzR\nPQSdfv76rLPOKtYvuuiiYn3OnDltax9++GFx3U5uvfXWYv3pp58u1nfu3NnX9rHnmKIbQBHhB5Ii\n/EBShB9IivADSRF+ICnCDyTFOD+wj2GcH0AR4QeSIvxAUoQfSIrwA0kRfiApwg8k1TH8tufZftL2\nBtuv2f6Havl1tt+1/Ur1983BtwugLh0v8rE9W9LsiHjJ9iGSXpR0nqRvS/ooIv61641xkQ8wcN1e\n5NNxxp6IGJc0Xt3fYXuDpLn9tQegaXt0zm97vqSTJD1fLbrM9i9sr7I9s806y22vs72ur04B1Krr\na/ttz5D0P5JujIj7bR8l6T1JIekGTZ4aXNzhNTjsBwas28P+rsJv+0BJP5f0aET8oEV9vqSfR8Sf\ndngdwg8MWG1f7LFtSXdK2jA1+NUHgbt8S9L6PW0SQHO6+bT/dElPS3pV0ufV4hWSlkg6UZOH/W9L\n+k714WDptdjzAwNW62F/XQg/MHh8nx9AEeEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeS\nIvxAUoQfSIrwA0kRfiCpjj/gWbP3JP1myuNZ1bJRNKq9jWpfEr31qs7ejuv2iUP9Pv+XNm6vi4hF\njTVQMKq9jWpfEr31qqneOOwHkiL8QFJNh39lw9svGdXeRrUvid561UhvjZ7zA2hO03t+AA0h/EBS\njYTf9jm2f2X7DdtXN9FDO7bftv1qNe14o/MLVnMgbrO9fsqyw20/bvvX1W3LORIb6m0kpm0vTCvf\n6Hs3atPdD/2c3/b+kjZKOlvSZkkvSFoSEb8caiNt2H5b0qKIaPyCENt/IekjST/cNRWa7X+R9EFE\n3FT9xzkzIv5pRHq7Tns4bfuAems3rfzfqsH3rs7p7uvQxJ7/FElvRMRbEfE7ST+RtLiBPkZeRDwl\n6YPdFi+WtLq6v1qT/3iGrk1vIyEixiPiper+Dkm7ppVv9L0r9NWIJsI/V9I7Ux5vVoNvQAsh6THb\nL9pe3nQzLRy1a1q06vbIhvvZXcdp24dpt2nlR+a962W6+7o1Ef5WUwmN0njjaRHxZ5L+StJ3q8Nb\ndOc2SWOanMNxXNL3m2ymmlZ+raTvRcT2JnuZqkVfjbxvTYR/s6R5Ux4fI2lLA320FBFbqtttkn6q\nydOUUTKxa4bk6nZbw/38XkRMRMRnEfG5pDvU4HtXTSu/VtKPIuL+anHj712rvpp635oI/wuSjrf9\nVdtfkXShpAcb6ONLbB9cfRAj2wdL+oZGb+rxByUtre4vlfRAg718wahM295uWnk1/N6N2nT3jVzh\nVw1l3CJpf0mrIuLGoTfRgu2vaXJvL01+3fnHTfZm+x5JZ2jyK58Tkq6V9DNJ90o6VtImSRdExNA/\neGvT2xnaw2nbB9Rbu2nln1eD712d093X0g+X9wI5cYUfkBThB5Ii/EBShB9IivADSRF+ICnCDyT1\n/x2VQ9c6BSuMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c2c511c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5,1,'3')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.imshow(x_img[0], cmap='gray')\n",
    "plt.show()\n",
    "plt.title(y_valid[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Little bit experimentation with pixels of picture to practice tensor subset. Let's not take all picture, some pixel values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAEICAYAAAByNDmmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAC51JREFUeJzt3V+IVnUex/HPp5nKUTcStr3wT2r2\nb91IrCEqIbYUtrZIiL0wKDJYpGUrkyhqCbroIhaibCGNWUuCpC4mLyIiFaqLCKRRg7IxESs1lZTY\nkjZTt+9ezBO4bs5zdM6vM8933i8QnPH07Yv49pznmeMZR4QA5HRG0wsAKIfAgcQIHEiMwIHECBxI\njMCBxAgcSIzAxzDbL9veZ/tb29tt/7npnVAvc6PL2GX7d5J2RMQPti+V9K6kmyNiU7OboS6cwcew\niNgaET/89GHrx6wGV0LNCHyMs73C9r8lbZO0T9KbDa+EGnGJDtnuknSNpN9L+ntEHG12I9SFMzgU\nEf+JiPckTZX0l6b3QX0IHMfrFq/BUyHwMcr2b2wvsj3RdpftP0i6XdLbTe+G+vAafIyyfZ6kfklz\nNPQX/ReS/hER/2x0MdSKwIHEuEQHEiNwIDECBxIjcCCx7hJDbfPOXSE9PT1F5s6aVebL32effXaR\nuYcPH6595vbt22ufKUlHj5a5MTAi3O6YIoGjnIsvvrjI3P7+/iJzL7zwwiJzS8S4YMGC2mdK0u7d\nu4vMrYJLdCAxAgcSI3AgMQIHEiNwIDECBxKrFLjtG21/anuH7UdKLwWgHm0Dbz3O5zlJN0maLel2\n27NLLwZg5Kqcwa/S0KN1d0bEEUmvSlpYdi0AdagS+BRJx9+Ks6f1uf9he4ntAdsDdS0HYGSq3Kr6\nc/e7/t+95hHRJ6lP4l50YLSocgbfI2nacR9PlbS3zDoA6lQl8A8kXWR7pu2zJC2S9HrZtQDUoe0l\nekQcs32vpHWSuiS9GBFbi28GYMQq/XPRiHhTfEsboONwJxuQGIEDiRE4kBiBA4kROJAYD10s5K67\n7ioy94knnigyd9q0ae0POg3ff/99kbn33HNP7TObfDhiKZzBgcQIHEiMwIHECBxIjMCBxAgcSIzA\ngcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHEOuqpqhMn\nTiwy96GHHqp95mOPPVb7TEk644wyfyd//fXXRebOmzevyNxt27YVmZsNZ3AgMQIHEiNwIDECBxIj\ncCAxAgcSI3AgsbaB255m+x3bg7a32l76SywGYOSq3OhyTNKDEbHZ9q8kbbK9ISI+KbwbgBFqewaP\niH0Rsbn180OSBiVNKb0YgJE7pVtVbc+QNFfSxp/5tSWSltSyFYBaVA7c9kRJr0l6ICK+PfHXI6JP\nUl/r2KhtQwCnrdK76LbP1FDcayJibdmVANSlyrvolvSCpMGIeLr8SgDqUuUMPk/SnZJusP1h68cf\nC+8FoAZtX4NHxHuS/AvsAqBm3MkGJEbgQGIEDiRG4EBiHfXQxZdeeqnI3Ntuu63I3BL6+/uLzH3m\nmWeKzOXhiM3iDA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBA\nYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJNZRT1WdNWtW0ys0bsWKFUXmvv/++0XmolmcwYHECBxI\njMCBxAgcSIzAgcQIHEiMwIHEKgduu8v2FttvlFwIQH1O5Qy+VNJgqUUA1K9S4LanSrpZ0qqy6wCo\nU9Uz+HJJD0v68WQH2F5ie8D2QC2bARixtoHbvkXSVxGxabjjIqIvInojore27QCMSJUz+DxJt9r+\nXNKrkm6w/XLRrQDUom3gEfFoREyNiBmSFkl6OyLuKL4ZgBHj6+BAYqf078Ej4l1J7xbZBEDtOIMD\niRE4kBiBA4kROJAYgQOJddRTVdevX19k7pw5c4rMLWHDhg1F5q5cubLI3CeffLLI3L179xaZmw1n\ncCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNw\nIDECBxIjcCAxAgcSI3AgMUdE/UPt+odK6unpKTFWa9asqX3mlVdeWftMSTr//POLzC1l//79ReYu\nXry49pnr1q2rfWZJEeF2x3AGBxIjcCAxAgcSI3AgMQIHEiNwIDECBxKrFLjtc233295me9D2NaUX\nAzByVb998LOS3oqIP9k+S9L4gjsBqEnbwG2fI+k6SYslKSKOSDpSdi0AdahyiX6BpAOSVtveYnuV\n7QknHmR7ie0B2wO1bwngtFQJvFvSFZJWRsRcSd9JeuTEgyKiLyJ6I6K35h0BnKYqge+RtCciNrY+\n7tdQ8ABGubaBR8R+SbttX9L61HxJnxTdCkAtqr6Lfp+kNa130HdKurvcSgDqUinwiPhQEq+tgQ7D\nnWxAYgQOJEbgQGIEDiRG4EBiHfVU1VLGjRtX+8zu7qpfgTw1hw4dKjK30xw+fLj2mcuWLat9piQ9\n//zzRebyVFVgjCNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDEC\nBxIjcCAxAgcSI3AgMQIHEiNwIDEeuthhLr/88iJzly9fXmTu9ddfX2RuCbt27Soyd/r06UXm8tBF\nYIwjcCAxAgcSI3AgMQIHEiNwIDECBxKrFLjtZba32v7Y9iu26/9ufQBq1zZw21Mk3S+pNyIuk9Ql\naVHpxQCMXNVL9G5JPba7JY2XtLfcSgDq0jbwiPhS0lOSdknaJ+mbiFh/4nG2l9gesD1Q/5oATkeV\nS/RJkhZKmilpsqQJtu848biI6IuI3ojorX9NAKejyiX6AkmfRcSBiDgqaa2ka8uuBaAOVQLfJelq\n2+NtW9J8SYNl1wJQhyqvwTdK6pe0WdJHrf+mr/BeAGrQXeWgiHhc0uOFdwFQM+5kAxIjcCAxAgcS\nI3AgMQIHEuOpqpAkTZo0qcjc1atXF5m7cOHCInNLmDx5cu0zDx48qCNHjvBUVWAsI3AgMQIHEiNw\nIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3Ag\nMQIHEiv1VNUDkr6ocOivJR2sfYFyOmnfTtpV6qx9R8Ou0yPivHYHFQm8KtsDEdHb2AKnqJP27aRd\npc7at5N25RIdSIzAgcSaDryv4f//qeqkfTtpV6mz9u2YXRt9DQ6grKbP4AAKInAgscYCt32j7U9t\n77D9SFN7tGN7mu13bA/a3mp7adM7VWG7y/YW2280vctwbJ9ru9/2ttbv8TVN7zQc28tafw4+tv2K\n7XFN7zScRgK33SXpOUk3SZot6Xbbs5vYpYJjkh6MiN9KulrSX0fxrsdbKmmw6SUqeFbSWxFxqaQ5\nGsU7254i6X5JvRFxmaQuSYua3Wp4TZ3Br5K0IyJ2RsQRSa9KGpXf8Dki9kXE5tbPD2noD+CUZrca\nnu2pkm6WtKrpXYZj+xxJ10l6QZIi4khE/KvZrdrqltRju1vSeEl7G95nWE0FPkXS7uM+3qNRHo0k\n2Z4haa6kjc1u0tZySQ9L+rHpRdq4QNIBSatbLydW2Z7Q9FInExFfSnpK0i5J+yR9ExHrm91qeE0F\n7p/53Kj+ep3tiZJek/RARHzb9D4nY/sWSV9FxKamd6mgW9IVklZGxFxJ30kaze/HTNLQleZMSZMl\nTbB9R7NbDa+pwPdImnbcx1M1ii91bJ+pobjXRMTapvdpY56kW21/rqGXPjfYfrnZlU5qj6Q9EfHT\nFVG/hoIfrRZI+iwiDkTEUUlrJV3b8E7DairwDyRdZHum7bM09EbF6w3tMizb1tBrxMGIeLrpfdqJ\niEcjYmpEzNDQ7+vbETEqzzIRsV/SbtuXtD41X9InDa7Uzi5JV9se3/pzMV+j+E1BaegS6RcXEcds\n3ytpnYbeiXwxIrY2sUsF8yTdKekj2x+2Pve3iHizwZ0yuU/SmtZf9Dsl3d3wPicVERtt90varKGv\nrmzRKL9tlVtVgcS4kw1IjMCBxAgcSIzAgcQIHEiMwIHECBxI7L9ryZoCpr5x+wAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1ca04438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5,1,'3')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.imshow(x_img[0,10:20,10:20], cmap='gray')\n",
    "plt.show()\n",
    "plt.title(y_valid[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plots6(ims, figsize = (10,6), rows = 2, titles=None):\n",
    "    f = plt.figure(figsize=figsize)\n",
    "    cols = len(ims)//rows\n",
    "    for i in range(len(ims)):\n",
    "        s = f.add_subplot(rows, cols, i+1)\n",
    "        if titles is not None: s.set_title(title[i])\n",
    "        plt.imshow(ims[i], cmap = 'gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkMAAAFpCAYAAACf5ixWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3X20VNV9//HPVx5qBCvgA16QBwUa\nq41IJWgqSawRFeMCbTSRaheiFrVaQ2oTKdXExNgYa4QsNWRhJGA0ulAEfGhUNKyCuhYR0RgI8tAU\nAbmCSBSIpDx9f38w/sKw9+XOvTN35pzZ79darnvvh3Pm7HOdr3w9s88+5u4CAABI1UG1HgAAAEAt\n0QwBAICk0QwBAICk0QwBAICk0QwBAICk0QwBAICk0QwBAICk0QwBAICkldUMmdm5ZrbczFaZ2fhK\nDQrIK2oCKEZNIA+stStQm1k7SSskDZO0TtKrkka5+28rNzwgP6gJoBg1gbxoX8a+QyStcvffSZKZ\nPSpppKQm3+RmxrM/UFHubrUewz6oCdQcNQEUK6UmyvmYrKektfv8vK6QAamiJoBi1ARyoZwrQ7FO\nK+jozWyspLFlHAfIC2oCKEZNIBfKaYbWSeq1z8/HSFq//0buPkXSFInLn6h71ARQjJpALpTzMdmr\nkgaY2bFm1lHSJZKerMywgFyiJoBi1ARyodVXhtx9l5ldL+k5Se0kTXX3pRUbGZAz1ARQjJpAXrT6\n1vpWHYzLn6iwjN0502LUBCqNmgCKtfXdZAAAALlHMwQAAJJGMwQAAJJWzq31AACgBjp37hzNr7zy\nyiAbOXJkkI0YMSK6/7Zt28obWE5xZQgAACSNZggAACSNZggAACSNZggAACSNCdQAAOTM6NGjo/nE\niRNL2v/EE0+M5gsXLmz1mPKMK0MAACBpNEMAACBpNEMAACBpNEMAACBpTKAu08CBA6P51772tSDr\n169fkB1yyCFBNmHChOhrHnbYYUH2i1/8Isi2bt0a3R8AkD+XX355kE2aNCm67c6dO4PsrrvuCrLF\nixeXPa56wpUhAACQNJohAACQNJohAACQNJohAACQtLImUJvZaklbJe2WtMvdB1diUEBeURNAMWoC\neWDu3vqd977JB7v7phK3b/3BMqBz585BtmbNmui2Xbp0aevhSJLeeeedIIvdySZJjz/+eFsPp+rc\n3Wo9hn2lVhMxTb33L7zwwiAbNGhQkA0dOjTIYrUnSZs3bw6yo48+Osjefffd6P7Tpk0Lsvvvvz/I\ndu/eHd0/i6iJfBsxYkSQzZo1K8g++uij6P7f/OY3g6zUR3TUq1Jqgo/JAABA0spthlzS82b2mpmN\nrcSAgJyjJoBi1AQyr9xFF0939/VmdpSkuWb2lrvP33eDwpufAkAqqAmgGDWBzCvrypC7ry983Shp\nlqQhkW2muPtgJs0hBdQEUIyaQB60+sqQmXWSdJC7by18f7ak71RsZBlkFs7BWrp0aXTb999/P8he\nf/31IItNIO3Tp0/0NXv16hVk3bp1C7I777wzuv+CBQuCbMOGDdFt0XIp1sQxxxwTZLNnz45uG3uv\nx2zZsiXIYrUjSR06dAiyDz74IMh69+4d3f++++4Lst///vdBNn/+/CCTpMbGxmiOvVKsiZbo2LFj\nkH3lK18JstjfPQsXLoy+ZuqTpVurnI/JukuaVfiX1F7Sz9392YqMCsgnagIoRk0gF1rdDLn77yTF\nn1IKJIiaAIpRE8gLbq0HAABJoxkCAABJK2sF6hYfLPGVRct1xBFHBNnXv/71kjJJGjNmTJBNnz69\n/IHVUNZW222pvNfE4sWLg2zgwPinIi+88EKQ3XjjjUG2aVO4UHFTK0iX6sgjj4zmv/jFL4Lsk5/8\nZJCNHz8+un9sAnatURP58e///u9BdttttwXZQw89FGRXXHFF9DV37dpV/sDqDCtQAwAANINmCAAA\nJI1mCAAAJI1mCAAAJI1mCAAAJK3cB7WiimJ32bz88stB1tTdZLHHIeT9bjJUT0NDQ5CdfPLJQTZj\nxozo/pdeemmQ7d69u/yBleC9996L5suXLw+yWJ3E6gwo1eDB8Ueu3XLLLUG2YsWKIIvdCVyt2kkF\nV4YAAEDSaIYAAEDSaIYAAEDSaIYAAEDSmECdI127dg2yCRMmlLx/jx49KjkcJCY2WdosXOV+/fr1\n0f1rOeHztNNOi+ajRo0Ksnnz5gVZ7Nwl6Y033ihvYKg7Bx0UXmNo6nEuHTt2DLKnnnoqyJgs3fa4\nMgQAAJJGMwQAAJJGMwQAAJLWbDNkZlPNbKOZLdkn62Zmc81sZeFrOJkFqFPUBFCMmkDembsfeAOz\nz0naJulBd/+rQnanpM3ufoeZjZfU1d1vavZgZgc+GCRJAwcOjOaPPfZYkPXv3z/IYiuYStKwYcOC\nbO3atS0cXba4eziDt41RE3+yZ8+eINu4cWN02yFDhgTZmjVrKj6mQw89NMheeeWV6LYrV64MsthK\n2ccdd1x0/6VLl7ZwdG2Pmqit2Ert77zzTsn733DDDUF27733ljWm1JVSE81eGXL3+ZI27xePlPTx\ncxymS7qgxaMDcoqaAIpRE8i71s4Z6u7ujZJU+HpU5YYE5BI1ARSjJpAbbb7OkJmNlTS2rY8D5AU1\nARSjJlBrrb0ytMHMGiSp8DU+SUCSu09x98HuHn9sL1AfqAmgGDWB3GhtM/SkpNGF70dLmlOZ4QC5\nRU0AxagJ5EYpd5M9IukMSUdI2iDpW5JmS5ohqbekNZIudvf9J8/FXivXdwm0hdGjRwfZd77znei2\nvXr1CrLt27cH2fnnnx/dP/aYgbyr0Z0z1ETBrbfeGmS33HJLdNvly5cH2TnnnBNk5d7h+PzzzwfZ\n5z//+ei2p5xySpAtWbIksmV+UBO1NWbMmCB74IEHotu+8MILQTZ8+PAg43Ec5SmlJpqdM+Tu4cN7\n9vpCi0cE1AFqAihGTSDvWIEaAAAkjWYIAAAkjWYIAAAkrdkJ1BU9WM4nxrVE586dg+xf//Vfg+zm\nm28OsoMOiveomzeHcw+HDh0aZG+99VYpQ6wLtZgsWkl5r4mDDz44yKZPnx7ZUrrooouCbNWqVUF2\nxhlnBFljY2P0NX/0ox8F2dix4XI1X//616P7T5w4MZrnGTVRPe3bh9Nuly1bFmR9+vSJ7n/ssccG\nWUse3YHSVORxHAAAAPWMZggAACSNZggAACSNZggAACStzR/Umqpp06YF2d/93d+VtO/jjz8ezSdN\nmhRkKU2WRvb88Y9/DLKrrroquu1RR4UPLY+tDP3f//3fQfbYY49FX/Oyyy4LspkzZwZZPU6URu3F\nbgro169fkF177bXR/Ws5Wfrcc8+N5iNGjAiyZ599NshiK71L8f8m5AFXhgAAQNJohgAAQNJohgAA\nQNJohgAAQNKYQN1GYpPoSjV58uRo/sorr7T6NYFq2bp1azQfOXJkkN16661BNm7cuCAbP358yce/\n5557St4WKEfv3r1L2q5jx45tPJIDu/zyy4Mstnq7FF9V/pprrgmyDz74ILr/7Nmzg+yKK65oZoS1\nx5UhAACQNJohAACQNJohAACQNJohAACQtGabITObamYbzWzJPtmtZvaOmb1R+Oe8th0mkB3UBFCM\nmkDelXI32TRJ90p6cL98orvfVfER1YnYUuUDBw5s9b5S/C6zO+64I8jWr19f0nHQatNETbTYli1b\nguyb3/xmkA0bNizITjjhhJKPc9ZZZwXZSy+9VPL+aJVpSrAm+vfvX9J21XxsUpcuXYLs7rvvDrLY\nXWOStGvXriCbO3dukA0dOjS6f+wROXVxN5m7z5e0uQpjAXKBmgCKURPIu3LmDF1vZm8WLo92rdiI\ngPyiJoBi1ARyobXN0GRJ/SSdLKlR0g+a2tDMxprZIjNb1MpjAXlATQDFqAnkRquaIXff4O673X2P\npPslDTnAtlPcfbC7D27tIIGsoyaAYtQE8qRVj+MwswZ3byz8eKGkJQfaPkWxxwzEJtudcsopQdbU\nEu/XX399kF100UVBNmbMmOj+zz33XDRH+aiJ1vnsZz8bZAMGDCjrNW+66aYge/vtt6Pb/vSnPy3r\nWGhaCjXRs2fPIHv33XeDLDYBua3EHr0Rm1T90EMPRff/4Q9/GGRr1qwJsmeffTa6/6c+9almRphN\nzTZDZvaIpDMkHWFm6yR9S9IZZnayJJe0WtLVbThGIFOoCaAYNYG8a7YZcvdRkfiBNhgLkAvUBFCM\nmkDesQI1AABIGs0QAABIWqsmUKN527dvD7JLL700yNq3D/8VxFbqbcrRRx8dZLNmzYpu+y//8i9B\n9uMf/7jkYwGV9rd/+7dB5u5BduGFF0b337w5XOfv6aefDrLY6u2StGnTpiB76qmnotsC+zv11FOD\nbMeOHTUYScs19aSCY445JsimTJkSZH/9138d3T+vN+pwZQgAACSNZggAACSNZggAACSNZggAACSN\nZggAACTNYndutNnBzKp3sBw76aSTovnEiRODLHY3TlNiS6r37du35P2zyN2t1mMoR0o1EXtfv/rq\nq0EWu/Nr3LhxJR/n4osvDrIHHoiv/2cWvn1OPPHEIIvVTlZRE9UTu8vq/PPPD7IePXpU/Nix964U\nr5Uf/KDJZ+SWJNYn/OhHP4puO2HChCDbunVrWccvVyk1wZUhAACQNJohAACQNJohAACQNJohAACQ\nNB7H0QKHHHJIkH300UcVP86bb74ZzS+66KIgmzp1apCNHDkyun/v3r2DrKGhIcgaGxubGyLQYoce\nemiQxR5H8/jjj5d1nMceeyzI+vTpE932+9//fpCdcsopQZanCdSorS5dugTZz372s+i2Dz30UJDF\nauKSSy4Jsm7dukVfc/jw4c0NUZL0hz/8IZq/9NJLQXbnnXcG2bx580o6Tl5wZQgAACSNZggAACSN\nZggAACSt2WbIzHqZ2TwzW2ZmS83sq4W8m5nNNbOVha9d2364QO1RE0AxagJ51+wK1GbWIKnB3Reb\n2aGSXpN0gaTLJW129zvMbLykru5+UzOvlZuVRfv16xdksYllzzzzTHT/JUuWBFlsYvKVV14ZZB06\ndIi+Zs+ePYOsf//+0W1j/ud//ifIBgwYUPL+WVSL1XZTrYlyXXfddUEWW6029j4v18EHHxzNf/3r\nXwfZ2rVrg+yss86q+JjaCjVRPddee22Q3XfffTUYyZ9s2bIlyGI3FXz3u9+N7v/2229XfEy1VpEV\nqN290d0XF77fKmmZpJ6SRkqaXthsuva+8YG6R00AxagJ5F2L5gyZWV9JgyQtlNTd3RulvYUg6ahK\nDw7IOmoCKEZNII9KXmfIzDpLmilpnLtvaeohcZH9xkoa27rhAdlFTQDFqAnkVUlXhsysg/a+wR92\n9ycK8YbC58Qff168Mbavu09x98HuPrgSAwaygJoAilETyLNS7iYzSQ9IWubud+/zR09KGl34frSk\nOZUfHpA91ARQjJpA3pXyMdnpkv5B0m/M7I1CNkHSHZJmmNmVktZIurhthlgbF18cns7RRx8dZFdc\ncUXFj93UpeXm7vz72LZt26L5Nddc0+oxoUiSNVGu2ONgfvWrX1Xl2Dt27Ijmv//974Pss5/9bJA1\n9eiDzZs3lzew+pFkTfz85z8Psthjk1auXBndv127diVlMQ8//HA0X716dZDF7iRGsWabIXd/SVJT\nH/x+obLDAbKPmgCKURPIO1agBgAASaMZAgAASaMZAgAASSt5naHUHH744bUeQmDmzJlBdttttwXZ\nxo3Ru1f17rvvVnxMQKliNwAMHTo0yC655JIg++Uvfxl9zc6dOwdZx44dg+z444+P7v/pT386yGKP\nU2CiNGI+/PDDIPvCF5gilUdcGQIAAEmjGQIAAEmjGQIAAEmjGQIAAEmzUlc1rsjBzKp3sDJ16NAh\nyM4888wgu+yyy6L79+jRI8hik+1i7rnnnmi+YMGCINu1a1dJr1mv3L20J0FmVJ5qolyjR48OsqlT\npwZZbAX2999/P/qapU6gbmpV95dffjnIRo4cGWR5mkBNTQDFSqkJrgwBAICk0QwBAICk0QwBAICk\n0QwBAICk0QwBAICkcTcZco07Z/Kje/fuQTZhwoQgiz2i4+STTy7r2DfffHM0j93NtmHDhrKOVWvU\nBFCMu8kAAACaQTMEAACSRjMEAACS1mwzZGa9zGyemS0zs6Vm9tVCfquZvWNmbxT+Oa/thwvUHjUB\nFKMmkHfNTqA2swZJDe6+2MwOlfSapAskfVnSNne/q+SDMTEOFVaLyaLUBLKMmgCKlVIT7Ut4kUZJ\njYXvt5rZMkk9yx8ekE/UBFCMmkDetWjOkJn1lTRI0sJCdL2ZvWlmU82sa4XHBmQeNQEUoyaQRyU3\nQ2bWWdJMSePcfYukyZL6STpZe/+P4AdN7DfWzBaZ2aIKjBfIDGoCKEZNIK9KWnTRzDpIelrSc+5+\nd+TP+0p62t3/qpnX4bNgVFStFpijJpBV1ARQrCKLLpqZSXpA0rJ93+CFCXMfu1DSktYMEsgbagIo\nRk0g70q5m2yopAWSfiNpTyGeIGmU9l76dEmrJV1dmER3oNei40dF1ejOGWoCmUVNAMVKqQmeTYZc\n4zlMQDFqAijGs8kAAACaQTMEAACSRjMEAACSRjMEAACSRjMEAACSRjMEAACSRjMEAACSRjMEAACS\n1r7Kx9sk6e3C90cUfq4n9XZOWT+fPrUeQAXUc03U2/lI2T+neqqJrP+uW4Nzqr6SaqKqK1AXHdhs\nkbsPrsnB20i9nVO9nU/W1dvvu97OR6rPc8qqevxdc07ZxcdkAAAgaTRDAAAgabVshqbU8Nhtpd7O\nqd7OJ+vq7fddb+cj1ec5ZVU9/q45p4yq2ZwhAACALOBjMgAAkLSqN0Nmdq6ZLTezVWY2vtrHrwQz\nm2pmG81syT5ZNzOba2YrC1+71nKMLWVmvcxsnpktM7OlZvbVQp7r88oDaiKbqInaoSayqZ5roqrN\nkJm1k3SfpOGSTpA0ysxOqOYYKmSapHP3y8ZLetHdB0h6sfBznuySdKO7/6Wk0yRdV/h3k/fzyjRq\nItOoiRqgJjKtbmui2leGhkha5e6/c/cdkh6VNLLKYyibu8+XtHm/eKSk6YXvp0u6oKqDKpO7N7r7\n4sL3WyUtk9RTOT+vHKAmMoqaqBlqIqPquSaq3Qz1lLR2n5/XFbJ60N3dG6W9bxhJR9V4PK1mZn0l\nDZK0UHV0XhlFTeQANVFV1EQO1FtNVLsZskjG7WwZYmadJc2UNM7dt9R6PAmgJjKOmqg6aiLj6rEm\nqt0MrZPUa5+fj5G0vspjaCsbzKxBkgpfN9Z4PC1mZh209w3+sLs/UYhzf14ZR01kGDVRE9REhtVr\nTVS7GXpV0gAzO9bMOkq6RNKTVR5DW3lS0ujC96MlzanhWFrMzEzSA5KWufvd+/xRrs8rB6iJjKIm\naoaayKh6romqL7poZudJmiSpnaSp7n57VQdQAWb2iKQztPdpvRskfUvSbEkzJPWWtEbSxe6+/+S5\nzDKzoZIWSPqNpD2FeIL2fh6c2/PKA2oim6iJ2qEmsqmea4IVqAEAQNJYgRoAACSNZggAACSNZggA\nACSNZggAACSNZggAACSNZggAACSNZggAACSNZggAACSNZggAACSNZggAACSNZggAACSNZggAACSt\nrGbIzM41s+VmtsrMxldqUEBeURNAMWoCedDqp9abWTtJKyQNk7RO0quSRrn7bys3PCA/qAmgGDWB\nvGhfxr5DJK1y999Jkpk9KmmkpCbf5GbWus4LaIK7W63HsA9qAjVHTQDFSqmJcj4m6ylp7T4/rytk\nQKqoCaAYNYFcKOfKUKzTCjp6MxsraWwZxwHygpoAilETyIVymqF1knrt8/Mxktbvv5G7T5E0ReLy\nJ+oeNQEUoyaQC+V8TPaqpAFmdqyZdZR0iaQnKzMsIJeoCaAYNYFcaPWVIXffZWbXS3pOUjtJU919\nacVGBuQMNQEUoyaQF62+tb5VB+PyJyosY3fOtBg1gUqjJoBibX03GQAAQO7RDAEAgKTRDAEAgKTR\nDAEAgKTRDAEAgKTRDAEAgKTRDAEAgKTRDAEAgKSV82wyACjL8OHDg+xrX/takA0bNiy6f2zR2JUr\nVwbZjBkzovtPnjw5yNavDx6dBaDOcWUIAAAkjWYIAAAkjWYIAAAkjWYIAAAkjafWI9d4Qnd+XHvt\ntUE2ceLEIOvYsWM1hiNJmjdvXpBddtllQdbY2FiN4VQENQEU46n1AAAAzaAZAgAASaMZAgAASaMZ\nAgAASStrBWozWy1pq6Tdkna5++BKDArIK2oCKEZNIA/Kupus8CYf7O6bStyeuwRQUVm7c4aakL74\nxS9G89gjMT7xiU8E2euvvx5k48ePj77m0qVLSxrTlVdeGc2//e1vB9m9994bZDfccENJx8kCaiJ7\nOnXqFM0nTJgQZDfffHOQxf6evu2226KvOXDgwCAbMWJEc0Osa9xNBgAA0IxymyGX9LyZvWZmYysx\nICDnqAmgGDWBzCv3qfWnu/t6MztK0lwze8vd5++7QeHNTwEgFdQEUIyaQOaVdWXI3dcXvm6UNEvS\nkMg2U9x9MJPmkAJqAihGTSAPWn1lyMw6STrI3bcWvj9b0ncqNjIgZ1KsifPPPz/IHnnkkei2scnS\ns2fPDrLYYzs2bNjQitH9yXe/+91oHnvMxtlnn13WsfAnKdZEzOGHHx7NYxOoZ86cGWSLFy8u+Vif\n+9zngqx79+5BVm5N1ZtyPibrLmmWmX38Oj9392crMiogn6gJoBg1gVxodTPk7r+TFN7DBySKmgCK\nURPIC26tBwAASaMZAgAASSv31vrkNTQ0RPN/+qd/KinbuXNnkK1Zsyb6mrfffnuQxSbWrV27Nro/\nUI727cP/XMRWcG5qtd0333wzyK6++uoge++991oxugNraqX9n/zkJ0E2a9asih8faevbt2/FXzP2\nd4ckHXbYYUF2wgknBBkTqItxZQgAACSNZggAACSNZggAACSNZggAACSNZggAACSNu8la4Ljjjguy\nyZMnR7cdNmxYq49z1FFHRfPYXS7/93//F2Snn356dP+WLOkO7O8f//Efg2zQoEFBFntPStLll18e\nZG1x51i53n///VoPAXXmM5/5TMVfc86cOdE8dofn4MHhI9/mzZtX8THlGVeGAABA0miGAABA0miG\nAABA0miGAABA0phA3YSePXsG2ZIlS4Is9ogCSZo4cWKQ3XPPPSUd5/jjj4++5n/+538GWZcuXYJs\nxowZ0f1PO+20INu0aVN0W2B///zP/1zSdtdcc000f+ONNyo5HCCT2rVrF2Rf+tKXotvu2bMnyHbt\n2lXxMaF5XBkCAABJoxkCAABJoxkCAABJa7YZMrOpZrbRzJbsk3Uzs7lmtrLwtWvbDhPIDmoCKEZN\nIO9KmUA9TdK9kh7cJxsv6UV3v8PMxhd+vqnyw6udb3zjG0EWmxh31VVXRfd/8MEHo/n+Vq9eHWQv\nv/xydNuDDz44yCZNmhRksZWyJWn+/PlBFptUvWXLluj++P+mKcGaKNW6detqPQRU3zRRE5Kk7t27\nB9mnP/3p6Lb/+7//G2RvvvlmScfZuXNnNN+9e3eQ9e/fv6TXTFmzV4bcfb6kzfvFIyVNL3w/XdIF\nFR4XkFnUBFCMmkDetXbOUHd3b5Skwtf4w7SAdFATQDFqArnR5usMmdlYSWPb+jhAXlATQDFqArXW\n2itDG8ysQZIKXzc2taG7T3H3we4ePjYXqB/UBFCMmkButLYZelLS6ML3oyXNqcxwgNyiJoBi1ARy\nw9z9wBuYPSLpDElHSNog6VuSZkuaIam3pDWSLnb3/SfPxV7rwAergT//8z+P5itXrgyyn/70p0E2\nfvz4io+pJZYvXx5kAwYMKHn/2GNDbrzxxrLGVE3ubtU+Zr3XxEknnRTNX3vttSD76KOPguzEE0+M\n7s9dZtVBTdRWjx49guydd96Jbvv8888H2TnnnFPScTp27BjNY38nbNwYXpQ79dRTSzpOPSilJpqd\nM+Tuo5r4oy+0eERAHaAmgGLUBPKOFagBAEDSaIYAAEDSaIYAAEDS2nydoaw7/vjjo/mRRx4ZZHPn\nzm3r4bRYbAL3E088Ed02Nll+zJgxQfYf//Ef0f3ff//9Fo4OedS+ffw/C7HH0cQmUDNRGik788wz\nS942dgNLqVpSpw0NDUHW1M1DqT6OiStDAAAgaTRDAAAgaTRDAAAgaTRDAAAgaclPoB40aFDJ277+\n+uttOJLW+a//+q8gW7VqVXTbfv36Bdkf//jHIPvDH/5Q/sCAVjr88MOD7Pzzz49uW+pq6atXr47m\nffv2DbJ33303yB5//PHo/rFV6Xfu3FnSmFCf/uZv/ibINmzYEN12wYIFrT5O7OYFSXrmmWeC7Jpr\nrgmyww47LLo/E6gBAAASRDMEAACSRjMEAACSRjMEAACSlvwE6pdeeima79mzJ8hiK1A3NbGzsbGx\nvIGV6JOf/GSQHXzwwdFtzznnnCA75JBDgiw2qRqI6datW5ANHjw4uu2iRYuCrH///kH2wgsvBFnv\n3r2jr7l9+/Yg+/Wvfx1kTU2gjuWxVdnPOuus6P6xmvrSl74U3Rb1p1OnTkF23nnnBdmOHTui+7fF\nzSoffPBBxV8zBVwZAgAASaMZAgAASaMZAgAASaMZAgAASWu2GTKzqWa20cyW7JPdambvmNkbhX/C\nGWNAnaImgGLUBPKulLvJpkm6V9KD++UT3f2uio+oypYuXRrNn3766SAbOXJkkC1btiy6/7PPPhtk\nM2fODLJf/vKXQdazZ8/oa8buHJs0aVKQNTQ0RPfftWtXkM2ZMye6LQ5omuq4JjZv3hzNP/zwwyCL\nLenf1DL/xx13XJDF3v/HHHNMkL344ovR17zuuuuCbMWKFdFtS/Xkk08G2axZs6LbHn/88WUdq45M\nUx3XRFNid+P26dMnyNauXVuN4UiK12lMU3VazbFmSbNXhtx9vqT4fx2BBFETQDFqAnlXzpyh683s\nzcLl0a4VGxGQX9QEUIyaQC60thmaLKmfpJMlNUr6QVMbmtlYM1tkZuGKa0D9oCaAYtQEcqNVzZC7\nb3D33e6+R9L9koYcYNsp7j6I4QGGAAAKV0lEQVTY3ePL0gJ1gJoAilETyJNWPY7DzBrc/ePnTVwo\nacmBts+jUaNGBdn3vve9ILvhhhui+3/5y18uKYtNVo094qAS2up1UV810dSjK2KPmIlNwvz7v//7\n6P4nnHBCkMUmS8cex3HhhRdGX7MtHmcQO/5PfvKT6LZnn312xY9fL+qpJsrVsWPHaH7KKacEWexx\nSLG/Jz7xiU9EX9PdSxrT5MmTo/mZZ54ZZDt37izpNfOs2WbIzB6RdIakI8xsnaRvSTrDzE6W5JJW\nS7q6DccIZAo1ARSjJpB3zTZD7h5eIpEeaIOxALlATQDFqAnkHStQAwCApNEMAQCApLVqAnUKtm/f\nHmTjxo0LshkzZkT3v/TSS0s6Tvfu3UseU2wS26mnnhpkxx57bHT/jz76qORjAfuLrVYeW4F5zJgx\nJb9mbGJyrM5q/d5tqk5jq7337t07yNasWVPxMSE/mnr/LFoUriQQe1LAtm3bgqypSdmxVbFjhg4d\nGs2/+MUvBtns2bNLes0848oQAABIGs0QAABIGs0QAABIGs0QAABIGs0QAABIGneTlemVV15pUV5p\nP/vZz4Ksb9++0W13797dxqNBPfv+978fZLHH1sTupmrKb3/72yCr9Z1jMbFHFEjSn/3ZnwVZU49J\nQP2JPSYj9timf/u3fyv5Ndu3D/9a7tKlS8sGVoLYnWyS9NRTT1X8WHnAlSEAAJA0miEAAJA0miEA\nAJA0miEAAJA0JlAnpF+/frUeAnLsgw8+CLLrrrsuyB599NHo/p06dQqy2267Lchij5i5/fbbo6+5\nZMmSaF6O4cOHB9nhhx8e3XbFihVBtnz58oqPCdkUuynllltuCbJ77rknun+sJmLvv9iNBrFMkj71\nqU8F2XPPPRdkO3bsiO6f6o02XBkCAABJoxkCAABJoxkCAABJa7YZMrNeZjbPzJaZ2VIz+2oh72Zm\nc81sZeFr17YfLlB71ARQjJpA3pUygXqXpBvdfbGZHSrpNTObK+lySS+6+x1mNl7SeEk3td1QEbNt\n27ZaDyFF1ETBM888E2QXXXRRdNvYCtYnnXRSkH3lK18JshEjRkRf86qrrgqyNWvWBNnq1auj+w8d\nOjTIfvjDHwZZU5NKf/WrX0XzBFETBbH3SmNjY8n7NzXZulQ9evQoa/9UNXtlyN0b3X1x4futkpZJ\n6ilppKTphc2mS7qgrQYJZAk1ARSjJpB3LZozZGZ9JQ2StFBSd3dvlPYWgqSjKj04IOuoCaAYNYE8\nKnmdITPrLGmmpHHuvsXMSt1vrKSxrRsekF3UBFCMmkBelXRlyMw6aO8b/GF3f6IQbzCzhsKfN0ja\nGNvX3ae4+2B3H1yJAQNZQE0AxagJ5Fkpd5OZpAckLXP3u/f5oycljS58P1rSnMoPD8geagIoRk0g\n78zdD7yB2VBJCyT9RtKeQjxBez8PniGpt6Q1ki52983NvNaBD4YWGzVqVJA9/PDD0W2//e1vl5Tl\nibuXdh2+gqiJ1jnyyCOD7Iorrgiyb3zjG0HWtWv17sjetWtXkDX1OJAs1g81kbYjjjgiyN56660g\na+oOyb/4i78Isg8//LD8gdVQKTXR7Jwhd39JUlMv9IWWDgrIO2oCKEZNIO9YgRoAACSNZggAACSN\nZggAACSt5HWGkE0HHRT2s02t7bFxY/SuVqAq3nvvvSCLPaLj/vvvD7Jrr702+pqxR38MHDiw5DGt\nXbs2yH784x8H2fe+972SXxOopU2bNgXZihUrguwzn/lMdP9DDjkkyPI+gboUXBkCAABJoxkCAABJ\noxkCAABJoxkCAABJYwJ1zu3ZsyfImltVHMiyzZvDBYqbWgG6qRzAnzz22GNB1tQE6iFDhgTZnDn1\n/xQVrgwBAICk0QwBAICk0QwBAICk0QwBAICk0QwBAICkcTdZQs4+++wgmzx5cg1GAgColldeeaXk\nbceNGxdk3E0GAABQ52iGAABA0miGAABA0ppthsysl5nNM7NlZrbUzL5ayG81s3fM7I3CP+e1/XCB\n2qMmgGLUBPKulAnUuyTd6O6LzexQSa+Z2dzCn01097vabnhozrZt20retn175stXCDUBFKMmMmzh\nwoVBZmY1GEl2Nfu3o7s3SmosfL/VzJZJ6tnWAwOyipoAilETyLsWzRkys76SBkn6uM283szeNLOp\nZta1wmMDMo+aAIpRE8ijkpshM+ssaaakce6+RdJkSf0knay9/0fwgyb2G2tmi8xsUQXGC2QGNQEU\noyaQVyU1Q2bWQXvf4A+7+xOS5O4b3H23u++RdL+kIbF93X2Kuw9298GVGjRQa9QEUIyaQJ41O2fI\n9s6yekDSMne/e5+8ofA5sSRdKGlJ2wwRB7JgwYKStz3zzDPbcCTpoCaAYtQE8q6U24tOl/QPkn5j\nZm8UsgmSRpnZyZJc0mpJV7fJCIHsoSaAYtQEcs3cvXoHM6vewRLRpUuXINu8eXN02+3btwdZp06d\nKj6manL3XN8fSk2g0qgJoFgpNcEK1AAAIGk0QwAAIGk0QwAAIGnMGUKuMT8CKEZNAMWYMwQAANAM\nmiEAAJA0miEAAJA0miEAAJC0UlagrqRNkt4ufH9E4ed6Um/nlPXz6VPrAVRAPddEvZ2PlP1zqqea\nyPrvujU4p+orqSaqejdZ0YHNFtXbQ/nq7Zzq7Xyyrt5+3/V2PlJ9nlNW1ePvmnPKLj4mAwAASaMZ\nAgAASatlMzSlhsduK/V2TvV2PllXb7/vejsfqT7PKavq8XfNOWVUzeYMAQAAZAEfkwEAgKRVvRky\ns3PNbLmZrTKz8dU+fiWY2VQz22hmS/bJupnZXDNbWfjatZZjbCkz62Vm88xsmZktNbOvFvJcn1ce\nUBPZRE3UDjWRTfVcE1VthsysnaT7JA2XdIKkUWZ2QjXHUCHTJJ27XzZe0ovuPkDSi4Wf82SXpBvd\n/S8lnSbpusK/m7yfV6ZRE5lGTdQANZFpdVsT1b4yNETSKnf/nbvvkPSopJFVHkPZ3H2+pM37xSMl\nTS98P13SBVUdVJncvdHdFxe+3yppmaSeyvl55QA1kVHURM1QExlVzzVR7Waop6S1+/y8rpDVg+7u\n3ijtfcNIOqrG42k1M+sraZCkhaqj88ooaiIHqImqoiZyoN5qotrNkEUybmfLEDPrLGmmpHHuvqXW\n40kANZFx1ETVURMZV481Ue1maJ2kXvv8fIyk9VUeQ1vZYGYNklT4urHG42kxM+ugvW/wh939iUKc\n+/PKOGoiw6iJmqAmMqxea6LazdCrkgaY2bFm1lHSJZKerPIY2sqTkkYXvh8taU4Nx9JiZmaSHpC0\nzN3v3uePcn1eOUBNZBQ1UTPUREbVc01UfdFFMztP0iRJ7SRNdffbqzqACjCzRySdob1P690g6VuS\nZkuaIam3pDWSLnb3/SfPZZaZDZW0QNJvJO0pxBO09/Pg3J5XHlAT2URN1A41kU31XBOsQA0AAJLG\nCtQAACBpNEMAACBpNEMAACBpNEMAACBpNEMAACBpNEMAACBpNEMAACBpNEMAACBp/w/JhvSeKqqH\n7QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1c1aad30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plots6(x_img[:6])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pytorch practice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random input and output data\n",
    "x = torch.randn(N, D_in).type(dtype)\n",
    "y = torch.randn(N, D_out).type(dtype)\n",
    "\n",
    "# Randomly initialize weights\n",
    "w1 = torch.randn(D_in, H).type(dtype)\n",
    "w2 = torch.randn(H, D_out).type(dtype)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "loss_list = []\n",
    "\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    loss_list.append((t, loss))\n",
    "\n",
    "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEDCAYAAADOc0QpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEpRJREFUeJzt3X9s3PV9x/HXq8YFr1R1adwVTIJp\nhzLoUhJ6omFME2M/AgyRrKNqUlbWjS2ioxtsKBPpJKYhpnaKBIzSwdKCKBMCBmQuQ3QpAyaoRGkv\nsUOSZSkpoyJOtLiASRkeJOG9P+57wXHux9f22efvx8+HdPLd5/vx3fvjXl9cPt/PfT+OCAEA0vKe\ndhcAAGg9wh0AEkS4A0CCCHcASBDhDgAJItwBIEFtDXfbd9neZ3tbjr432x7Mbj+yPTITNQJAEbmd\n69xt/6qkNyTdExG/NIHf+1NJSyLiD6etOAAosLZ+co+IpyW9OrbN9sds/5vtTbafsf2LNX51laT7\nZqRIACigY9pdQA3rJV0ZES/Y/pSkf5B0fvWg7VMknSrpyTbVBwCz3qwKd9vHS/plSQ/arjYfO67b\nSkkPRcShmawNAIpkVoW7KtNEIxGxuEGflZKumqF6AKCQZtVSyIjYL+m/bX9GklxxZvW47YWSPijp\n2TaVCACF0O6lkPepEtQLbe+2fYWkyyRdYXuLpO2Slo/5lVWS7g8uZQkADbV1KSQAYHrMqmkZAEBr\ntO2E6rx586Kvr69dLw8AhbRp06afRkRPs35tC/e+vj6Vy+V2vTwAFJLtn+Tpx7QMACSIcAeABBHu\nAJAgwh0AEkS4A0CCZtu1ZRrqHxjSuo07tWdkVCd1d2nNsoVasaS33WUBwKxTmHDvHxjS2g1bNXqg\ncjHIoZFRrd2wVZIIeAAYpzDTMus27jwc7FWjBw5p3cadbaoIAGavwoT7npHRCbUDwFxWmHA/qbtr\nQu0AMJcVJtzXLFuors6OI9q6Oju0ZtnCNlUEALNXYU6oVk+asloGAJorTLhLlYAnzAGgucJMywAA\n8iPcASBBTcPd9nzbT9neYXu77atr9DnP9uu2B7Pb9dNTLgAgjzxz7gclXRsRm22/X9Im249HxH+O\n6/dMRFzc+hIBABPV9JN7ROyNiM3Z/Z9J2iGJs5oAMItNaM7ddp+kJZKeq3H4HNtbbH/H9sfr/P5q\n22Xb5eHh4QkXCwDIJ3e42z5e0sOSromI/eMOb5Z0SkScKelrkvprPUdErI+IUkSUenqa7u8KAJik\nXOFuu1OVYL83IjaMPx4R+yPijez+Y5I6bc9raaUAgNzyrJaxpDsl7YiIm+r0+UjWT7bPzp73lVYW\nCgDIL89qmXMlfV7SVtuDWduXJS2QpIi4Q9Klkr5o+6CkUUkrIyKmoV4AQA5Nwz0ivifJTfrcJum2\nVhUFAJgavqEKAAki3AEgQYQ7ACSIcAeABBHuAJAgwh0AEkS4A0CCCHcASBDhDgAJItwBIEGEOwAk\niHAHgAQR7gCQIMIdABJEuANAggh3AEgQ4Q4ACSLcASBBhDsAJIhwB4AEEe4AkCDCHQASRLgDQIII\ndwBIEOEOAAki3AEgQce0u4CJ6h8Y0rqNO7VnZFQndXdpzbKFWrGkt91lAcCsUqhw7x8Y0toNWzV6\n4JAkaWhkVGs3bJUkAh4AxijUtMy6jTsPB3vV6IFDWrdxZ5sqAoDZqWm4255v+ynbO2xvt311jT62\nfavtXbaft33WdBS7Z2R0Qu0AMFfl+eR+UNK1EXG6pKWSrrJ9xrg+F0o6LbutlnR7S6vMnNTdNaF2\nAJirmoZ7ROyNiM3Z/Z9J2iFp/AT3ckn3RMX3JXXbPrHVxa5ZtlBdnR1HtHV1dmjNsoWtfikAKLQJ\nnVC13SdpiaTnxh3qlfTymMe7s7a9435/tSqf7LVgwYKJVap3T5qyWgYAGssd7raPl/SwpGsiYv/4\nwzV+JY5qiFgvab0klUqlo47nsWJJL2EOAE3kWi1ju1OVYL83IjbU6LJb0vwxj0+WtGfq5QEAJiPP\nahlLulPSjoi4qU63RyRdnq2aWSrp9YjYW6cvAGCa5ZmWOVfS5yVttT2YtX1Z0gJJiog7JD0m6SJJ\nuyS9KekPWl8qACCvpuEeEd9T7Tn1sX1C0lWtKgoAMDWF+oYqACAfwh0AEkS4A0CCCHcASBDhDgAJ\nItwBIEGEOwAkiHAHgAQR7gCQIMIdABJEuANAgia0Wcds0T8wxIYdANBA4cK9f2BIazds1eiBQ5Kk\noZFRrd2wVZIIeADIFG5aZt3GnYeDvWr0wCGt27izTRUBwOxTuHDfMzI6oXYAmIsKF+4ndXdNqB0A\n5qLChfuaZQvV1dlxRFtXZ4fWLFvYpooAYPYp3AnV6klTVssAQH2FC3epEvCEOQDUV7hpGQBAc4Q7\nACSIcAeABBHuAJAgwh0AEkS4A0CCCHcASBDhDgAJItwBIEFNw932Xbb32d5W5/h5tl+3PZjdrm99\nmQCAichz+YG7Jd0m6Z4GfZ6JiItbUlEO7MQEAI01DfeIeNp23/SXkg87MQFAc62acz/H9hbb37H9\n8XqdbK+2XbZdHh4entQLsRMTADTXinDfLOmUiDhT0tck9dfrGBHrI6IUEaWenp5JvRg7MQFAc1MO\n94jYHxFvZPcfk9Rpe96UK6uDnZgAoLkph7vtj9h2dv/s7Dlfmerz1sNOTADQXNMTqrbvk3SepHm2\nd0v6a0mdkhQRd0i6VNIXbR+UNCppZUTEdBXMTkwA0JynMYcbKpVKUS6X2/LaAFBUtjdFRKlZP76h\nCgAJItwBIEGEOwAkiHAHgAQR7gCQIMIdABJEuANAggh3AEgQ4Q4ACSLcASBBhDsAJCjPNnuzElvt\nAUB9hQx3ttoDgMYKOS3DVnsA0Fghw52t9gCgsUKGO1vtAUBjhQx3ttoDgMYKeUKVrfYAoLFChrtU\nCXjCHABqK+S0DACgMcIdABJEuANAggh3AEgQ4Q4ACSLcASBBhDsAJIhwB4AEEe4AkCDCHQAS1DTc\nbd9le5/tbXWO2/attnfZft72Wa0vEwAwEXk+ud8t6YIGxy+UdFp2Wy3p9qmXBQCYiqYXDouIp233\nNeiyXNI9ERGSvm+72/aJEbG3RTXWxB6qAFBfK+bceyW9PObx7qztKLZX2y7bLg8PD0/6Bat7qA6N\njCr07h6q/QNDk35OAEhJK8LdNdqiVseIWB8RpYgo9fT0TPoF2UMVABprRbjvljR/zOOTJe1pwfPW\nxR6qANBYK8L9EUmXZ6tmlkp6fbrn29lDFQAay7MU8j5Jz0paaHu37StsX2n7yqzLY5JelLRL0jck\n/cm0VZthD1UAaCzPaplVTY6HpKtaVlEO7KEKAI2xhyoAJIjLDwBAggh3AEgQ4Q4ACSLcASBBhDsA\nJIhwB4AEEe4AkCDCHQASRLgDQIIIdwBIEOEOAAkq7LVlJLbaA4B6Chvu1a32qjsyVbfak0TAA5jz\nCjstw1Z7AFBfYcOdrfYAoL7Chjtb7QFAfYUNd7baA4D6CntCla32AKC+woa7xFZ7AFBPYadlAAD1\nEe4AkCDCHQASRLgDQIIIdwBIEOEOAAki3AEgQYVe5y5x2V8AqCXXJ3fbF9jeaXuX7etqHP+C7WHb\ng9ntj1pf6tGql/0dGhlV6N3L/vYPDM3EywPArNU03G13SPq6pAslnSFple0zanR9ICIWZ7dvtrjO\nmrjsLwDUlueT+9mSdkXEixHxtqT7JS2f3rLy4bK/AFBbnnDvlfTymMe7s7bxftf287Yfsj2/1hPZ\nXm27bLs8PDw8iXKPxGV/AaC2POHuGm0x7vG/SuqLiE9I+ndJ36r1RBGxPiJKEVHq6emZWKU1cNlf\nAKgtT7jvljT2k/jJkvaM7RARr0TEW9nDb0j6ZGvKa2zFkl595dOL1NvdJUvq7e7SVz69iNUyAOa8\nPEshfyjpNNunShqStFLS58Z2sH1iROzNHl4iaUdLq2yAy/4CwNGahntEHLT9JUkbJXVIuisittu+\nQVI5Ih6R9Ge2L5F0UNKrkr4wjTUDAJpwxPjp85lRKpWiXC635bUBoKhsb4qIUrN+XH4AABJEuANA\ngri2DAAkqNDhXr22TPUSBNVry0gi4AHMaYWeluHaMgBQW6HDnWvLAEBthQ53ri0DALUVOty5tgwA\n1FboE6rVk6aslgGAIxU63CWuLQMAtRQ+3CXWugPAeIUPd9a6A8DRCn1CVWKtOwDUUvhwZ607AByt\n8OHOWncAOFrhw5217gBwtMKHe3Uf1e6uzsNtx3UWflgAMCXJpOBbB985fP+1Nw9o7Yat6h8YamNF\nANA+SYQ7K2YA4EhJhDsrZgDgSEmE+wfGzLfnaQeA1CUR7vbE2gEgdUmE+8ibB2q2v1anHQBSl0S4\n1/vCkiVWzACYk5II9zXLFqrWDExIrJgBMCclEe4rlvQq6hwbGhnl0zuAOSeJcJekjgZnT//inwcJ\neABzSjLhfijqfXaX3gnpmgcGteSG7xLyAOaEwm/WUdXb3aWhJl9aeu3NA7rmgUFd88DgEe2WdNnS\nBbpxxaJprBAAZo6jwSfew53sCyT9vaQOSd+MiK+OO36spHskfVLSK5I+GxEvNXrOUqkU5XJ5kmUf\nrX9gSH/+wGDduXcAmE3e994O/e3vLJrwjnG2N0VEqVm/ptMytjskfV3ShZLOkLTK9hnjul0h6bWI\n+AVJN0v6uwlV2wIrlvTqsqULZvplAWBS/vftQ7r2wS3TNlWcZ879bEm7IuLFiHhb0v2Slo/rs1zS\nt7L7D0n6dXvmvx9644pF+j0CHkBBHHonpm25dp5w75X08pjHu7O2mn0i4qCk1yV9aPwT2V5tu2y7\nPDw8PLmKm7hxxSKd+7ETpuW5AaDVpusCh3nCvd73gybaRxGxPiJKEVHq6enJU9+k3PvH5/AJHkAh\nTNeWoHnCfbek+WMenyxpT70+to+R9AFJr7aiwMm6ccUivfTV39Ytn12sLnZmAjALdbzH07YlaJ6l\nkD+UdJrtUyUNSVop6XPj+jwi6fclPSvpUklPRp5lODNgxZLeumej+weGtHbD8xo98E7N4wAwXSa7\nWiavpuEeEQdtf0nSRlWWQt4VEdtt3yCpHBGPSLpT0j/Z3qXKJ/aV01JtizUKfgAoslxfYoqIxyQ9\nNq7t+jH3/0/SZ1pbGgBgspiMBoAEEe4AkCDCHQASRLgDQIJyXThsWl7YHpb0k0n++jxJP21hOUXA\nmOcGxjw3TGXMp0RE02+Bti3cp8J2Oc9V0VLCmOcGxjw3zMSYmZYBgAQR7gCQoKKG+/p2F9AGjHlu\nYMxzw7SPuZBz7gCAxor6yR0A0ADhDgAJKly4277A9k7bu2xf1+56WsX2Xbb32d42pu0E24/bfiH7\n+cGs3bZvzf4Gz9s+q32VT57t+bafsr3D9nbbV2ftyY7b9nG2f2B7Szbmv8naT7X9XDbmB2y/N2s/\nNnu8Kzve1876J8t2h+0B249mj5MeryTZfsn2VtuDtstZ24y9twsV7jk36y6quyVdMK7tOklPRMRp\nkp7IHkuV8Z+W3VZLun2Gamy1g5KujYjTJS2VdFX2v2fK435L0vkRcaakxZIusL1UlU3lb87G/Joq\nm85Ls2Dz+Ra5WtKOMY9TH2/Vr0XE4jFr2mfuvR0RhblJOkfSxjGP10pa2+66Wji+PknbxjzeKenE\n7P6JknZm9/9R0qpa/Yp8k/RtSb85V8Yt6eckbZb0KVW+rXhM1n74fa7KPgrnZPePyfq53bVPcJwn\nZ0F2vqRHVdmWM9nxjhn3S5LmjWubsfd2oT65K99m3Sn5+YjYK0nZzw9n7cn9HbJ/fi+R9JwSH3c2\nRTEoaZ+kxyX9WNJIVDaXl44cV67N52e5WyT9paTqlmcfUtrjrQpJ37W9yfbqrG3G3tu5NuuYRXJt\nxD0HJPV3sH28pIclXRMR++1aw6t0rdFWuHFHxCFJi213S/oXSafX6pb9LPSYbV8saV9EbLJ9XrW5\nRtckxjvOuRGxx/aHJT1u+78a9G35uIv2yT3PZt0p+R/bJ0pS9nNf1p7M38F2pyrBfm9EbMiakx+3\nJEXEiKT/UOV8Q3e2ubx05Lhm3ebzE3SupEtsvyTpflWmZm5RuuM9LCL2ZD/3qfIf8bM1g+/tooX7\n4c26s7PrK1XZnDtV1Y3Hlf389pj2y7Mz7EslvV79p16RuPIR/U5JOyLipjGHkh237Z7sE7tsd0n6\nDVVOND6lyuby0tFjrv4tZtXm83lExNqIODki+lT5/+uTEXGZEh1vle332X5/9b6k35K0TTP53m73\nSYdJnKS4SNKPVJmn/Kt219PCcd0naa+kA6r8V/wKVeYan5D0QvbzhKyvVVk19GNJWyWV2l3/JMf8\nK6r80/N5SYPZ7aKUxy3pE5IGsjFvk3R91v5RST+QtEvSg5KOzdqPyx7vyo5/tN1jmMLYz5P06FwY\nbza+LdltezWrZvK9zeUHACBBRZuWAQDkQLgDQIIIdwBIEOEOAAki3AEgQYQ7ACSIcAeABP0/G7GQ\nLK5jhKYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c21f7dc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x = [i[0] for i in loss_list], y = [i[1] for i in loss_list])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64, 100)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.dot(w1).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`mm` is matrix multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 2.5893 -1.6666\n",
       "-4.9597  2.8256\n",
       "[torch.FloatTensor of size 2x2]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.randn(2,3)\n",
    "b = torch.randn(3,2)\n",
    "\n",
    "a.mm(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`clamp` does min or max operation on all elements of a tensor matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.2827  0.3575 -1.0719\n",
       "-1.7055 -0.4994 -0.8551\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 1.0000  1.0000  1.2978\n",
       " 1.0000  1.0000  1.0000\n",
       " 1.0000  1.0000  2.6871\n",
       "[torch.FloatTensor of size 3x3]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.clamp(min = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`pow` takes power of all the elemnets of a tensor. `_` is doing inplace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/groverprince/anaconda/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:1: UserWarning: torch.range is deprecated in favor of torch.arange and will be removed in 0.3. Note that arange generates values in [start; end), not [start; end].\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "   1    8   27\n",
       "  64  125  216\n",
       " 343  512  729\n",
       "[torch.FloatTensor of size 3x3]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.range(1,9)\n",
    "a.resize_(3,3)\n",
    "\n",
    "a.pow(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.001463377382606268"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((y_pred - y)*2).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "1.00000e-03 *\n",
       "  1.4634\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((y_pred - y)*2).sum(0).sum(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\n",
       "  1  2  3\n",
       "  4  5  6\n",
       "  7  8  9\n",
       " [torch.FloatTensor of size 3x3], \n",
       "  1  4  7\n",
       "  2  5  8\n",
       "  3  6  9\n",
       " [torch.FloatTensor of size 3x3])"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a,a.t()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using `Variable` for backpropagatio "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold input and outputs, and wrap them in Variables.\n",
    "# Setting requires_grad=False indicates that we do not need to compute gradients\n",
    "# with respect to these Variables during the backward pass.\n",
    "x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)\n",
    "y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)\n",
    "\n",
    "# Create random Tensors for weights, and wrap them in Variables.\n",
    "# Setting requires_grad=True indicates that we want to compute gradients with\n",
    "# respect to these Variables during the backward pass.\n",
    "w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)\n",
    "w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y using operations on Variables; these\n",
    "    # are exactly the same operations we used to compute the forward pass using\n",
    "    # Tensors, but we do not need to keep references to intermediate values since\n",
    "    # we are not implementing the backward pass by hand.\n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "\n",
    "    # Compute and print loss using operations on Variables.\n",
    "    # Now loss is a Variable of shape (1,) and loss.data is a Tensor of shape\n",
    "    # (1,); loss.data[0] is a scalar value holding the loss.\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    #print(t, loss.data[0])\n",
    "\n",
    "    # Use autograd to compute the backward pass. This call will compute the\n",
    "    # gradient of loss with respect to all Variables with requires_grad=True.\n",
    "    # After this call w1.grad and w2.grad will be Variables holding the gradient\n",
    "    # of the loss with respect to w1 and w2 respectively.\n",
    "    loss.backward()\n",
    "\n",
    "    # Update weights using gradient descent; w1.data and w2.data are Tensors,\n",
    "    # w1.grad and w2.grad are Variables and w1.grad.data and w2.grad.data are\n",
    "    # Tensors.\n",
    "    w1.data -= learning_rate * w1.grad.data\n",
    "    w2.data -= learning_rate * w2.grad.data\n",
    "\n",
    "    # Manually zero the gradients after updating weights\n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.4709 -1.2256  0.7913\n",
       "-2.3437 -1.9122 -0.2198\n",
       " 1.7673 -2.0683  0.0533\n",
       "[torch.FloatTensor of size 3x3]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.randn(3,3)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.4709 -1.2256  0.7913\n",
       "-2.3437 -1.9122 -0.2198\n",
       " 1.7673 -2.0683  0.0533\n",
       "[torch.FloatTensor of size 3x3]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = Variable(a)\n",
    "b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Self auto grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class MyReLU(torch.autograd.Function):\n",
    "    \"\"\"\n",
    "    We can implement our own custom autograd Functions by subclassing\n",
    "    torch.autograd.Function and implementing the forward and backward passes\n",
    "    which operate on Tensors.\n",
    "    \"\"\"\n",
    "\n",
    "    def forward(self, input):\n",
    "        \"\"\"\n",
    "        In the forward pass we receive a Tensor containing the input and return a\n",
    "        Tensor containing the output. You can cache arbitrary Tensors for use in the\n",
    "        backward pass using the save_for_backward method.\n",
    "        \"\"\"\n",
    "        self.save_for_backward(input)\n",
    "        return input.clamp(min=0)\n",
    "\n",
    "    def backward(self, grad_output):\n",
    "        \"\"\"\n",
    "        In the backward pass we receive a Tensor containing the gradient of the loss\n",
    "        with respect to the output, and we need to compute the gradient of the loss\n",
    "        with respect to the input.\n",
    "        \"\"\"\n",
    "        input, = self.saved_tensors\n",
    "        grad_input = grad_output.clone()\n",
    "        grad_input[input < 0] = 0\n",
    "        return grad_input\n",
    "\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold input and outputs, and wrap them in Variables.\n",
    "x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)\n",
    "y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)\n",
    "\n",
    "# Create random Tensors for weights, and wrap them in Variables.\n",
    "w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)\n",
    "w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Construct an instance of our MyReLU class to use in our network\n",
    "    relu = MyReLU()\n",
    "\n",
    "    # Forward pass: compute predicted y using operations on Variables; we compute\n",
    "    # ReLU using our custom autograd operation.\n",
    "    y_pred = relu(x.mm(w1)).mm(w2)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    #print(t, loss.data[0])\n",
    "\n",
    "    # Use autograd to compute the backward pass.\n",
    "    loss.backward()\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1.data -= learning_rate * w1.grad.data\n",
    "    w2.data -= learning_rate * w2.grad.data\n",
    "\n",
    "    # Manually zero the gradients after updating weights\n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now I have done a lot of practice on `pytorch`, I am going to implement neural net for logistic regression from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.metrics import *\n",
    "from fastai.model import *\n",
    "from fastai.dataset import *\n",
    "\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First of all, `loss` in binary = `- (y*log(y_hat) + (1-y)*(log(1-y_hat)))`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(\n",
    "        nn.Linear(28*28, 100),\n",
    "        nn.ReLU(),\n",
    "        nn.Linear(100,10),\n",
    "        nn.LogSoftmax()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential (\n",
       "  (0): Linear (784 -> 100)\n",
       "  (1): ReLU ()\n",
       "  (2): Linear (100 -> 10)\n",
       "  (3): LogSoftmax ()\n",
       ")"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mmnsit\u001b[m\u001b[m             mnsitmnist.pkl.gz\r\n"
     ]
    }
   ],
   "source": [
    "! ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50000, 784), (10000, 784), (50000,), (10000,))"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape, x_valid.shape, y.shape, y_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ImageClassifierData.from_arrays(path, (x,y), (x_valid,y_valid) ) # give in form of tuple as written in docstring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = nn.NLLLoss()\n",
    "metrics = [accuracy] # from fastai metrics.py\n",
    "opt = optim.SGD(net.parameters(), lr = 1e-1, momentum=0.9) # will talk in detail about momentum later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_loss(y, y_hat):\n",
    "    return np.mean(-(y*np.log(y_hat) + (1-y)*np.log(1-y_hat)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22708064055624455"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# toy example of loss\n",
    "actuals = np.array([1,1,0,0])\n",
    "preds = np.array([0.7,0.8, 0.1, 0.2])\n",
    "binary_loss(actuals, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.010050335853501451, -6.9077552789821368)"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log(0.99), np.log(0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is `-ve` in formula because `log` is -ve for numbers b/w 0-1\n",
    "Accuracy is 100% and loss is 0.22. Because we for 1 it was prediction 0.7, not 0.99. Therefore less confident. (maybe we could use loss to get CI for this method -- compare loss and accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3922f197080d4fc7b9c3a71b4d161840",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/782 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|          | 0/782 [00:00<?, ?it/s, loss=2.32]\u001b[A\n",
      "  0%|          | 0/782 [00:00<?, ?it/s, loss=2.31]\u001b[A\n",
      "  0%|          | 2/782 [00:00<01:02, 12.45it/s, loss=2.31]\u001b[A\n",
      "  0%|          | 2/782 [00:00<01:12, 10.78it/s, loss=2.29]\u001b[A\n",
      "  0%|          | 2/782 [00:00<01:24,  9.20it/s, loss=2.27]\u001b[A\n",
      "  0%|          | 2/782 [00:00<01:32,  8.43it/s, loss=2.26]\u001b[A\n",
      "  0%|          | 2/782 [00:00<01:46,  7.34it/s, loss=2.24]\u001b[A\n",
      "  1%|          | 6/782 [00:00<00:37, 20.71it/s, loss=2.24]\u001b[A\n",
      "  1%|          | 6/782 [00:00<00:39, 19.76it/s, loss=2.22]\u001b[A\n",
      "  1%|          | 6/782 [00:00<00:41, 18.88it/s, loss=2.18]\u001b[A\n",
      "  1%|          | 6/782 [00:00<00:42, 18.11it/s, loss=2.15]\u001b[A\n",
      "  1%|          | 6/782 [00:00<00:46, 16.65it/s, loss=2.1] \u001b[A\n",
      "  1%|          | 6/782 [00:00<00:49, 15.83it/s, loss=2.06]\u001b[A\n",
      "  1%|▏         | 11/782 [00:00<00:28, 27.38it/s, loss=2.06]\u001b[A\n",
      "  1%|▏         | 11/782 [00:00<00:30, 25.54it/s, loss=2]   \u001b[A\n",
      "  1%|▏         | 11/782 [00:00<00:31, 24.50it/s, loss=1.93]\u001b[A\n",
      "  1%|▏         | 11/782 [00:00<00:33, 22.91it/s, loss=1.87]\u001b[A\n",
      "  2%|▏         | 14/782 [00:00<00:27, 28.32it/s, loss=1.87]\u001b[A\n",
      "  2%|▏         | 14/782 [00:00<00:28, 27.12it/s, loss=1.82]\u001b[A\n",
      "  2%|▏         | 14/782 [00:00<00:30, 25.32it/s, loss=1.76]\u001b[A\n",
      "  2%|▏         | 14/782 [00:00<00:31, 24.53it/s, loss=1.7] \u001b[A\n",
      "  2%|▏         | 14/782 [00:00<00:32, 23.92it/s, loss=1.65]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:25, 29.73it/s, loss=1.65]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:26, 28.99it/s, loss=1.58]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:26, 28.31it/s, loss=1.52]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:27, 27.70it/s, loss=1.48]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:28, 26.96it/s, loss=1.43]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:29, 26.30it/s, loss=1.39]\u001b[A\n",
      "  2%|▏         | 18/782 [00:00<00:29, 25.60it/s, loss=1.35]\u001b[A\n",
      "  3%|▎         | 24/782 [00:00<00:22, 33.02it/s, loss=1.35]\u001b[A\n",
      "  3%|▎         | 24/782 [00:00<00:23, 31.66it/s, loss=1.31]\u001b[A\n",
      "  3%|▎         | 24/782 [00:00<00:24, 30.64it/s, loss=1.28]\u001b[A\n",
      "  3%|▎         | 24/782 [00:00<00:26, 28.92it/s, loss=1.24]\u001b[A\n",
      "  3%|▎         | 24/782 [00:00<00:27, 28.05it/s, loss=1.22]\u001b[A\n",
      "  4%|▎         | 28/782 [00:00<00:23, 32.34it/s, loss=1.22]\u001b[A\n",
      "  4%|▎         | 28/782 [00:00<00:23, 31.78it/s, loss=1.19]\u001b[A\n",
      "  4%|▎         | 28/782 [00:00<00:24, 31.12it/s, loss=1.18]\u001b[A\n",
      "  4%|▎         | 28/782 [00:00<00:25, 30.04it/s, loss=1.16]\u001b[A\n",
      "  4%|▎         | 28/782 [00:00<00:25, 29.13it/s, loss=1.13]\u001b[A\n",
      "  4%|▍         | 32/782 [00:00<00:22, 32.92it/s, loss=1.13]\u001b[A\n",
      "  4%|▍         | 32/782 [00:00<00:23, 32.34it/s, loss=1.11]\u001b[A\n",
      "  4%|▍         | 32/782 [00:01<00:23, 31.52it/s, loss=1.09]\u001b[A\n",
      "  4%|▍         | 32/782 [00:01<00:24, 30.87it/s, loss=1.07]\u001b[A\n",
      "  4%|▍         | 32/782 [00:01<00:24, 30.33it/s, loss=1.06]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:22, 33.33it/s, loss=1.06]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:23, 32.35it/s, loss=1.05]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:23, 31.80it/s, loss=1.03]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:23, 31.16it/s, loss=1.02]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:24, 30.54it/s, loss=1.01]\u001b[A\n",
      "  5%|▌         | 40/782 [00:01<00:22, 33.72it/s, loss=1.01]\u001b[A\n",
      "  5%|▌         | 40/782 [00:01<00:22, 33.19it/s, loss=0.993]\u001b[A\n",
      "  5%|▌         | 40/782 [00:01<00:22, 32.36it/s, loss=0.967]\u001b[A\n",
      "  5%|▌         | 40/782 [00:01<00:23, 31.70it/s, loss=0.957]\u001b[A\n",
      "  5%|▌         | 40/782 [00:01<00:23, 31.26it/s, loss=0.945]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:21, 34.03it/s, loss=0.945]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:22, 33.54it/s, loss=0.927]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:22, 33.09it/s, loss=0.911]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:22, 32.62it/s, loss=0.901]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:22, 32.40it/s, loss=0.886]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:22, 32.16it/s, loss=0.865]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:23, 32.02it/s, loss=0.852]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:23, 31.87it/s, loss=0.848]\u001b[A\n",
      "  6%|▌         | 44/782 [00:01<00:23, 31.69it/s, loss=0.851]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:19, 37.27it/s, loss=0.851]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:19, 37.11it/s, loss=0.839]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:19, 36.95it/s, loss=0.83] \u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:19, 36.80it/s, loss=0.818]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:19, 36.64it/s, loss=0.805]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 36.49it/s, loss=0.791]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 36.35it/s, loss=0.784]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 36.17it/s, loss=0.774]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 36.01it/s, loss=0.764]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 35.84it/s, loss=0.752]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 35.55it/s, loss=0.739]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 35.26it/s, loss=0.725]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 35.06it/s, loss=0.717]\u001b[A\n",
      "  7%|▋         | 52/782 [00:01<00:20, 34.80it/s, loss=0.715]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:16, 43.36it/s, loss=0.715]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:16, 43.00it/s, loss=0.707]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:16, 42.67it/s, loss=0.697]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:16, 42.45it/s, loss=0.689]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:16, 42.27it/s, loss=0.681]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 42.09it/s, loss=0.673]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 41.89it/s, loss=0.665]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 41.75it/s, loss=0.654]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 41.58it/s, loss=0.645]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 41.40it/s, loss=0.636]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 41.21it/s, loss=0.635]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 40.99it/s, loss=0.638]\u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 40.78it/s, loss=0.63] \u001b[A\n",
      "  8%|▊         | 65/782 [00:01<00:17, 40.56it/s, loss=0.624]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:14, 48.50it/s, loss=0.624]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:14, 48.24it/s, loss=0.618]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:14, 47.93it/s, loss=0.611]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:14, 47.63it/s, loss=0.611]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:14, 47.44it/s, loss=0.603]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:14, 47.11it/s, loss=0.593]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 46.85it/s, loss=0.585]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 46.62it/s, loss=0.581]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 46.44it/s, loss=0.572]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 46.25it/s, loss=0.569]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 46.09it/s, loss=0.561]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 45.90it/s, loss=0.551]\u001b[A\n",
      " 10%|▉         | 78/782 [00:01<00:15, 45.74it/s, loss=0.547]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 52.65it/s, loss=0.547]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 52.47it/s, loss=0.541]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 52.26it/s, loss=0.538]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 52.06it/s, loss=0.535]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 51.85it/s, loss=0.534]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 51.61it/s, loss=0.529]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 50.73it/s, loss=0.523]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 50.54it/s, loss=0.522]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 50.16it/s, loss=0.519]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 49.96it/s, loss=0.515]\u001b[A\n",
      " 12%|█▏        | 90/782 [00:01<00:13, 49.79it/s, loss=0.51] \u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 55.20it/s, loss=0.51]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 55.01it/s, loss=0.506]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 54.84it/s, loss=0.503]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 54.60it/s, loss=0.5]  \u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 54.43it/s, loss=0.497]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 54.20it/s, loss=0.491]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 54.05it/s, loss=0.487]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 53.89it/s, loss=0.482]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 53.70it/s, loss=0.478]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 53.51it/s, loss=0.475]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 53.28it/s, loss=0.47] \u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 53.08it/s, loss=0.465]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:12, 52.59it/s, loss=0.465]\u001b[A\n",
      " 13%|█▎        | 100/782 [00:01<00:13, 52.38it/s, loss=0.46] \u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 59.09it/s, loss=0.46]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 58.89it/s, loss=0.456]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 58.70it/s, loss=0.452]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 58.48it/s, loss=0.447]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 58.06it/s, loss=0.444]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 57.73it/s, loss=0.441]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:01<00:11, 57.08it/s, loss=0.436]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:02<00:11, 56.47it/s, loss=0.435]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:02<00:11, 56.04it/s, loss=0.431]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:02<00:12, 55.62it/s, loss=0.43] \u001b[A\n",
      " 14%|█▍        | 113/782 [00:02<00:12, 55.29it/s, loss=0.434]\u001b[A\n",
      " 14%|█▍        | 113/782 [00:02<00:12, 55.13it/s, loss=0.435]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:10, 60.38it/s, loss=0.435]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:10, 60.21it/s, loss=0.43] \u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:10, 60.02it/s, loss=0.427]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:10, 59.83it/s, loss=0.427]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 59.61it/s, loss=0.428]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 59.42it/s, loss=0.423]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 59.27it/s, loss=0.424]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 59.11it/s, loss=0.426]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 58.91it/s, loss=0.425]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 58.72it/s, loss=0.425]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 58.55it/s, loss=0.426]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 58.39it/s, loss=0.422]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 58.22it/s, loss=0.428]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 58.06it/s, loss=0.425]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 57.90it/s, loss=0.422]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 57.75it/s, loss=0.418]\u001b[A\n",
      " 16%|█▌        | 124/782 [00:02<00:11, 57.59it/s, loss=0.416]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:09, 64.91it/s, loss=0.416]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:09, 64.75it/s, loss=0.416]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:09, 64.60it/s, loss=0.413]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:09, 64.44it/s, loss=0.413]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 64.00it/s, loss=0.408]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 63.64it/s, loss=0.409]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 63.44it/s, loss=0.404]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 63.26it/s, loss=0.4]  \u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 62.85it/s, loss=0.4]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 62.68it/s, loss=0.4]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 62.52it/s, loss=0.401]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 62.32it/s, loss=0.395]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 62.14it/s, loss=0.391]\u001b[A\n",
      " 18%|█▊        | 140/782 [00:02<00:10, 61.97it/s, loss=0.395]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 67.62it/s, loss=0.395]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 67.44it/s, loss=0.389]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 67.23it/s, loss=0.384]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 66.99it/s, loss=0.382]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 66.66it/s, loss=0.38] \u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 66.37it/s, loss=0.377]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 66.19it/s, loss=0.374]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 66.02it/s, loss=0.371]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 65.83it/s, loss=0.372]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 65.67it/s, loss=0.367]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 65.52it/s, loss=0.366]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 65.36it/s, loss=0.364]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 65.19it/s, loss=0.362]\u001b[A\n",
      " 20%|█▉        | 153/782 [00:02<00:09, 64.81it/s, loss=0.364]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 70.17it/s, loss=0.364]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 69.94it/s, loss=0.363]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 69.76it/s, loss=0.364]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 69.58it/s, loss=0.36] \u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 69.40it/s, loss=0.361]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 69.23it/s, loss=0.36] \u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 69.05it/s, loss=0.356]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 68.85it/s, loss=0.357]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:08, 68.49it/s, loss=0.353]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:09, 68.28it/s, loss=0.354]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:09, 68.06it/s, loss=0.354]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:09, 67.90it/s, loss=0.353]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:09, 67.68it/s, loss=0.349]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:09, 67.51it/s, loss=0.346]\u001b[A\n",
      " 21%|██        | 166/782 [00:02<00:09, 67.30it/s, loss=0.344]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 72.86it/s, loss=0.344]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 72.66it/s, loss=0.345]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 72.46it/s, loss=0.342]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 72.10it/s, loss=0.34] \u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 71.88it/s, loss=0.339]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 71.69it/s, loss=0.339]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 71.38it/s, loss=0.338]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 71.10it/s, loss=0.335]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 70.90it/s, loss=0.334]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 70.67it/s, loss=0.329]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 70.48it/s, loss=0.327]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 70.28it/s, loss=0.329]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 70.10it/s, loss=0.327]\u001b[A\n",
      " 23%|██▎       | 180/782 [00:02<00:08, 69.83it/s, loss=0.324]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:07, 74.70it/s, loss=0.324]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:07, 74.53it/s, loss=0.326]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:07, 74.32it/s, loss=0.327]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:07, 74.07it/s, loss=0.325]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:07, 73.89it/s, loss=0.321]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:07, 73.72it/s, loss=0.319]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 73.54it/s, loss=0.319]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 73.36it/s, loss=0.316]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 73.19it/s, loss=0.315]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 73.00it/s, loss=0.312]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 72.83it/s, loss=0.309]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 72.66it/s, loss=0.312]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 72.32it/s, loss=0.312]\u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 72.11it/s, loss=0.31] \u001b[A\n",
      " 25%|██▍       | 193/782 [00:02<00:08, 71.82it/s, loss=0.309]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 76.92it/s, loss=0.309]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 76.74it/s, loss=0.309]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 76.55it/s, loss=0.307]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 76.38it/s, loss=0.306]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 76.18it/s, loss=0.303]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 75.99it/s, loss=0.305]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 75.84it/s, loss=0.303]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 75.67it/s, loss=0.302]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 75.48it/s, loss=0.302]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 75.24it/s, loss=0.301]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 74.95it/s, loss=0.301]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 74.72it/s, loss=0.297]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 74.57it/s, loss=0.293]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 74.41it/s, loss=0.293]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 74.24it/s, loss=0.293]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 79.16it/s, loss=0.293]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 79.00it/s, loss=0.291]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 78.83it/s, loss=0.29] \u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 78.62it/s, loss=0.286]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 78.45it/s, loss=0.285]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 78.28it/s, loss=0.281]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 77.94it/s, loss=0.281]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 77.75it/s, loss=0.283]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 77.59it/s, loss=0.282]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 221/782 [00:02<00:07, 77.40it/s, loss=0.28] \u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 77.25it/s, loss=0.279]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 77.08it/s, loss=0.279]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 76.92it/s, loss=0.279]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 76.73it/s, loss=0.279]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 76.56it/s, loss=0.279]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:07, 76.35it/s, loss=0.279]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 81.43it/s, loss=0.279]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 81.25it/s, loss=0.277]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 80.90it/s, loss=0.277]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 80.66it/s, loss=0.277]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 80.41it/s, loss=0.279]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 80.21it/s, loss=0.277]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 80.05it/s, loss=0.274]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 79.87it/s, loss=0.275]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 79.73it/s, loss=0.277]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 79.54it/s, loss=0.276]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 79.37it/s, loss=0.275]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 79.20it/s, loss=0.274]\u001b[A\n",
      " 30%|███       | 236/782 [00:02<00:06, 79.00it/s, loss=0.269]\u001b[A\n",
      " 30%|███       | 236/782 [00:03<00:06, 78.65it/s, loss=0.268]\u001b[A\n",
      " 30%|███       | 236/782 [00:03<00:06, 78.39it/s, loss=0.265]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 82.92it/s, loss=0.265]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 82.70it/s, loss=0.263]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 82.54it/s, loss=0.263]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 82.37it/s, loss=0.263]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 82.18it/s, loss=0.261]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 82.03it/s, loss=0.26] \u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 81.87it/s, loss=0.259]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 81.57it/s, loss=0.258]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 81.33it/s, loss=0.258]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 81.14it/s, loss=0.261]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 80.93it/s, loss=0.259]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 80.77it/s, loss=0.257]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 80.60it/s, loss=0.255]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 80.42it/s, loss=0.256]\u001b[A\n",
      " 32%|███▏      | 250/782 [00:03<00:06, 80.27it/s, loss=0.256]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 84.67it/s, loss=0.256]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 84.50it/s, loss=0.256]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 84.31it/s, loss=0.254]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 83.97it/s, loss=0.254]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 83.68it/s, loss=0.252]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 83.51it/s, loss=0.252]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 83.26it/s, loss=0.25] \u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 83.05it/s, loss=0.25]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 82.88it/s, loss=0.251]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 82.65it/s, loss=0.251]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 82.42it/s, loss=0.249]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 82.10it/s, loss=0.246]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 81.83it/s, loss=0.244]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 81.60it/s, loss=0.242]\u001b[A\n",
      " 34%|███▍      | 264/782 [00:03<00:06, 81.44it/s, loss=0.239]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 85.65it/s, loss=0.239]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 85.46it/s, loss=0.238]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 85.32it/s, loss=0.238]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 85.16it/s, loss=0.239]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 85.02it/s, loss=0.237]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 84.83it/s, loss=0.237]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 84.52it/s, loss=0.238]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 84.34it/s, loss=0.24] \u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:05, 84.11it/s, loss=0.239]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:06, 83.93it/s, loss=0.243]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:06, 83.74it/s, loss=0.246]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:06, 83.55it/s, loss=0.242]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:06, 83.39it/s, loss=0.245]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:06, 83.18it/s, loss=0.243]\u001b[A\n",
      " 36%|███▌      | 278/782 [00:03<00:06, 83.01it/s, loss=0.243]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 87.10it/s, loss=0.243]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 86.94it/s, loss=0.241]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 86.61it/s, loss=0.24] \u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 86.45it/s, loss=0.246]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 86.20it/s, loss=0.244]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 86.04it/s, loss=0.249]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 85.90it/s, loss=0.249]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 85.73it/s, loss=0.246]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 85.56it/s, loss=0.244]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 85.36it/s, loss=0.242]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 85.18it/s, loss=0.243]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 85.03it/s, loss=0.244]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 84.82it/s, loss=0.243]\u001b[A\n",
      " 37%|███▋      | 292/782 [00:03<00:05, 84.58it/s, loss=0.242]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 88.13it/s, loss=0.242]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 87.55it/s, loss=0.239]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 87.31it/s, loss=0.239]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 87.03it/s, loss=0.24] \u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 86.88it/s, loss=0.241]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 86.51it/s, loss=0.238]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 86.18it/s, loss=0.238]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 85.43it/s, loss=0.237]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 85.23it/s, loss=0.24] \u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 85.09it/s, loss=0.239]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 84.95it/s, loss=0.24] \u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 84.82it/s, loss=0.239]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 84.68it/s, loss=0.238]\u001b[A\n",
      " 39%|███▉      | 305/782 [00:03<00:05, 84.51it/s, loss=0.238]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 88.02it/s, loss=0.238]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 87.86it/s, loss=0.239]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 87.55it/s, loss=0.239]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 87.37it/s, loss=0.24] \u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 87.21it/s, loss=0.239]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.99it/s, loss=0.237]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.84it/s, loss=0.235]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.71it/s, loss=0.237]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.57it/s, loss=0.235]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.42it/s, loss=0.233]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.25it/s, loss=0.23] \u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 86.11it/s, loss=0.228]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 85.96it/s, loss=0.227]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 85.81it/s, loss=0.225]\u001b[A\n",
      " 41%|████      | 318/782 [00:03<00:05, 85.65it/s, loss=0.225]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 89.33it/s, loss=0.225]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 89.18it/s, loss=0.224]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 89.04it/s, loss=0.226]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 88.91it/s, loss=0.222]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 88.79it/s, loss=0.222]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 88.66it/s, loss=0.219]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 88.39it/s, loss=0.217]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 88.14it/s, loss=0.216]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.98it/s, loss=0.215]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.83it/s, loss=0.215]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.59it/s, loss=0.214]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.45it/s, loss=0.211]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.30it/s, loss=0.209]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.17it/s, loss=0.216]\u001b[A\n",
      " 42%|████▏     | 332/782 [00:03<00:05, 87.01it/s, loss=0.213]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 90.59it/s, loss=0.213]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 90.46it/s, loss=0.21] \u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 90.33it/s, loss=0.212]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 90.17it/s, loss=0.212]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 90.03it/s, loss=0.212]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 89.90it/s, loss=0.213]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 89.76it/s, loss=0.212]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 89.51it/s, loss=0.213]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 89.34it/s, loss=0.213]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 89.20it/s, loss=0.213]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 89.01it/s, loss=0.21] \u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 88.88it/s, loss=0.211]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 88.73it/s, loss=0.211]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 88.60it/s, loss=0.214]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 88.46it/s, loss=0.217]\u001b[A\n",
      " 44%|████▍     | 346/782 [00:03<00:04, 88.30it/s, loss=0.218]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 92.02it/s, loss=0.218]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 91.87it/s, loss=0.218]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 91.71it/s, loss=0.216]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 91.44it/s, loss=0.221]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 91.20it/s, loss=0.223]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 91.01it/s, loss=0.226]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 90.83it/s, loss=0.228]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 90.68it/s, loss=0.227]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 90.54it/s, loss=0.227]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:03<00:04, 90.37it/s, loss=0.224]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:04<00:04, 90.23it/s, loss=0.226]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:04<00:04, 90.00it/s, loss=0.226]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:04<00:04, 89.81it/s, loss=0.226]\u001b[A\n",
      " 46%|████▌     | 361/782 [00:04<00:04, 89.69it/s, loss=0.23] \u001b[A\n",
      " 46%|████▌     | 361/782 [00:04<00:04, 89.53it/s, loss=0.229]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 92.88it/s, loss=0.229]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 92.73it/s, loss=0.228]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 92.58it/s, loss=0.226]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 92.41it/s, loss=0.226]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 92.18it/s, loss=0.225]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 92.00it/s, loss=0.23] \u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 91.82it/s, loss=0.233]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 91.66it/s, loss=0.234]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 91.50it/s, loss=0.232]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 91.34it/s, loss=0.234]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 91.20it/s, loss=0.233]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 91.07it/s, loss=0.23] \u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 90.90it/s, loss=0.229]\u001b[A\n",
      " 48%|████▊     | 375/782 [00:04<00:04, 90.64it/s, loss=0.228]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 93.70it/s, loss=0.228]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 93.56it/s, loss=0.227]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 93.37it/s, loss=0.225]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 93.22it/s, loss=0.223]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 93.04it/s, loss=0.223]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 92.87it/s, loss=0.221]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 92.71it/s, loss=0.22] \u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 92.58it/s, loss=0.221]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 92.30it/s, loss=0.221]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 92.16it/s, loss=0.221]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 92.01it/s, loss=0.218]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 91.81it/s, loss=0.217]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 91.69it/s, loss=0.217]\u001b[A\n",
      " 50%|████▉     | 388/782 [00:04<00:04, 91.56it/s, loss=0.216]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 94.54it/s, loss=0.216]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 94.36it/s, loss=0.218]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 94.22it/s, loss=0.217]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 94.03it/s, loss=0.215]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 93.78it/s, loss=0.214]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 93.63it/s, loss=0.213]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 93.41it/s, loss=0.211]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 93.28it/s, loss=0.211]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 93.15it/s, loss=0.209]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 93.02it/s, loss=0.21] \u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 92.88it/s, loss=0.208]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 92.75it/s, loss=0.206]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 92.62it/s, loss=0.206]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 92.49it/s, loss=0.207]\u001b[A\n",
      " 51%|█████▏    | 401/782 [00:04<00:04, 92.26it/s, loss=0.208]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 95.36it/s, loss=0.208]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 95.19it/s, loss=0.208]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 95.02it/s, loss=0.211]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.82it/s, loss=0.213]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.69it/s, loss=0.211]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.54it/s, loss=0.209]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.40it/s, loss=0.208]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.28it/s, loss=0.208]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.16it/s, loss=0.205]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 94.02it/s, loss=0.204]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 93.88it/s, loss=0.203]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 93.75it/s, loss=0.202]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 93.62it/s, loss=0.206]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 93.47it/s, loss=0.204]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 93.35it/s, loss=0.206]\u001b[A\n",
      " 53%|█████▎    | 415/782 [00:04<00:03, 93.12it/s, loss=0.204]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 96.38it/s, loss=0.204]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 96.22it/s, loss=0.203]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 96.05it/s, loss=0.203]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.90it/s, loss=0.204]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.77it/s, loss=0.206]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.63it/s, loss=0.205]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.49it/s, loss=0.205]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.35it/s, loss=0.204]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.19it/s, loss=0.203]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 95.05it/s, loss=0.202]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 94.90it/s, loss=0.202]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 94.76it/s, loss=0.2]  \u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 94.64it/s, loss=0.202]\u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 94.37it/s, loss=0.2]  \u001b[A\n",
      " 55%|█████▍    | 430/782 [00:04<00:03, 94.23it/s, loss=0.198]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 97.22it/s, loss=0.198]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.97it/s, loss=0.196]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.79it/s, loss=0.197]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.63it/s, loss=0.196]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.47it/s, loss=0.196]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.33it/s, loss=0.196]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.19it/s, loss=0.196]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 96.07it/s, loss=0.195]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 95.95it/s, loss=0.199]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 95.82it/s, loss=0.198]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 95.68it/s, loss=0.198]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 95.54it/s, loss=0.203]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 95.42it/s, loss=0.21] \u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 95.30it/s, loss=0.21]\u001b[A\n",
      " 57%|█████▋    | 444/782 [00:04<00:03, 94.98it/s, loss=0.207]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.89it/s, loss=0.207]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.76it/s, loss=0.205]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.57it/s, loss=0.203]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.43it/s, loss=0.201]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.30it/s, loss=0.2]  \u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.18it/s, loss=0.199]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 97.03it/s, loss=0.197]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.90it/s, loss=0.196]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.79it/s, loss=0.195]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.65it/s, loss=0.195]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.51it/s, loss=0.195]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.38it/s, loss=0.193]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.25it/s, loss=0.192]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 96.02it/s, loss=0.192]\u001b[A\n",
      " 59%|█████▊    | 458/782 [00:04<00:03, 95.84it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 98.68it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 98.48it/s, loss=0.194]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 98.34it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 98.19it/s, loss=0.193]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 98.06it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 97.93it/s, loss=0.194]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 97.80it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 97.68it/s, loss=0.193]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 97.54it/s, loss=0.193]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 97.41it/s, loss=0.195]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 97.26it/s, loss=0.193]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 96.99it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 96.78it/s, loss=0.192]\u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 96.63it/s, loss=0.19] \u001b[A\n",
      " 60%|██████    | 472/782 [00:04<00:03, 96.45it/s, loss=0.188]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:02, 99.23it/s, loss=0.188]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:02, 99.09it/s, loss=0.186]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:02, 98.96it/s, loss=0.185]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:02, 98.81it/s, loss=0.183]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 98.52it/s, loss=0.184]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 98.35it/s, loss=0.186]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 98.18it/s, loss=0.184]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 98.04it/s, loss=0.185]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 97.92it/s, loss=0.189]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 97.80it/s, loss=0.188]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 97.67it/s, loss=0.188]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 97.41it/s, loss=0.186]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:04<00:03, 97.28it/s, loss=0.189]\u001b[A\n",
      " 62%|██████▏   | 486/782 [00:05<00:03, 97.14it/s, loss=0.188]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 99.66it/s, loss=0.188]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 99.50it/s, loss=0.188]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 99.36it/s, loss=0.193]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 99.24it/s, loss=0.194]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 99.10it/s, loss=0.195]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.87it/s, loss=0.192]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.74it/s, loss=0.191]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.55it/s, loss=0.19] \u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.37it/s, loss=0.19]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.24it/s, loss=0.192]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.13it/s, loss=0.19] \u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 98.02it/s, loss=0.191]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 97.88it/s, loss=0.189]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:05<00:02, 97.68it/s, loss=0.191]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 100.13it/s, loss=0.191]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 100.02it/s, loss=0.19] \u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 99.86it/s, loss=0.192]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 99.75it/s, loss=0.19] \u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 99.64it/s, loss=0.188]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 99.38it/s, loss=0.188]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 99.26it/s, loss=0.189]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 99.07it/s, loss=0.186]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 98.87it/s, loss=0.187]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 98.76it/s, loss=0.186]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 98.61it/s, loss=0.187]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 98.48it/s, loss=0.186]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 98.33it/s, loss=0.183]\u001b[A\n",
      " 65%|██████▌   | 512/782 [00:05<00:02, 98.20it/s, loss=0.187]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 100.59it/s, loss=0.187]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 100.46it/s, loss=0.189]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 100.35it/s, loss=0.189]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 100.19it/s, loss=0.19] \u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 100.07it/s, loss=0.19]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.94it/s, loss=0.193]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.83it/s, loss=0.193]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.72it/s, loss=0.193]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.61it/s, loss=0.199]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.50it/s, loss=0.197]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.39it/s, loss=0.195]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.24it/s, loss=0.196]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 99.13it/s, loss=0.199]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 98.88it/s, loss=0.197]\u001b[A\n",
      " 67%|██████▋   | 525/782 [00:05<00:02, 98.73it/s, loss=0.198]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 101.27it/s, loss=0.198]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 101.08it/s, loss=0.197]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.97it/s, loss=0.2]  \u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.86it/s, loss=0.201]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.74it/s, loss=0.202]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.62it/s, loss=0.201]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.50it/s, loss=0.201]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.26it/s, loss=0.205]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 100.13it/s, loss=0.21] \u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 99.96it/s, loss=0.211]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 99.80it/s, loss=0.209]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 99.60it/s, loss=0.207]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 99.50it/s, loss=0.206]\u001b[A\n",
      " 69%|██████▉   | 539/782 [00:05<00:02, 99.39it/s, loss=0.207]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 101.71it/s, loss=0.207]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 101.57it/s, loss=0.205]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 101.47it/s, loss=0.206]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 101.31it/s, loss=0.208]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 101.13it/s, loss=0.208]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 101.01it/s, loss=0.205]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.88it/s, loss=0.208]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.73it/s, loss=0.206]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.63it/s, loss=0.206]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.52it/s, loss=0.204]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.40it/s, loss=0.202]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.31it/s, loss=0.207]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.20it/s, loss=0.206]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 100.09it/s, loss=0.208]\u001b[A\n",
      " 71%|███████   | 552/782 [00:05<00:02, 99.94it/s, loss=0.207] \u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 102.39it/s, loss=0.207]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 102.25it/s, loss=0.205]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 102.09it/s, loss=0.204]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.97it/s, loss=0.201]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.86it/s, loss=0.2]  \u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.76it/s, loss=0.197]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.65it/s, loss=0.196]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.55it/s, loss=0.197]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.43it/s, loss=0.195]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.32it/s, loss=0.193]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.21it/s, loss=0.192]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 101.10it/s, loss=0.194]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 100.98it/s, loss=0.192]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 100.86it/s, loss=0.196]\u001b[A\n",
      " 72%|███████▏  | 566/782 [00:05<00:02, 100.63it/s, loss=0.197]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 103.04it/s, loss=0.197]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.92it/s, loss=0.195]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.76it/s, loss=0.194]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.56it/s, loss=0.196]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.42it/s, loss=0.194]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.31it/s, loss=0.193]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.20it/s, loss=0.194]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 102.08it/s, loss=0.195]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.96it/s, loss=0.194]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.83it/s, loss=0.198]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.71it/s, loss=0.195]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.61it/s, loss=0.197]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.51it/s, loss=0.194]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.39it/s, loss=0.194]\u001b[A\n",
      " 74%|███████▍  | 580/782 [00:05<00:01, 101.27it/s, loss=0.192]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 103.62it/s, loss=0.192]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 103.43it/s, loss=0.191]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 103.30it/s, loss=0.19] \u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 103.16it/s, loss=0.188]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 103.03it/s, loss=0.187]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.92it/s, loss=0.185]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.81it/s, loss=0.184]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.69it/s, loss=0.185]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.57it/s, loss=0.182]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.46it/s, loss=0.182]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.36it/s, loss=0.179]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.24it/s, loss=0.179]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 102.13it/s, loss=0.178]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 101.99it/s, loss=0.181]\u001b[A\n",
      " 76%|███████▌  | 594/782 [00:05<00:01, 101.78it/s, loss=0.18] \u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 104.08it/s, loss=0.18]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.94it/s, loss=0.18]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.83it/s, loss=0.18]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.73it/s, loss=0.177]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.62it/s, loss=0.178]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.51it/s, loss=0.182]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.39it/s, loss=0.182]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.14it/s, loss=0.182]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 103.02it/s, loss=0.181]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 102.82it/s, loss=0.179]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 102.70it/s, loss=0.18] \u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 102.57it/s, loss=0.179]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 102.45it/s, loss=0.177]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 102.35it/s, loss=0.178]\u001b[A\n",
      " 78%|███████▊  | 608/782 [00:05<00:01, 102.23it/s, loss=0.179]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 104.43it/s, loss=0.179]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 104.28it/s, loss=0.177]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 104.17it/s, loss=0.176]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 104.02it/s, loss=0.177]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 103.91it/s, loss=0.176]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 103.82it/s, loss=0.176]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:05<00:01, 103.72it/s, loss=0.176]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 103.61it/s, loss=0.176]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 103.51it/s, loss=0.176]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 103.30it/s, loss=0.177]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 103.17it/s, loss=0.178]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 103.06it/s, loss=0.177]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 102.92it/s, loss=0.177]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 102.81it/s, loss=0.175]\u001b[A\n",
      " 80%|███████▉  | 622/782 [00:06<00:01, 102.71it/s, loss=0.174]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.96it/s, loss=0.174]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.84it/s, loss=0.173]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.72it/s, loss=0.174]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.62it/s, loss=0.175]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.38it/s, loss=0.175]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.27it/s, loss=0.179]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.10it/s, loss=0.182]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 104.00it/s, loss=0.18] \u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.89it/s, loss=0.181]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.78it/s, loss=0.179]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.70it/s, loss=0.177]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.59it/s, loss=0.176]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.39it/s, loss=0.176]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.27it/s, loss=0.176]\u001b[A\n",
      " 81%|████████▏ | 636/782 [00:06<00:01, 103.16it/s, loss=0.178]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 105.36it/s, loss=0.178]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 105.22it/s, loss=0.176]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 105.11it/s, loss=0.178]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 105.00it/s, loss=0.178]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.90it/s, loss=0.178]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.80it/s, loss=0.176]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.57it/s, loss=0.175]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.40it/s, loss=0.18] \u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.29it/s, loss=0.178]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.18it/s, loss=0.176]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 104.05it/s, loss=0.175]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 103.96it/s, loss=0.175]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 103.85it/s, loss=0.176]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 103.75it/s, loss=0.174]\u001b[A\n",
      " 83%|████████▎ | 650/782 [00:06<00:01, 103.65it/s, loss=0.173]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.82it/s, loss=0.173]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.63it/s, loss=0.171]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.52it/s, loss=0.169]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.43it/s, loss=0.169]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.33it/s, loss=0.171]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.19it/s, loss=0.171]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 105.07it/s, loss=0.17] \u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.97it/s, loss=0.169]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.84it/s, loss=0.17] \u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.75it/s, loss=0.172]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.65it/s, loss=0.175]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.56it/s, loss=0.175]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.47it/s, loss=0.174]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.36it/s, loss=0.174]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.25it/s, loss=0.172]\u001b[A\n",
      " 85%|████████▍ | 664/782 [00:06<00:01, 104.16it/s, loss=0.174]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 106.41it/s, loss=0.174]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 106.24it/s, loss=0.175]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 106.12it/s, loss=0.173]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.98it/s, loss=0.173]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.87it/s, loss=0.175]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.77it/s, loss=0.175]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.67it/s, loss=0.173]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.58it/s, loss=0.174]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.47it/s, loss=0.173]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.36it/s, loss=0.174]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.22it/s, loss=0.173]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 105.07it/s, loss=0.174]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 104.91it/s, loss=0.176]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 104.80it/s, loss=0.175]\u001b[A\n",
      " 87%|████████▋ | 679/782 [00:06<00:00, 104.70it/s, loss=0.176]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.78it/s, loss=0.176]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.65it/s, loss=0.176]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.56it/s, loss=0.175]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.46it/s, loss=0.174]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.36it/s, loss=0.172]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.24it/s, loss=0.171]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.15it/s, loss=0.172]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 106.04it/s, loss=0.174]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.94it/s, loss=0.173]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.85it/s, loss=0.172]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.74it/s, loss=0.175]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.63it/s, loss=0.177]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.53it/s, loss=0.178]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.44it/s, loss=0.177]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.34it/s, loss=0.175]\u001b[A\n",
      " 89%|████████▊ | 693/782 [00:06<00:00, 105.16it/s, loss=0.175]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 107.36it/s, loss=0.175]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 107.20it/s, loss=0.175]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 107.06it/s, loss=0.173]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.96it/s, loss=0.173]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.84it/s, loss=0.17] \u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.73it/s, loss=0.172]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.60it/s, loss=0.173]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.50it/s, loss=0.176]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.40it/s, loss=0.176]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.32it/s, loss=0.174]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.22it/s, loss=0.172]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.12it/s, loss=0.171]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 106.03it/s, loss=0.171]\u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 105.93it/s, loss=0.17] \u001b[A\n",
      " 91%|█████████ | 708/782 [00:06<00:00, 105.75it/s, loss=0.168]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.76it/s, loss=0.168]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.62it/s, loss=0.166]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.50it/s, loss=0.167]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.40it/s, loss=0.167]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.26it/s, loss=0.17] \u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.16it/s, loss=0.167]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 107.05it/s, loss=0.169]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.96it/s, loss=0.167]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.83it/s, loss=0.164]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.69it/s, loss=0.163]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.59it/s, loss=0.162]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.45it/s, loss=0.163]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.36it/s, loss=0.161]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.25it/s, loss=0.158]\u001b[A\n",
      " 92%|█████████▏| 722/782 [00:06<00:00, 106.17it/s, loss=0.157]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 108.17it/s, loss=0.157]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 108.07it/s, loss=0.158]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.92it/s, loss=0.159]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.83it/s, loss=0.159]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.72it/s, loss=0.159]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.46it/s, loss=0.159]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.35it/s, loss=0.158]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.26it/s, loss=0.157]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.19it/s, loss=0.156]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 107.06it/s, loss=0.155]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 106.96it/s, loss=0.158]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 106.87it/s, loss=0.155]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 106.78it/s, loss=0.156]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 106.69it/s, loss=0.159]\u001b[A\n",
      " 94%|█████████▍| 736/782 [00:06<00:00, 106.60it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.57it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.49it/s, loss=0.159]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.39it/s, loss=0.159]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.31it/s, loss=0.158]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.23it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.14it/s, loss=0.158]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 108.06it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.97it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.80it/s, loss=0.156]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.67it/s, loss=0.156]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.57it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.44it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.34it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.25it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:06<00:00, 107.16it/s, loss=0.157]\u001b[A\n",
      " 96%|█████████▌| 750/782 [00:07<00:00, 107.02it/s, loss=0.156]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 109.08it/s, loss=0.156]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.96it/s, loss=0.156]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.84it/s, loss=0.157]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.75it/s, loss=0.155]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.66it/s, loss=0.157]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.57it/s, loss=0.158]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.48it/s, loss=0.16] \u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.37it/s, loss=0.16]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.26it/s, loss=0.159]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.17it/s, loss=0.16] \u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 108.05it/s, loss=0.16]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 107.80it/s, loss=0.159]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 107.69it/s, loss=0.158]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 107.56it/s, loss=0.157]\u001b[A\n",
      " 98%|█████████▊| 765/782 [00:07<00:00, 107.46it/s, loss=0.158]\u001b[A\n",
      "100%|█████████▉| 779/782 [00:07<00:00, 109.37it/s, loss=0.158]\u001b[A\n",
      "100%|█████████▉| 779/782 [00:07<00:00, 109.28it/s, loss=0.159]\u001b[A\n",
      "100%|█████████▉| 779/782 [00:07<00:00, 109.19it/s, loss=0.16] \u001b[A\n",
      "100%|█████████▉| 779/782 [00:07<00:00, 109.08it/s, loss=0.165]\u001b[A\n",
      "                                                              \u001b[A[ 0.       0.16491  0.13399  0.96089]\n",
      "\n",
      "  0%|          | 0/782 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|          | 0/782 [00:00<?, ?it/s, loss=0.166]\u001b[A\n",
      "  0%|          | 0/782 [00:00<?, ?it/s, loss=0.168]\u001b[A\n",
      "  0%|          | 0/782 [00:00<?, ?it/s, loss=0.167]\u001b[A\n",
      "  0%|          | 3/782 [00:00<00:31, 24.45it/s, loss=0.167]\u001b[A\n",
      "  0%|          | 3/782 [00:00<00:36, 21.20it/s, loss=0.167]\u001b[A\n",
      "  0%|          | 3/782 [00:00<00:43, 17.86it/s, loss=0.166]\u001b[A\n",
      "  0%|          | 3/782 [00:00<00:49, 15.73it/s, loss=0.17] \u001b[A\n",
      "  0%|          | 3/782 [00:00<00:54, 14.33it/s, loss=0.167]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:25, 30.53it/s, loss=0.167]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:27, 28.35it/s, loss=0.165]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:28, 26.75it/s, loss=0.166]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:30, 25.48it/s, loss=0.164]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:31, 24.23it/s, loss=0.162]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:33, 23.05it/s, loss=0.161]\u001b[A\n",
      "  1%|          | 7/782 [00:00<00:35, 21.60it/s, loss=0.16] \u001b[A\n",
      "  2%|▏         | 13/782 [00:00<00:19, 38.86it/s, loss=0.16]\u001b[A\n",
      "  2%|▏         | 13/782 [00:00<00:20, 36.81it/s, loss=0.16]\u001b[A\n",
      "  2%|▏         | 13/782 [00:00<00:22, 34.71it/s, loss=0.158]\u001b[A\n",
      "  2%|▏         | 13/782 [00:00<00:24, 31.49it/s, loss=0.157]\u001b[A\n",
      "  2%|▏         | 16/782 [00:00<00:21, 35.77it/s, loss=0.157]\u001b[A\n",
      "  2%|▏         | 16/782 [00:00<00:23, 32.75it/s, loss=0.157]\u001b[A\n",
      "  2%|▏         | 16/782 [00:00<00:24, 31.52it/s, loss=0.156]\u001b[A\n",
      "  2%|▏         | 16/782 [00:00<00:25, 30.45it/s, loss=0.155]\u001b[A\n",
      "  2%|▏         | 16/782 [00:00<00:26, 29.37it/s, loss=0.154]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:21, 36.00it/s, loss=0.154]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:22, 34.46it/s, loss=0.153]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:22, 33.63it/s, loss=0.152]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:23, 32.92it/s, loss=0.153]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:23, 32.18it/s, loss=0.152]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:24, 31.32it/s, loss=0.151]\u001b[A\n",
      "  3%|▎         | 20/782 [00:00<00:25, 30.15it/s, loss=0.148]\u001b[A\n",
      "  3%|▎         | 26/782 [00:00<00:19, 38.24it/s, loss=0.148]\u001b[A\n",
      "  3%|▎         | 26/782 [00:00<00:20, 37.26it/s, loss=0.147]\u001b[A\n",
      "  3%|▎         | 26/782 [00:00<00:21, 35.84it/s, loss=0.146]\u001b[A\n",
      "  3%|▎         | 26/782 [00:00<00:21, 34.84it/s, loss=0.145]\u001b[A\n",
      "  3%|▎         | 26/782 [00:00<00:22, 34.01it/s, loss=0.146]\u001b[A\n",
      "  3%|▎         | 26/782 [00:00<00:22, 32.88it/s, loss=0.147]\u001b[A\n",
      "  4%|▍         | 31/782 [00:00<00:19, 38.52it/s, loss=0.147]\u001b[A\n",
      "  4%|▍         | 31/782 [00:00<00:19, 37.70it/s, loss=0.145]\u001b[A\n",
      "  4%|▍         | 31/782 [00:00<00:20, 36.73it/s, loss=0.149]\u001b[A\n",
      "  4%|▍         | 31/782 [00:00<00:20, 35.79it/s, loss=0.148]\u001b[A\n",
      "  4%|▍         | 31/782 [00:00<00:21, 35.07it/s, loss=0.146]\u001b[A\n",
      "  4%|▍         | 31/782 [00:00<00:21, 34.50it/s, loss=0.145]\u001b[A\n",
      "  5%|▍         | 36/782 [00:00<00:18, 39.59it/s, loss=0.145]\u001b[A\n",
      "  5%|▍         | 36/782 [00:00<00:19, 38.73it/s, loss=0.145]\u001b[A\n",
      "  5%|▍         | 36/782 [00:00<00:19, 37.82it/s, loss=0.143]\u001b[A\n",
      "  5%|▍         | 36/782 [00:00<00:20, 36.75it/s, loss=0.144]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:20, 35.99it/s, loss=0.145]\u001b[A\n",
      "  5%|▍         | 36/782 [00:01<00:21, 35.39it/s, loss=0.146]\u001b[A\n",
      "  5%|▌         | 41/782 [00:01<00:18, 39.56it/s, loss=0.146]\u001b[A\n",
      "  5%|▌         | 41/782 [00:01<00:19, 38.61it/s, loss=0.146]\u001b[A\n",
      "  5%|▌         | 41/782 [00:01<00:19, 37.31it/s, loss=0.143]\u001b[A\n",
      "  5%|▌         | 41/782 [00:01<00:21, 35.07it/s, loss=0.145]\u001b[A\n",
      "  5%|▌         | 41/782 [00:01<00:21, 33.91it/s, loss=0.143]\u001b[A\n",
      "  5%|▌         | 41/782 [00:01<00:22, 33.64it/s, loss=0.143]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:19, 37.63it/s, loss=0.143]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:19, 37.38it/s, loss=0.141]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:19, 37.19it/s, loss=0.141]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:19, 37.02it/s, loss=0.139]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:19, 36.84it/s, loss=0.14] \u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 36.56it/s, loss=0.139]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 36.36it/s, loss=0.137]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 36.19it/s, loss=0.135]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 36.03it/s, loss=0.135]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 35.82it/s, loss=0.134]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 35.66it/s, loss=0.133]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 35.51it/s, loss=0.131]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:20, 35.23it/s, loss=0.132]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:21, 35.03it/s, loss=0.131]\u001b[A\n",
      "  6%|▌         | 46/782 [00:01<00:21, 34.87it/s, loss=0.132]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:15, 45.34it/s, loss=0.132]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 45.02it/s, loss=0.134]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 44.83it/s, loss=0.134]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 44.65it/s, loss=0.134]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 44.46it/s, loss=0.135]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 44.14it/s, loss=0.136]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 43.92it/s, loss=0.138]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 43.73it/s, loss=0.138]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 43.53it/s, loss=0.139]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 43.30it/s, loss=0.139]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 43.11it/s, loss=0.138]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 42.94it/s, loss=0.137]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:16, 42.74it/s, loss=0.135]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:17, 42.43it/s, loss=0.138]\u001b[A\n",
      "  8%|▊         | 60/782 [00:01<00:17, 42.20it/s, loss=0.14] \u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:13, 51.92it/s, loss=0.14]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:13, 51.62it/s, loss=0.14]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:13, 51.37it/s, loss=0.141]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:13, 51.17it/s, loss=0.14] \u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:13, 50.96it/s, loss=0.138]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:13, 50.72it/s, loss=0.14] \u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 50.35it/s, loss=0.142]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 50.12it/s, loss=0.142]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 49.87it/s, loss=0.143]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 49.62it/s, loss=0.142]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 49.39it/s, loss=0.142]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 49.21it/s, loss=0.14] \u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 49.03it/s, loss=0.138]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 48.80it/s, loss=0.136]\u001b[A\n",
      "  9%|▉         | 74/782 [00:01<00:14, 48.43it/s, loss=0.135]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 57.44it/s, loss=0.135]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 57.20it/s, loss=0.136]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 56.92it/s, loss=0.137]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 56.65it/s, loss=0.138]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 56.45it/s, loss=0.136]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 56.26it/s, loss=0.134]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 56.03it/s, loss=0.134]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 55.66it/s, loss=0.133]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 55.38it/s, loss=0.134]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 55.09it/s, loss=0.133]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 54.79it/s, loss=0.131]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 54.61it/s, loss=0.13] \u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 54.40it/s, loss=0.129]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 54.16it/s, loss=0.131]\u001b[A\n",
      " 11%|█▏        | 88/782 [00:01<00:12, 53.97it/s, loss=0.131]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:10, 62.41it/s, loss=0.131]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:10, 61.96it/s, loss=0.132]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 61.70it/s, loss=0.131]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 61.48it/s, loss=0.132]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 61.21it/s, loss=0.13] \u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 60.98it/s, loss=0.129]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 60.75it/s, loss=0.128]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 60.50it/s, loss=0.127]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 60.27it/s, loss=0.127]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 60.08it/s, loss=0.127]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 59.55it/s, loss=0.125]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 59.17it/s, loss=0.125]\u001b[A\n",
      " 13%|█▎        | 102/782 [00:01<00:11, 58.90it/s, loss=0.123]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 65.70it/s, loss=0.123]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 65.48it/s, loss=0.122]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 65.24it/s, loss=0.122]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 65.02it/s, loss=0.12] \u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 64.79it/s, loss=0.12]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 64.56it/s, loss=0.121]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 64.26it/s, loss=0.123]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 63.91it/s, loss=0.123]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 63.55it/s, loss=0.125]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 63.36it/s, loss=0.123]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 63.08it/s, loss=0.121]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 62.85it/s, loss=0.12] \u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 62.60it/s, loss=0.121]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 62.35it/s, loss=0.122]\u001b[A\n",
      " 15%|█▍        | 114/782 [00:01<00:10, 62.13it/s, loss=0.125]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 69.61it/s, loss=0.125]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 69.39it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 68.89it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 68.64it/s, loss=0.125]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 68.25it/s, loss=0.125]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 67.98it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 67.72it/s, loss=0.127]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 67.51it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 67.26it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 67.05it/s, loss=0.125]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 66.83it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 66.60it/s, loss=0.128]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 66.23it/s, loss=0.126]\u001b[A\n",
      " 16%|█▋        | 128/782 [00:01<00:09, 65.85it/s, loss=0.129]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:08, 72.39it/s, loss=0.129]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 141/782 [00:01<00:08, 72.11it/s, loss=0.129]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:08, 71.84it/s, loss=0.132]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:08, 71.61it/s, loss=0.133]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:08, 71.34it/s, loss=0.132]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:09, 71.14it/s, loss=0.132]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:09, 70.90it/s, loss=0.134]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:01<00:09, 70.67it/s, loss=0.134]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 70.45it/s, loss=0.135]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 70.21it/s, loss=0.134]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 69.85it/s, loss=0.138]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 69.58it/s, loss=0.14] \u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 69.36it/s, loss=0.14]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 69.08it/s, loss=0.139]\u001b[A\n",
      " 18%|█▊        | 141/782 [00:02<00:09, 68.81it/s, loss=0.137]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 75.50it/s, loss=0.137]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 75.25it/s, loss=0.138]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 75.04it/s, loss=0.143]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 74.80it/s, loss=0.143]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 74.54it/s, loss=0.146]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 74.23it/s, loss=0.148]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 73.78it/s, loss=0.147]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 73.52it/s, loss=0.147]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 73.31it/s, loss=0.149]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 73.09it/s, loss=0.148]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 72.81it/s, loss=0.149]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 72.55it/s, loss=0.149]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 72.28it/s, loss=0.148]\u001b[A\n",
      " 20%|█▉        | 155/782 [00:02<00:08, 72.05it/s, loss=0.146]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:07, 77.89it/s, loss=0.146]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:07, 77.49it/s, loss=0.144]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:07, 77.20it/s, loss=0.144]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:07, 76.96it/s, loss=0.145]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 76.74it/s, loss=0.143]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 76.51it/s, loss=0.142]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 76.32it/s, loss=0.14] \u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 76.12it/s, loss=0.14]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 75.90it/s, loss=0.139]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 75.48it/s, loss=0.139]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 75.27it/s, loss=0.138]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 75.02it/s, loss=0.138]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 74.78it/s, loss=0.137]\u001b[A\n",
      " 21%|██▏       | 168/782 [00:02<00:08, 74.56it/s, loss=0.135]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 80.19it/s, loss=0.135]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 79.97it/s, loss=0.133]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 79.71it/s, loss=0.132]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 79.32it/s, loss=0.131]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 79.00it/s, loss=0.13] \u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 78.77it/s, loss=0.128]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 78.57it/s, loss=0.127]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 78.34it/s, loss=0.126]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 78.10it/s, loss=0.127]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 77.87it/s, loss=0.127]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 77.51it/s, loss=0.127]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 77.23it/s, loss=0.129]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 77.05it/s, loss=0.127]\u001b[A\n",
      " 23%|██▎       | 181/782 [00:02<00:07, 76.72it/s, loss=0.126]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 82.10it/s, loss=0.126]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 81.89it/s, loss=0.126]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 81.70it/s, loss=0.126]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 81.49it/s, loss=0.126]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 81.10it/s, loss=0.125]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 80.86it/s, loss=0.123]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 80.52it/s, loss=0.123]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 80.30it/s, loss=0.121]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 80.11it/s, loss=0.122]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 79.90it/s, loss=0.124]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 79.70it/s, loss=0.13] \u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 79.46it/s, loss=0.131]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 79.13it/s, loss=0.131]\u001b[A\n",
      " 25%|██▍       | 194/782 [00:02<00:07, 78.85it/s, loss=0.13] \u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 84.01it/s, loss=0.13]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 83.81it/s, loss=0.133]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 83.59it/s, loss=0.134]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 83.34it/s, loss=0.132]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 83.13it/s, loss=0.131]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 82.90it/s, loss=0.133]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 82.69it/s, loss=0.132]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 82.50it/s, loss=0.131]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:06, 82.30it/s, loss=0.132]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 82.10it/s, loss=0.13] \u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 81.85it/s, loss=0.13]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 81.55it/s, loss=0.129]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 81.28it/s, loss=0.127]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 81.08it/s, loss=0.127]\u001b[A\n",
      " 26%|██▋       | 207/782 [00:02<00:07, 80.80it/s, loss=0.127]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 86.12it/s, loss=0.127]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 85.90it/s, loss=0.126]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 85.67it/s, loss=0.126]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 85.46it/s, loss=0.125]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 85.25it/s, loss=0.125]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 85.06it/s, loss=0.125]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 84.85it/s, loss=0.124]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 84.66it/s, loss=0.122]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 84.46it/s, loss=0.122]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 84.25it/s, loss=0.122]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 84.04it/s, loss=0.121]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 83.74it/s, loss=0.123]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 83.47it/s, loss=0.122]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 83.22it/s, loss=0.123]\u001b[A\n",
      " 28%|██▊       | 221/782 [00:02<00:06, 82.96it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 88.10it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 87.88it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 87.68it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 87.47it/s, loss=0.12] \u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 87.24it/s, loss=0.12]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 86.90it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 86.65it/s, loss=0.122]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 86.43it/s, loss=0.124]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 86.17it/s, loss=0.122]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 85.96it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 85.76it/s, loss=0.122]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 85.59it/s, loss=0.122]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 85.39it/s, loss=0.121]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 85.19it/s, loss=0.119]\u001b[A\n",
      " 30%|███       | 235/782 [00:02<00:06, 84.97it/s, loss=0.119]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:05, 89.85it/s, loss=0.119]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:05, 89.60it/s, loss=0.119]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:05, 89.40it/s, loss=0.117]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:05, 89.10it/s, loss=0.117]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:05, 88.91it/s, loss=0.116]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 88.70it/s, loss=0.116]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 88.50it/s, loss=0.116]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 88.32it/s, loss=0.116]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 88.12it/s, loss=0.115]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 87.89it/s, loss=0.115]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 87.69it/s, loss=0.113]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 87.51it/s, loss=0.113]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 87.28it/s, loss=0.115]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 86.73it/s, loss=0.117]\u001b[A\n",
      " 32%|███▏      | 249/782 [00:02<00:06, 86.55it/s, loss=0.115]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 91.29it/s, loss=0.115]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 91.08it/s, loss=0.114]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 90.89it/s, loss=0.113]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 90.70it/s, loss=0.115]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 90.50it/s, loss=0.116]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 89.98it/s, loss=0.115]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 89.38it/s, loss=0.114]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 89.18it/s, loss=0.115]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 88.96it/s, loss=0.119]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 88.75it/s, loss=0.12] \u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 88.56it/s, loss=0.121]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 88.38it/s, loss=0.123]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 88.20it/s, loss=0.121]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 88.03it/s, loss=0.123]\u001b[A\n",
      " 34%|███▎      | 263/782 [00:02<00:05, 87.85it/s, loss=0.125]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:02<00:05, 92.40it/s, loss=0.125]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 91.98it/s, loss=0.123]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 91.78it/s, loss=0.123]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 91.53it/s, loss=0.123]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 91.36it/s, loss=0.123]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 91.17it/s, loss=0.121]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 90.96it/s, loss=0.123]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 90.77it/s, loss=0.121]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 90.56it/s, loss=0.121]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 90.26it/s, loss=0.127]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 90.01it/s, loss=0.128]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 89.84it/s, loss=0.129]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 89.65it/s, loss=0.129]\u001b[A\n",
      " 35%|███▌      | 277/782 [00:03<00:05, 89.47it/s, loss=0.131]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 93.56it/s, loss=0.131]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 93.37it/s, loss=0.13] \u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 93.02it/s, loss=0.13]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 92.77it/s, loss=0.131]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 92.60it/s, loss=0.131]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 92.42it/s, loss=0.132]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 92.23it/s, loss=0.132]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 92.02it/s, loss=0.132]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 91.82it/s, loss=0.131]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 91.45it/s, loss=0.132]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 91.28it/s, loss=0.133]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 91.11it/s, loss=0.133]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 90.80it/s, loss=0.132]\u001b[A\n",
      " 37%|███▋      | 290/782 [00:03<00:05, 90.62it/s, loss=0.131]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 94.58it/s, loss=0.131]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 94.34it/s, loss=0.132]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 94.02it/s, loss=0.131]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 93.82it/s, loss=0.133]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 93.64it/s, loss=0.135]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 93.43it/s, loss=0.133]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 93.26it/s, loss=0.136]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 93.07it/s, loss=0.134]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 92.89it/s, loss=0.133]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 92.61it/s, loss=0.135]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 92.39it/s, loss=0.134]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 92.20it/s, loss=0.131]\u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 91.97it/s, loss=0.13] \u001b[A\n",
      " 39%|███▊      | 303/782 [00:03<00:05, 91.79it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 95.63it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 95.45it/s, loss=0.13] \u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 95.28it/s, loss=0.13]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 94.96it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 94.76it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 94.53it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 94.36it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 94.17it/s, loss=0.129]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 93.98it/s, loss=0.13] \u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 93.82it/s, loss=0.128]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 93.64it/s, loss=0.126]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:04, 93.30it/s, loss=0.127]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:05, 93.14it/s, loss=0.128]\u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:05, 92.99it/s, loss=0.13] \u001b[A\n",
      " 40%|████      | 316/782 [00:03<00:05, 92.74it/s, loss=0.131]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 96.72it/s, loss=0.131]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 96.55it/s, loss=0.131]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 96.36it/s, loss=0.129]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 96.17it/s, loss=0.128]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 95.84it/s, loss=0.127]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 95.66it/s, loss=0.126]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 95.43it/s, loss=0.127]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 95.24it/s, loss=0.127]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 95.08it/s, loss=0.125]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 94.85it/s, loss=0.128]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 94.70it/s, loss=0.128]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 94.42it/s, loss=0.129]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 94.21it/s, loss=0.128]\u001b[A\n",
      " 42%|████▏     | 330/782 [00:03<00:04, 93.98it/s, loss=0.127]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 97.58it/s, loss=0.127]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 97.42it/s, loss=0.127]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 97.26it/s, loss=0.127]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 97.10it/s, loss=0.127]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 96.93it/s, loss=0.125]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 96.74it/s, loss=0.125]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 96.45it/s, loss=0.126]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 96.28it/s, loss=0.126]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 96.11it/s, loss=0.124]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 95.89it/s, loss=0.124]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 95.71it/s, loss=0.125]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 95.52it/s, loss=0.124]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 95.36it/s, loss=0.124]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 95.20it/s, loss=0.124]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 95.03it/s, loss=0.125]\u001b[A\n",
      " 44%|████▍     | 343/782 [00:03<00:04, 94.75it/s, loss=0.124]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 98.76it/s, loss=0.124]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 98.60it/s, loss=0.122]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 98.41it/s, loss=0.122]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 98.21it/s, loss=0.121]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 98.01it/s, loss=0.122]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 97.82it/s, loss=0.12] \u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 97.66it/s, loss=0.118]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 97.50it/s, loss=0.118]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 97.28it/s, loss=0.117]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 97.02it/s, loss=0.117]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 96.84it/s, loss=0.12] \u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 96.69it/s, loss=0.12]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 96.48it/s, loss=0.119]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 96.28it/s, loss=0.119]\u001b[A\n",
      " 46%|████▌     | 358/782 [00:03<00:04, 96.11it/s, loss=0.121]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 99.77it/s, loss=0.121]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 99.58it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 99.39it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 99.20it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 98.97it/s, loss=0.119]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 372/782 [00:03<00:04, 98.78it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 98.52it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 98.36it/s, loss=0.118]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 98.19it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 97.99it/s, loss=0.118]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 97.84it/s, loss=0.119]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 97.69it/s, loss=0.123]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 97.54it/s, loss=0.121]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 97.39it/s, loss=0.121]\u001b[A\n",
      " 48%|████▊     | 372/782 [00:03<00:04, 97.24it/s, loss=0.124]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 100.81it/s, loss=0.124]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 100.46it/s, loss=0.124]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 100.23it/s, loss=0.122]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 100.02it/s, loss=0.121]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 99.87it/s, loss=0.12]  \u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 99.72it/s, loss=0.12]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 99.55it/s, loss=0.119]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 99.39it/s, loss=0.118]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:03, 99.18it/s, loss=0.116]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:04, 98.89it/s, loss=0.115]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:04, 98.70it/s, loss=0.114]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:04, 98.56it/s, loss=0.116]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:04, 98.39it/s, loss=0.116]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:04, 98.20it/s, loss=0.115]\u001b[A\n",
      " 49%|████▉     | 386/782 [00:03<00:04, 97.97it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 101.38it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 101.20it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 101.00it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 100.85it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 100.70it/s, loss=0.113]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 100.55it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 100.37it/s, loss=0.114]\u001b[A\n",
      " 51%|█████     | 400/782 [00:03<00:03, 100.17it/s, loss=0.115]\u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 99.89it/s, loss=0.117] \u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 99.66it/s, loss=0.116]\u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 99.51it/s, loss=0.116]\u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 99.35it/s, loss=0.122]\u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 99.19it/s, loss=0.131]\u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 99.00it/s, loss=0.131]\u001b[A\n",
      " 51%|█████     | 400/782 [00:04<00:03, 98.74it/s, loss=0.13] \u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 102.08it/s, loss=0.13]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 101.90it/s, loss=0.128]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 101.75it/s, loss=0.128]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 101.57it/s, loss=0.127]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 101.42it/s, loss=0.127]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 101.28it/s, loss=0.128]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 101.08it/s, loss=0.129]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 100.82it/s, loss=0.13] \u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 100.64it/s, loss=0.129]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 100.48it/s, loss=0.129]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 100.29it/s, loss=0.129]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 100.13it/s, loss=0.128]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 99.98it/s, loss=0.127] \u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 99.82it/s, loss=0.125]\u001b[A\n",
      " 53%|█████▎    | 414/782 [00:04<00:03, 99.57it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 102.83it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 102.66it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 102.45it/s, loss=0.124]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 102.32it/s, loss=0.127]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 102.17it/s, loss=0.128]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 102.02it/s, loss=0.127]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 101.88it/s, loss=0.126]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 101.61it/s, loss=0.125]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 101.45it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 101.25it/s, loss=0.127]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 101.10it/s, loss=0.125]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 100.96it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 100.81it/s, loss=0.124]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 100.66it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 100.53it/s, loss=0.123]\u001b[A\n",
      " 55%|█████▍    | 428/782 [00:04<00:03, 100.28it/s, loss=0.123]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 103.66it/s, loss=0.123]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 103.50it/s, loss=0.121]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 103.36it/s, loss=0.119]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 103.16it/s, loss=0.119]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 103.02it/s, loss=0.117]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 102.87it/s, loss=0.117]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 102.72it/s, loss=0.119]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 102.58it/s, loss=0.118]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 102.32it/s, loss=0.117]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 102.13it/s, loss=0.121]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 101.96it/s, loss=0.12] \u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 101.81it/s, loss=0.12]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 101.65it/s, loss=0.121]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 101.47it/s, loss=0.121]\u001b[A\n",
      " 57%|█████▋    | 443/782 [00:04<00:03, 101.32it/s, loss=0.121]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 104.43it/s, loss=0.121]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 104.28it/s, loss=0.12] \u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 104.12it/s, loss=0.119]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 103.83it/s, loss=0.117]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 103.62it/s, loss=0.119]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 103.46it/s, loss=0.118]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 103.31it/s, loss=0.118]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 103.16it/s, loss=0.119]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 103.00it/s, loss=0.118]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 102.85it/s, loss=0.119]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 102.68it/s, loss=0.119]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 102.53it/s, loss=0.12] \u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 102.27it/s, loss=0.121]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 102.08it/s, loss=0.119]\u001b[A\n",
      " 58%|█████▊    | 457/782 [00:04<00:03, 101.93it/s, loss=0.118]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:02, 104.76it/s, loss=0.118]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:02, 104.63it/s, loss=0.12] \u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:02, 104.17it/s, loss=0.12]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:02, 103.99it/s, loss=0.12]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:02, 103.73it/s, loss=0.121]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 103.34it/s, loss=0.119]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 103.14it/s, loss=0.12] \u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 102.87it/s, loss=0.121]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 102.71it/s, loss=0.12] \u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 102.49it/s, loss=0.12]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 102.23it/s, loss=0.118]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 101.78it/s, loss=0.122]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 101.65it/s, loss=0.124]\u001b[A\n",
      " 60%|██████    | 471/782 [00:04<00:03, 101.51it/s, loss=0.123]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 104.23it/s, loss=0.123]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 104.09it/s, loss=0.123]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 103.96it/s, loss=0.122]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 103.82it/s, loss=0.121]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 103.58it/s, loss=0.119]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 103.41it/s, loss=0.118]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 103.25it/s, loss=0.12] \u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 103.07it/s, loss=0.12]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.92it/s, loss=0.119]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.80it/s, loss=0.119]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.69it/s, loss=0.117]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.55it/s, loss=0.118]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.40it/s, loss=0.117]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.24it/s, loss=0.119]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 102.13it/s, loss=0.117]\u001b[A\n",
      " 62%|██████▏   | 484/782 [00:04<00:02, 101.91it/s, loss=0.117]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.96it/s, loss=0.117]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.80it/s, loss=0.117]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.66it/s, loss=0.116]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.47it/s, loss=0.115]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.33it/s, loss=0.115]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.18it/s, loss=0.114]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 104.04it/s, loss=0.117]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 103.92it/s, loss=0.12] \u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 103.79it/s, loss=0.121]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 103.63it/s, loss=0.121]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 103.51it/s, loss=0.12] \u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 103.38it/s, loss=0.121]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 103.10it/s, loss=0.121]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 102.92it/s, loss=0.121]\u001b[A\n",
      " 64%|██████▍   | 499/782 [00:04<00:02, 102.78it/s, loss=0.123]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 105.55it/s, loss=0.123]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 105.42it/s, loss=0.122]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 105.29it/s, loss=0.121]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 105.16it/s, loss=0.121]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 105.03it/s, loss=0.12] \u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 104.90it/s, loss=0.12]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 104.77it/s, loss=0.121]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 104.65it/s, loss=0.121]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 104.38it/s, loss=0.12] \u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 104.20it/s, loss=0.12]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 104.04it/s, loss=0.119]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 103.90it/s, loss=0.119]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 103.78it/s, loss=0.119]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 103.65it/s, loss=0.119]\u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 103.53it/s, loss=0.12] \u001b[A\n",
      " 66%|██████▌   | 513/782 [00:04<00:02, 103.41it/s, loss=0.119]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:04<00:02, 106.34it/s, loss=0.119]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:04<00:02, 106.07it/s, loss=0.121]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:04<00:02, 105.94it/s, loss=0.121]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:04<00:02, 105.74it/s, loss=0.122]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:04<00:02, 105.62it/s, loss=0.126]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 105.48it/s, loss=0.125]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 105.35it/s, loss=0.124]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 105.23it/s, loss=0.123]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.99it/s, loss=0.125]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.83it/s, loss=0.127]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.66it/s, loss=0.129]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.54it/s, loss=0.129]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.41it/s, loss=0.131]\u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.26it/s, loss=0.13] \u001b[A\n",
      " 68%|██████▊   | 528/782 [00:05<00:02, 104.12it/s, loss=0.132]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 106.80it/s, loss=0.132]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 106.62it/s, loss=0.131]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 106.38it/s, loss=0.13] \u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 106.25it/s, loss=0.128]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 106.07it/s, loss=0.129]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.96it/s, loss=0.128]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.85it/s, loss=0.126]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.72it/s, loss=0.125]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.55it/s, loss=0.124]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.43it/s, loss=0.124]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.28it/s, loss=0.124]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 105.05it/s, loss=0.124]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 104.91it/s, loss=0.122]\u001b[A\n",
      " 69%|██████▉   | 542/782 [00:05<00:02, 104.75it/s, loss=0.123]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 107.17it/s, loss=0.123]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 107.04it/s, loss=0.123]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.93it/s, loss=0.121]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.79it/s, loss=0.12] \u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.68it/s, loss=0.119]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.54it/s, loss=0.118]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.29it/s, loss=0.117]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.14it/s, loss=0.116]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 106.02it/s, loss=0.116]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 105.85it/s, loss=0.117]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 105.72it/s, loss=0.117]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 105.60it/s, loss=0.119]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 105.47it/s, loss=0.116]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 105.34it/s, loss=0.115]\u001b[A\n",
      " 71%|███████   | 555/782 [00:05<00:02, 105.20it/s, loss=0.116]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 107.78it/s, loss=0.116]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 107.53it/s, loss=0.119]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 107.35it/s, loss=0.12] \u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 107.19it/s, loss=0.119]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 107.05it/s, loss=0.121]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 106.93it/s, loss=0.12] \u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 106.80it/s, loss=0.12]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 106.68it/s, loss=0.118]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:01, 106.53it/s, loss=0.117]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:02, 106.41it/s, loss=0.116]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:02, 106.29it/s, loss=0.115]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:02, 106.17it/s, loss=0.114]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:02, 105.94it/s, loss=0.113]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:02, 105.78it/s, loss=0.114]\u001b[A\n",
      " 73%|███████▎  | 569/782 [00:05<00:02, 105.65it/s, loss=0.113]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 108.12it/s, loss=0.113]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.99it/s, loss=0.112]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.85it/s, loss=0.113]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.73it/s, loss=0.113]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.62it/s, loss=0.112]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.49it/s, loss=0.114]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.37it/s, loss=0.115]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.25it/s, loss=0.114]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 107.02it/s, loss=0.115]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 106.86it/s, loss=0.115]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 106.72it/s, loss=0.114]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 106.59it/s, loss=0.114]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 106.48it/s, loss=0.116]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 106.34it/s, loss=0.115]\u001b[A\n",
      " 75%|███████▍  | 583/782 [00:05<00:01, 106.23it/s, loss=0.114]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 108.71it/s, loss=0.114]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 108.59it/s, loss=0.116]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 108.46it/s, loss=0.119]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 108.23it/s, loss=0.118]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 108.08it/s, loss=0.118]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.93it/s, loss=0.117]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.80it/s, loss=0.115]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.67it/s, loss=0.116]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.55it/s, loss=0.116]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.44it/s, loss=0.114]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.31it/s, loss=0.116]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.20it/s, loss=0.114]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 76%|███████▋  | 597/782 [00:05<00:01, 107.08it/s, loss=0.113]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 106.96it/s, loss=0.119]\u001b[A\n",
      " 76%|███████▋  | 597/782 [00:05<00:01, 106.76it/s, loss=0.123]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 109.16it/s, loss=0.123]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 109.01it/s, loss=0.121]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.84it/s, loss=0.12] \u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.71it/s, loss=0.122]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.59it/s, loss=0.12] \u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.46it/s, loss=0.119]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.34it/s, loss=0.121]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.24it/s, loss=0.122]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 108.11it/s, loss=0.121]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 107.99it/s, loss=0.12] \u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 107.88it/s, loss=0.122]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 107.76it/s, loss=0.122]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 107.55it/s, loss=0.121]\u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 107.33it/s, loss=0.12] \u001b[A\n",
      " 78%|███████▊  | 611/782 [00:05<00:01, 107.22it/s, loss=0.121]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 109.60it/s, loss=0.121]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 109.46it/s, loss=0.125]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 109.33it/s, loss=0.125]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 109.22it/s, loss=0.123]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 109.10it/s, loss=0.122]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.88it/s, loss=0.121]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.73it/s, loss=0.12] \u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.59it/s, loss=0.121]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.45it/s, loss=0.119]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.32it/s, loss=0.118]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.20it/s, loss=0.118]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 108.07it/s, loss=0.117]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 107.85it/s, loss=0.122]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 107.69it/s, loss=0.123]\u001b[A\n",
      " 80%|███████▉  | 625/782 [00:05<00:01, 107.53it/s, loss=0.122]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 109.86it/s, loss=0.122]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 109.73it/s, loss=0.122]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 109.60it/s, loss=0.124]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 109.46it/s, loss=0.122]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 109.31it/s, loss=0.121]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 109.10it/s, loss=0.122]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.97it/s, loss=0.12] \u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.85it/s, loss=0.121]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.70it/s, loss=0.121]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.57it/s, loss=0.122]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.46it/s, loss=0.121]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.32it/s, loss=0.121]\u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.21it/s, loss=0.12] \u001b[A\n",
      " 82%|████████▏ | 639/782 [00:05<00:01, 108.00it/s, loss=0.118]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 110.08it/s, loss=0.118]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.98it/s, loss=0.117]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.87it/s, loss=0.118]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.70it/s, loss=0.121]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.55it/s, loss=0.121]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.42it/s, loss=0.119]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.29it/s, loss=0.122]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 109.16it/s, loss=0.122]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 108.94it/s, loss=0.12] \u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 108.81it/s, loss=0.119]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:05<00:01, 108.70it/s, loss=0.118]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:06<00:01, 108.57it/s, loss=0.124]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:06<00:01, 108.43it/s, loss=0.123]\u001b[A\n",
      " 83%|████████▎ | 652/782 [00:06<00:01, 108.32it/s, loss=0.121]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 110.41it/s, loss=0.121]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 110.30it/s, loss=0.121]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 110.19it/s, loss=0.119]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.98it/s, loss=0.117]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.82it/s, loss=0.116]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.69it/s, loss=0.116]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.58it/s, loss=0.117]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.48it/s, loss=0.118]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.36it/s, loss=0.117]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.23it/s, loss=0.12] \u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.13it/s, loss=0.119]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 109.02it/s, loss=0.121]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 108.81it/s, loss=0.119]\u001b[A\n",
      " 85%|████████▌ | 665/782 [00:06<00:01, 108.67it/s, loss=0.118]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.71it/s, loss=0.118]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.59it/s, loss=0.118]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.47it/s, loss=0.12] \u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.37it/s, loss=0.119]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.27it/s, loss=0.119]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.16it/s, loss=0.119]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 110.04it/s, loss=0.12] \u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.88it/s, loss=0.12]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.67it/s, loss=0.12]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.54it/s, loss=0.118]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.42it/s, loss=0.118]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.31it/s, loss=0.117]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.19it/s, loss=0.115]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 109.06it/s, loss=0.116]\u001b[A\n",
      " 87%|████████▋ | 678/782 [00:06<00:00, 108.92it/s, loss=0.116]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 111.10it/s, loss=0.116]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 111.00it/s, loss=0.114]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.89it/s, loss=0.116]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.79it/s, loss=0.115]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.69it/s, loss=0.114]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.58it/s, loss=0.113]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.43it/s, loss=0.117]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.32it/s, loss=0.115]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.22it/s, loss=0.115]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 110.09it/s, loss=0.114]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 109.98it/s, loss=0.116]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 109.88it/s, loss=0.119]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 109.77it/s, loss=0.119]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 109.61it/s, loss=0.119]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 109.50it/s, loss=0.118]\u001b[A\n",
      " 88%|████████▊ | 692/782 [00:06<00:00, 109.19it/s, loss=0.117]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 111.46it/s, loss=0.117]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 111.32it/s, loss=0.119]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 111.20it/s, loss=0.121]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 111.09it/s, loss=0.121]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.97it/s, loss=0.121]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.87it/s, loss=0.122]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.76it/s, loss=0.121]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.61it/s, loss=0.122]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.50it/s, loss=0.123]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.38it/s, loss=0.122]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.28it/s, loss=0.122]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 110.18it/s, loss=0.121]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 109.91it/s, loss=0.124]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 109.79it/s, loss=0.125]\u001b[A\n",
      " 90%|█████████ | 707/782 [00:06<00:00, 109.67it/s, loss=0.125]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.77it/s, loss=0.125]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.67it/s, loss=0.124]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.57it/s, loss=0.125]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.44it/s, loss=0.124]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.33it/s, loss=0.124]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.22it/s, loss=0.124]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 111.11it/s, loss=0.122]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.87it/s, loss=0.125]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.75it/s, loss=0.123]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.63it/s, loss=0.123]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.48it/s, loss=0.121]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.38it/s, loss=0.12] \u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.28it/s, loss=0.12]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.18it/s, loss=0.12]\u001b[A\n",
      " 92%|█████████▏| 721/782 [00:06<00:00, 110.08it/s, loss=0.123]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 112.15it/s, loss=0.123]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 112.03it/s, loss=0.124]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 111.91it/s, loss=0.122]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 111.79it/s, loss=0.122]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 111.55it/s, loss=0.122]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 111.35it/s, loss=0.126]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 111.20it/s, loss=0.125]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 111.07it/s, loss=0.124]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.97it/s, loss=0.123]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.87it/s, loss=0.124]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.76it/s, loss=0.124]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.51it/s, loss=0.127]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.35it/s, loss=0.127]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.25it/s, loss=0.125]\u001b[A\n",
      " 94%|█████████▍| 735/782 [00:06<00:00, 110.14it/s, loss=0.124]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 112.17it/s, loss=0.124]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 112.06it/s, loss=0.124]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.94it/s, loss=0.127]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.85it/s, loss=0.13] \u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.71it/s, loss=0.129]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.51it/s, loss=0.129]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.39it/s, loss=0.13] \u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.24it/s, loss=0.13]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.12it/s, loss=0.128]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 111.03it/s, loss=0.128]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 110.93it/s, loss=0.129]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 110.82it/s, loss=0.128]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 110.72it/s, loss=0.127]\u001b[A\n",
      " 96%|█████████▌| 749/782 [00:06<00:00, 110.53it/s, loss=0.126]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 112.37it/s, loss=0.126]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 112.27it/s, loss=0.128]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 112.14it/s, loss=0.13] \u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 112.01it/s, loss=0.129]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.91it/s, loss=0.127]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.81it/s, loss=0.126]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.61it/s, loss=0.127]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.49it/s, loss=0.127]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.39it/s, loss=0.128]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.27it/s, loss=0.127]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.16it/s, loss=0.125]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 111.05it/s, loss=0.124]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 110.95it/s, loss=0.124]\u001b[A\n",
      " 97%|█████████▋| 762/782 [00:06<00:00, 110.82it/s, loss=0.124]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 112.62it/s, loss=0.124]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 112.49it/s, loss=0.126]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 112.36it/s, loss=0.126]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 112.23it/s, loss=0.124]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 112.13it/s, loss=0.125]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 112.03it/s, loss=0.124]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 111.94it/s, loss=0.125]\u001b[A\n",
      " 99%|█████████▉| 775/782 [00:06<00:00, 111.85it/s, loss=0.128]\u001b[A\n",
      "                                                              \u001b[A[ 1.       0.12843  0.10644  0.96766]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fit(net, md, epochs= 2, opt=opt, metrics=metrics, crit= loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can change the learning rate for our optimizer using `set_lrs` from fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e327af36cc14dc89620497284a76fe7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       0.04212  0.07826  0.97681]                          \n",
      "[ 1.       0.03474  0.07826  0.97681]                          \n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_lrs(opt, 1e-2)\n",
    "fit(net, md, epochs= 2, opt=opt, metrics=metrics, crit= loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By decreasing learning rate, it improved a bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = predict(net, md.val_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 10)"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape # gave 10 predictions for all classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 8, 6, 9, 6, 9, 5, 3, 8, 4])"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(preds, axis=1)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 8, 6, 9, 6, 4, 5, 3, 8, 4])"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97670000000000001"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's check validationa accuracy manually too (just for practice)\n",
    "\n",
    "np.mean(y_valid == np.argmax(preds, axis=1))\n",
    "\n",
    "# it is same as shown in widget above"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Things from scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. `net` part from scratch (logistic layers)\n",
    "\n",
    "previously, we used `net` = nn.layers(l1,l2,l3) which are just combination of linear and non linear functions using matrix multiplications."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Like what I did for `gradient descent` above i.e. `Variable(torch object, require_grad = True)` to create a Variable, so that we can backtrack gradient from it, I will do same thing to save and backtrack weights. \n",
    "\n",
    "In order for `Variable` to be considered as nn module parameter, we can use very handy `nn_parameter` function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weights(*dims):\n",
    "    return nn.Parameter(torch.randn(dims)/dims[0])  # input is parameter tensor. # will start randomly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "class logitreg(nn.Module):  # taking some functionalities of torch's nn module - Base class for all neural network modules\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1_w = get_weights(28*28, 10) # random normal weights initialized (considering only 1 layer)\n",
    "        self.l1_b = get_weights(10) # constants to add\n",
    "    \n",
    "    \"\"\"\n",
    "    We have 28*28 features, 10k input and 10 output. We want each\n",
    "    \"\"\"\n",
    "    \n",
    "    def forward(self, x): # will multiply weights to x\n",
    "        y1 = x.mm(self.l1_w) + self.l1_b # linear layer\n",
    "        y2 = torch.log(torch.exp(y1)/torch.exp(y1).sum(dim = 0) ) # non linear logsoftmax layer\n",
    "        return y2 # output from logit    i.e. 10000x10 matrix        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using all above steps again, just replacing `net` with out `logit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "net2 = logitreg()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keeping opt same as previous\n",
    "opt = optim.SGD(net2.parameters(), lr = 1e-1, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96af71fc24c74c65801bb548593e8c76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       2.49141  2.45471  0.89401]                        \n",
      "[ 1.       2.498    2.44274  0.89829]                        \n",
      "\n"
     ]
    }
   ],
   "source": [
    "fit(net2, md, epochs=2, crit=loss, metrics = metrics, opt=opt )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok. Accuracy looks OKaish but what happended to loss? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "xm, ym = next(iter(md.trn_dl)) # because it was minibatch with bs = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
       " 0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
       " 0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
       "          ...             ⋱             ...          \n",
       " 0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
       " 0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
       " 0.0000  0.0000  0.0000  ...   0.0000  0.0000  0.0000\n",
       "[torch.FloatTensor of size 64x784]"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xmv = Variable(xm)\n",
    "xmv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 7\n",
       " 1\n",
       " 7\n",
       " 3\n",
       " 6\n",
       " 7\n",
       " 2\n",
       " 6\n",
       " 6\n",
       " 4\n",
       " 9\n",
       " 9\n",
       " 0\n",
       " 7\n",
       " 5\n",
       " 2\n",
       " 1\n",
       " 1\n",
       " 9\n",
       " 7\n",
       " 4\n",
       " 1\n",
       " 4\n",
       " 4\n",
       " 1\n",
       " 1\n",
       " 2\n",
       " 9\n",
       " 9\n",
       " 7\n",
       " 1\n",
       " 8\n",
       " 2\n",
       " 1\n",
       " 0\n",
       " 4\n",
       " 3\n",
       " 4\n",
       " 1\n",
       " 3\n",
       " 9\n",
       " 3\n",
       " 9\n",
       " 4\n",
       " 1\n",
       " 0\n",
       " 0\n",
       " 8\n",
       " 0\n",
       " 1\n",
       " 2\n",
       " 6\n",
       " 4\n",
       " 4\n",
       " 3\n",
       " 5\n",
       " 7\n",
       " 0\n",
       " 4\n",
       " 3\n",
       " 9\n",
       " 8\n",
       " 7\n",
       " 6\n",
       "[torch.LongTensor of size 64]"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds2 = net2(xmv).exp()\n",
    "preds2.max(1)[1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8962"
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = predict(net2, md.val_dl).argmax(1)\n",
    "np.mean(preds == y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10016"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_train = predict(net2, md.trn_dl).argmax(1)\n",
    "np.mean(preds_train == y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Why is it performing bad in training set?? Not possible"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. `Fit` from scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We fed `model, data, optimizer, loss criteria` into `fit` method. Need to take care of those."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is `fit` doing? -> It is doing forward pass (not actually), backward propagation, seeing gradient wrt weights and updating weights as to lower loss. So trainign is basically training of weights given our xs and ys. Let's try to replicate this ourselves. \n",
    "\n",
    "Basically, it's same as doing gradient descent in a loop (there used range(500)), so that it minimized loss funciton"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this, we need to use and understand `iterator`. For iterator, think like a function with a for loop, but rather than running all the loops at once and returning object, it returns objects after each iteration and stops. For next object we will have to use `next`, then it will go to next iteration of loop.\n",
    "\n",
    "`Iterator` of what? Now we will be using stochastic gradient descent. So input will have `bs` images at 1st iteration, do prediction, calculate gradient, change weights, THEN go to 2nd, 3rd etc... Here our data was in `ImageLoader`. So let's try to iterate data first. \n",
    "\n",
    "Btw, before we continue, gradients will be saved by putting weight tensors in `Variables` (because we take d(loss)/d(wi). Something to keep in mind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# initializing logit model (has random weights initially)\n",
    "\n",
    "net3 = logitreg() \n",
    "optimizer = optim.SGD(net3.parameters(), lr = 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let try iterator for Imagedata which was md. We can not iter(md), need it's input/output as iterable. \n",
    "# what do you mean by having model as iterable. nothing\n",
    "\n",
    "it = iter(md.trn_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [],
   "source": [
    "xit, yit = next(it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 784])"
      ]
     },
     "execution_count": 441,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xit.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets predict for this small batch and compare. net3 is forward pass\n",
    "\n",
    "y_pred_it = net3(Variable(xit))\n",
    "y_pred_data = y_pred_it.data.numpy()\n",
    "\n",
    "yit_data = Variable(yit)\n",
    "yit_data_np = yit_data.data.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**to_np** in Jeremy's nb is just function to convert `Variable` to `np`. I will use my own.\n",
    "\n",
    "To get np of Variable, we can just use Variable.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09375"
      ]
     },
     "execution_count": 443,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lets see accuracy on this random initialized net3\n",
    "\n",
    "np.mean(yit_data_np == y_pred_data.argmax(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3, 3, 3, 4, 5, 0, 2, 5, 6, 2, 0, 4, 4, 7, 3, 2, 3, 6, 7, 4, 9, 8, 6, 7, 4, 7, 6, 1, 7, 1, 0, 3, 5, 2,\n",
       "        7, 6, 2, 4, 1, 6, 4, 2, 6, 1, 4, 0, 7, 7, 7, 3, 7, 6, 6, 9, 7, 2, 1, 5, 5, 6, 7, 9, 1, 4]),\n",
       " array([1, 1, 1, 1, 9, 2, 0, 7, 0, 2, 4, 7, 5, 8, 2, 9, 9, 5, 5, 4, 0, 1, 9, 1, 7, 0, 9, 7, 7, 1, 4, 1, 9, 4,\n",
       "        2, 0, 6, 2, 2, 8, 9, 0, 0, 7, 4, 3, 5, 2, 8, 8, 2, 7, 5, 0, 5, 0, 2, 7, 9, 7, 8, 9, 2, 7]))"
      ]
     },
     "execution_count": 444,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yit_data_np, y_pred_data.argmax(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 4.1595\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 445,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# what is loss?\n",
    "\n",
    "l = loss(y_pred_it, yit_data);l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now above are accuracy and loss of data in first minibatch. Loss is high, accuracy is low. Should be because initial weights were random and we haven't even started gradient descent yet.\n",
    "\n",
    "Let's try this changing weights in a  loop. Theoritically, one should set `gradients` (gradient of weights) to 0 after each minibatch because it gradient for each minibatch are independent of each other. Let's try without it and how it affects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gradient of calculated loss above. \n",
    "l.backward()\n",
    "\n",
    "# optimizer with move one more step based on our gradient. \n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Second step** of predictions now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {},
   "outputs": [],
   "source": [
    "xit2, yit2 = next(it)\n",
    "y_pred = net3(Variable(xit2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 4.1594\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l2 = loss(y_pred, Variable(yit2));l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2, 7, 3, 4, 4, 7, 9, 7, 9, 7, 0, 3, 5, 0, 2, 0, 4, 1, 8, 2, 6, 3, 3, 5, 2, 5, 1, 7, 0, 6, 9, 4, 8, 1,\n",
       "        4, 9, 5, 2, 1, 0, 3, 5, 5, 0, 1, 2, 2, 5, 5, 5, 0, 2, 2, 9, 9, 9, 5, 5, 0, 1, 3, 8, 8, 2]),\n",
       " array([9, 1, 8, 1, 2, 7, 6, 1, 8, 4, 2, 5, 4, 4, 9, 6, 6, 6, 5, 8, 8, 8, 8, 4, 7, 1, 3, 9, 0, 3, 9, 0, 5, 7,\n",
       "        1, 8, 7, 4, 3, 0, 8, 4, 7, 6, 2, 1, 7, 7, 3, 7, 2, 1, 8, 4, 8, 8, 1, 5, 6, 8, 0, 8, 5, 2]))"
      ]
     },
     "execution_count": 463,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.data.numpy().argmax(1), Variable(yit2).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.109375"
      ]
     },
     "execution_count": 464,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy\n",
    "\n",
    "np.mean(Variable(yit2).data.numpy() == y_pred.data.numpy().argmax(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy went up a little. Loss will not change much and keep in mind it is stochastic gradient descent. Loss can increase too when moving from 1 minibatch to other"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Third step** of predictions now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gradient of calculated loss above. \n",
    "l2.backward()\n",
    "\n",
    "# optimizer with move one more step based on our gradient. \n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 471,
   "metadata": {},
   "outputs": [],
   "source": [
    "xit3, yit3 = next(it)\n",
    "y_pred = net3(Variable(xit3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 4.1559\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 472,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l3 = loss(y_pred, Variable(yit3));l3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2, 7, 6, 8, 1, 0, 0, 7, 0, 9, 0, 4, 5, 7, 9, 0, 6, 0, 2, 2, 0, 8, 9, 0, 6, 1, 2, 9, 2, 7, 8, 6, 8, 4,\n",
       "        9, 4, 6, 7, 9, 5, 7, 1, 2, 2, 9, 4, 2, 6, 7, 7, 4, 5, 7, 1, 8, 1, 7, 1, 0, 8, 2, 8, 5, 9]),\n",
       " array([8, 4, 8, 9, 3, 0, 6, 1, 6, 0, 0, 0, 6, 6, 7, 5, 2, 4, 9, 9, 9, 8, 3, 6, 2, 3, 8, 8, 8, 9, 3, 2, 6, 6,\n",
       "        8, 0, 4, 1, 8, 8, 9, 1, 9, 9, 3, 8, 2, 2, 1, 6, 4, 4, 5, 1, 9, 3, 1, 3, 2, 7, 8, 7, 8, 8]))"
      ]
     },
     "execution_count": 473,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.data.numpy().argmax(1), Variable(yit3).data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 474,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.109375"
      ]
     },
     "execution_count": 474,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accuracy\n",
    "\n",
    "np.mean(Variable(yit3).data.numpy() == y_pred.data.numpy().argmax(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dampening': 0,\n",
       "  'lr': 0.001,\n",
       "  'momentum': 0,\n",
       "  'nesterov': False,\n",
       "  'params': [Parameter containing:\n",
       "    1.1370e-03 -5.3062e-04 -4.8454e-04  ...  -2.6071e-04  1.7006e-03  6.2953e-04\n",
       "    2.6783e-03  6.2643e-04  1.4280e-03  ...   3.1493e-04  7.1249e-04 -9.8977e-04\n",
       "   -3.0561e-03 -1.1602e-03  3.9273e-04  ...   9.8646e-05  1.3782e-05  1.6302e-03\n",
       "                   ...                   ⋱                   ...                \n",
       "    1.3710e-03 -6.6718e-05  5.8136e-04  ...  -3.1057e-04 -7.8875e-04  7.1566e-04\n",
       "   -6.0872e-04  1.6282e-04 -2.2107e-03  ...  -7.0135e-04 -1.0192e-03 -1.8679e-03\n",
       "    1.7978e-03  1.7530e-03 -4.0816e-05  ...   8.0427e-04  2.2990e-04 -6.8521e-04\n",
       "   [torch.FloatTensor of size 784x10], Parameter containing:\n",
       "   -0.0989\n",
       "   -0.0604\n",
       "    0.1079\n",
       "    0.0808\n",
       "   -0.0405\n",
       "    0.1836\n",
       "    0.1635\n",
       "   -0.0095\n",
       "   -0.0195\n",
       "   -0.1505\n",
       "   [torch.FloatTensor of size 10]],\n",
       "  'weight_decay': 0}]"
      ]
     },
     "execution_count": 475,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.param_groups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets run same thing in a loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 503,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss =  4.159508228302002 \t accuracy =  0.0625\n",
      "loss =  4.148634910583496 \t accuracy =  0.34375\n",
      "loss =  4.137960910797119 \t accuracy =  0.671875\n",
      "loss =  4.125006198883057 \t accuracy =  0.78125\n",
      "loss =  4.114231586456299 \t accuracy =  0.75\n",
      "loss =  4.106197834014893 \t accuracy =  0.734375\n",
      "loss =  4.091577529907227 \t accuracy =  0.796875\n",
      "loss =  4.086956977844238 \t accuracy =  0.703125\n",
      "loss =  4.0747246742248535 \t accuracy =  0.75\n",
      "loss =  4.050030708312988 \t accuracy =  0.78125\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    xit, yit = next(it)\n",
    "    y_pred = net3(Variable(xit))\n",
    "    \n",
    "    #loss\n",
    "    l = loss(y_pred, Variable(yit))\n",
    "    \n",
    "    # accuracy\n",
    "    acc = np.mean(Variable(yit).data.numpy() == y_pred.data.numpy().argmax(1))\n",
    "    \n",
    "    if i%10 == 0:\n",
    "        print(\"loss = \", l.data[0], \"\\t accuracy = \", acc)\n",
    "    \n",
    "    optimizer.zero_grad() # need to re-initialize gradient before calling l.backward()\n",
    "    l.backward() # this will calculate grad and store them in w.grad()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, till above step, I tried without using `optimizer.zero_grad` and it gave pathetic results. Accuracy not improving, loss not decreasing after many iterations. Reason for this is that - **gradients in backprapagation when we used loss.backpro() were calculated using chain rule. means gradients were accumulated after each function and multiplies to give final gradient of loss wrt weigths**. Now after each minibatch completion, we restart this process, only update weights and gradient should start from 0. \n",
    "\n",
    "Now I edited and used .zero_grad and as we see accuracy is improving. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Putting above loop code in training loop to **spit out final validation score** and ** to run for n epochs**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 507,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "782"
      ]
     },
     "execution_count": 507,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "it = iter(md.trn_dl)\n",
    "len(it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {},
   "outputs": [],
   "source": [
    "#defining score to use in validation score test\n",
    "\n",
    "def score(xit, yit):  # we will give direct outout of next in this, i.e. similar to xit, yit\n",
    "    y_pred = net.forward(Variable(xit))\n",
    "    y_pred_np = y_pred.data.numpy()\n",
    "    return np.mean(yit.numpy() == y_pred_np.argmax(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final validation score  0.808618630573\n",
      "final validation score  0.833001592357\n",
      "final validation score  0.845242834395\n"
     ]
    }
   ],
   "source": [
    "net = logitreg()\n",
    "loss = nn.NLLLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 1e-3)\n",
    "epochs = 3\n",
    "\n",
    "for i in range(epochs):\n",
    "    losses = []\n",
    "    it = iter(md.trn_dl)\n",
    "    \n",
    "    for j in range(len(it)):\n",
    "    \n",
    "        xit, yit = next(it)\n",
    "        y_pred = net.forward(Variable(xit))\n",
    "\n",
    "        #loss\n",
    "        l = loss(y_pred, Variable(yit))\n",
    "        losses.append(l)\n",
    "\n",
    "        # accuracy\n",
    "        #acc = np.mean(Variable(yit).data.numpy() == y_pred.data.numpy().argmax(1))\n",
    "\n",
    "        #if i%10 == 0:\n",
    "         #   print(\"loss = \", l.data[0], \"\\t accuracy = \", acc)\n",
    "\n",
    "        optimizer.zero_grad() # need to re-initialize gradient before calling l.backward()\n",
    "        l.backward() # this will calculate grad and store them in w.grad()\n",
    "        optimizer.step() # this will update parameters(weights based on gradient descend)\n",
    "    \n",
    "    # calculating validation score after completing 1 epoch\n",
    "    val = iter(md.val_dl)\n",
    "    val_scores = [score(*next(val)) for k in range(len(val)) ] # *next will iterate through all nexts\n",
    "    print(\"final validation score \", np.mean(val_scores)) \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. `Optimizer`  and `backward` from scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, there are 2 things that we used from pytorch. \n",
    "\n",
    "1. **Optimizer** - which updated weights based on SGD . \n",
    "2. **loss.backward()** - which used chain rule to calculate derivates of loss wrt weights . \n",
    "\n",
    "Let's  make our own functions for those"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Formula for updatig weights =\n",
    "\n",
    "`weight_new = weight_previous - learning_rate * gradient_loss_wrt_weight`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copying baseline code from 1 step above and changing wherever required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final validation score  0.869625796178\n",
      "final validation score  0.884355095541\n",
      "final validation score  0.890923566879\n"
     ]
    }
   ],
   "source": [
    "net = logitreg()\n",
    "loss = nn.NLLLoss()\n",
    "# optimizer = optim.SGD(net.parameters(), lr = 1e-3)  -- this sir we don't have now\n",
    "epochs = 3\n",
    "lr = 1e-2\n",
    "\n",
    "w, b = net.l1_w, net.l1_b # according to our logreg function, we have them here\n",
    "\n",
    "for i in range(epochs):\n",
    "    losses = []\n",
    "    it = iter(md.trn_dl)\n",
    "    \n",
    "    for j in range(len(it)):\n",
    "    \n",
    "        xit, yit = next(it)\n",
    "        y_pred = net.forward(Variable(xit))\n",
    "\n",
    "        #loss\n",
    "        l = loss(y_pred, Variable(yit))\n",
    "        losses.append(l)\n",
    "\n",
    "        # optimizer.zero_grad() # how to make grad 0 yourself?\n",
    "        \n",
    "        \"\"\"\n",
    "        so, we know l.backward had saved gradients in w.grads. Just pull those and make em 0\n",
    "        but with an if condition as for first iteration, it won't find w.grads at all as\n",
    "        we don't have l.backward() run yet\n",
    "        \"\"\"\n",
    "        if w.grad is not None:\n",
    "            w.grad.data.zero_() # _ is for inplace\n",
    "            b.grad.data.zero_()\n",
    "   \n",
    "    \n",
    "        l.backward() # this will calculate grad and store them in w.grad()\n",
    "        \n",
    "        # optimizer.step() # this will update parameters(weights based on gradient descend)\n",
    "        \n",
    "        \"\"\"\n",
    "        now we need to update weights in parameters of net i.e. net.l1_w and net.l1_b\n",
    "        \"\"\"\n",
    "        \n",
    "        w.data = w.data - lr*w.grad.data\n",
    "        b.data = b.data - lr*b.grad.data\n",
    "        \n",
    "    \n",
    "    # calculating validation score after completing 1 epoch\n",
    "    val = iter(md.val_dl)\n",
    "    val_scores = [score(*next(val)) for k in range(len(val)) ] # *next will iterate through all nexts\n",
    "    print(\"final validation score \", np.mean(val_scores)) \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Didn't do `backward` from scratch yet. later. sleepy!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}