wrn_dfw_c10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D8mTA6N9FhYW"
   },
   "source": [
    "# Training WideResNet on Cifar10 using DFW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2136,
     "status": "ok",
     "timestamp": 1619336416846,
     "user": {
      "displayName": "Mansi Khamkar",
      "photoUrl": "",
      "userId": "07087059808651280926"
     },
     "user_tz": -330
    },
    "id": "8RhnLkxuxe_m",
    "outputId": "b2e45d9a-5a49-47e5-a326-60616f2c8610"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n",
      "/content/drive/MyDrive/DL_project\n",
      "/content/drive/MyDrive/DL_project\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive', force_remount=True)\n",
    "%cd /content/drive/MyDrive/DL_project/\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aD3z7C8Wxvmb"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision.datasets import CIFAR10\n",
    "from torchvision.datasets import CIFAR100\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from torchvision.transforms import *\n",
    "import os\n",
    "import time\n",
    "import gc\n",
    "import shutil\n",
    "import h5py\n",
    "\n",
    "from dfw.dfw import DFW\n",
    "from dfw.dfw.losses import set_smoothing_enabled\n",
    "from dfw.dfw.losses import MultiClassHingeLoss\n",
    "from dfw.experiments.models.densenet import DenseNet3\n",
    "from dfw.experiments.models.wide_resnet import WideResNet\n",
    "\n",
    "from train_utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5111,
     "status": "ok",
     "timestamp": 1619336450871,
     "user": {
      "displayName": "Mansi Khamkar",
      "photoUrl": "",
      "userId": "07087059808651280926"
     },
     "user_tz": -330
    },
    "id": "ikw6QVQ1x1Hg",
    "outputId": "d4f3f3fc-9a1a-45c7-dc89-3dbc8bdc367d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "50000 10000\n",
      "Files already downloaded and verified\n",
      "50000 10000\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose([transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "os.makedirs(\"data/cifar\", exist_ok=True)\n",
    "train_dataset_10 = CIFAR10(root='data/cifar', download=True, transform=transform)\n",
    "test_dataset_10 = CIFAR10(root='data/cifar', train=False, transform=transform)\n",
    "print(len(train_dataset_10), len(test_dataset_10))\n",
    "\n",
    "train_dataset_100 = CIFAR100(root='data/cifar', download=True, transform=transform)\n",
    "test_dataset_100 = CIFAR100(root='data/cifar', train=False, transform=transform)\n",
    "print(len(train_dataset_100), len(test_dataset_100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ISYgKg5rx2KK",
    "outputId": "1e67a53f-a8ff-4bf1-d681-73bb224bbbe1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda Tesla T4\n",
      "Epoch:0   Loss:0.82   Val loss:1.323   Accuracy:46.549   Best val loss:1.323   Time taken:170.656\n",
      "Epoch:1   Loss:0.567   Val loss:0.95   Accuracy:62.648   Best val loss:0.95   Time taken:170.26\n",
      "Epoch:2   Loss:0.428   Val loss:0.825   Accuracy:68.305   Best val loss:0.825   Time taken:169.958\n",
      "Epoch:3   Loss:0.359   Val loss:0.933   Accuracy:65.763   Best val loss:0.825   Time taken:169.937\n",
      "Epoch:4   Loss:0.332   Val loss:0.581   Accuracy:76.978   Best val loss:0.581   Time taken:169.473\n",
      "Epoch:5   Loss:0.304   Val loss:0.845   Accuracy:69.284   Best val loss:0.581   Time taken:169.655\n",
      "Epoch:6   Loss:0.289   Val loss:1.012   Accuracy:64.132   Best val loss:0.581   Time taken:169.203\n",
      "Epoch:7   Loss:0.28   Val loss:0.678   Accuracy:74.575   Best val loss:0.581   Time taken:169.273\n",
      "Epoch:8   Loss:0.269   Val loss:0.637   Accuracy:76.859   Best val loss:0.581   Time taken:169.535\n",
      "Epoch:9   Loss:0.261   Val loss:0.599   Accuracy:77.403   Best val loss:0.581   Time taken:169.614\n",
      "Epoch:10   Loss:0.253   Val loss:0.569   Accuracy:78.758   Best val loss:0.569   Time taken:169.42\n",
      "Epoch:11   Loss:0.25   Val loss:0.685   Accuracy:75.386   Best val loss:0.569   Time taken:169.466\n",
      "Epoch:12   Loss:0.243   Val loss:0.88   Accuracy:68.869   Best val loss:0.569   Time taken:168.443\n",
      "Epoch:13   Loss:0.239   Val loss:0.598   Accuracy:76.622   Best val loss:0.569   Time taken:168.87\n",
      "Epoch:14   Loss:0.235   Val loss:0.527   Accuracy:79.974   Best val loss:0.527   Time taken:169.591\n",
      "Epoch:15   Loss:0.225   Val loss:0.608   Accuracy:77.888   Best val loss:0.527   Time taken:169.719\n",
      "Epoch:16   Loss:0.225   Val loss:0.589   Accuracy:78.174   Best val loss:0.527   Time taken:169.586\n",
      "Epoch:17   Loss:0.22   Val loss:0.785   Accuracy:72.211   Best val loss:0.527   Time taken:169.224\n",
      "Epoch:18   Loss:0.219   Val loss:0.529   Accuracy:80.192   Best val loss:0.527   Time taken:168.021\n",
      "Epoch:19   Loss:0.214   Val loss:0.637   Accuracy:76.998   Best val loss:0.527   Time taken:168.011\n",
      "Epoch:20   Loss:0.211   Val loss:0.602   Accuracy:76.81   Best val loss:0.527   Time taken:169.087\n",
      "Epoch:21   Loss:0.207   Val loss:0.736   Accuracy:72.676   Best val loss:0.527   Time taken:169.121\n",
      "Epoch:22   Loss:0.203   Val loss:0.562   Accuracy:79.262   Best val loss:0.527   Time taken:168.641\n",
      "Epoch:23   Loss:0.202   Val loss:0.617   Accuracy:77.275   Best val loss:0.527   Time taken:168.17\n",
      "Epoch:24   Loss:0.197   Val loss:0.707   Accuracy:75.91   Best val loss:0.527   Time taken:168.751\n",
      "Epoch:25   Loss:0.192   Val loss:0.61   Accuracy:77.65   Best val loss:0.527   Time taken:168.731\n",
      "Epoch:26   Loss:0.181   Val loss:0.619   Accuracy:78.807   Best val loss:0.527   Time taken:169.019\n",
      "Epoch:27   Loss:0.17   Val loss:0.567   Accuracy:78.135   Best val loss:0.527   Time taken:168.848\n",
      "Epoch:28   Loss:0.151   Val loss:0.596   Accuracy:78.649   Best val loss:0.527   Time taken:168.634\n",
      "Epoch:29   Loss:0.128   Val loss:0.447   Accuracy:83.653   Best val loss:0.447   Time taken:168.929\n",
      "Epoch:30   Loss:0.086   Val loss:0.375   Accuracy:86.027   Best val loss:0.375   Time taken:168.445\n",
      "Epoch:31   Loss:0.043   Val loss:0.36   Accuracy:87.5   Best val loss:0.36   Time taken:167.459\n",
      "Epoch:32   Loss:0.018   Val loss:0.288   Accuracy:89.606   Best val loss:0.288   Time taken:166.722\n",
      "Epoch:33   Loss:0.009   Val loss:0.277   Accuracy:89.804   Best val loss:0.277   Time taken:167.737\n",
      "Epoch:34   Loss:0.005   Val loss:0.26   Accuracy:90.081   Best val loss:0.26   Time taken:167.936\n",
      "Epoch:35   Loss:0.004   Val loss:0.256   Accuracy:89.982   Best val loss:0.256   Time taken:167.448\n",
      "Epoch:36   Loss:0.003   Val loss:0.26   Accuracy:89.943   Best val loss:0.256   Time taken:166.958\n",
      "Epoch:37   Loss:0.003   Val loss:0.243   Accuracy:90.378   Best val loss:0.243   Time taken:167.962\n",
      "Epoch:38   Loss:0.002   Val loss:0.24   Accuracy:90.506   Best val loss:0.24   Time taken:167.676\n",
      "Epoch:39   Loss:0.002   Val loss:0.232   Accuracy:90.585   Best val loss:0.232   Time taken:167.824\n",
      "Epoch:40   Loss:0.002   Val loss:0.236   Accuracy:90.427   Best val loss:0.232   Time taken:167.982\n",
      "Epoch:41   Loss:0.002   Val loss:0.237   Accuracy:90.526   Best val loss:0.232   Time taken:168.062\n",
      "Epoch:42   Loss:0.002   Val loss:0.231   Accuracy:90.506   Best val loss:0.231   Time taken:168.386\n",
      "Epoch:43   Loss:0.002   Val loss:0.231   Accuracy:90.467   Best val loss:0.231   Time taken:168.106\n",
      "Epoch:44   Loss:0.002   Val loss:0.228   Accuracy:90.437   Best val loss:0.228   Time taken:167.848\n",
      "Epoch:45   Loss:0.002   Val loss:0.229   Accuracy:90.338   Best val loss:0.228   Time taken:168.817\n",
      "Epoch:46   Loss:0.003   Val loss:0.233   Accuracy:90.249   Best val loss:0.228   Time taken:168.435\n",
      "Epoch:47   Loss:0.002   Val loss:0.229   Accuracy:90.398   Best val loss:0.228   Time taken:168.222\n",
      "Epoch:48   Loss:0.002   Val loss:0.228   Accuracy:90.407   Best val loss:0.228   Time taken:168.551\n",
      "Epoch:49   Loss:0.003   Val loss:0.227   Accuracy:90.467   Best val loss:0.227   Time taken:168.4\n",
      "Epoch:50   Loss:0.003   Val loss:0.228   Accuracy:90.388   Best val loss:0.227   Time taken:168.574\n",
      "Epoch:51   Loss:0.003   Val loss:0.229   Accuracy:90.18   Best val loss:0.227   Time taken:168.592\n",
      "Epoch:52   Loss:0.002   Val loss:0.227   Accuracy:90.21   Best val loss:0.227   Time taken:168.384\n",
      "Epoch:53   Loss:0.002   Val loss:0.228   Accuracy:90.477   Best val loss:0.227   Time taken:168.804\n",
      "Epoch:54   Loss:0.003   Val loss:0.231   Accuracy:90.338   Best val loss:0.227   Time taken:168.998\n",
      "Epoch:55   Loss:0.003   Val loss:0.231   Accuracy:90.249   Best val loss:0.227   Time taken:169.136\n",
      "Epoch:56   Loss:0.002   Val loss:0.226   Accuracy:90.437   Best val loss:0.226   Time taken:169.013\n",
      "Epoch:57   Loss:0.003   Val loss:0.227   Accuracy:90.556   Best val loss:0.226   Time taken:169.3\n",
      "Epoch:58   Loss:0.002   Val loss:0.229   Accuracy:90.536   Best val loss:0.226   Time taken:168.941\n",
      "Epoch:59   Loss:0.002   Val loss:0.225   Accuracy:90.843   Best val loss:0.225   Time taken:169.334\n",
      "Epoch:60   Loss:0.002   Val loss:0.229   Accuracy:90.279   Best val loss:0.225   Time taken:169.53\n",
      "Epoch:61   Loss:0.003   Val loss:0.227   Accuracy:90.536   Best val loss:0.225   Time taken:169.653\n",
      "Epoch:62   Loss:0.003   Val loss:0.233   Accuracy:89.923   Best val loss:0.225   Time taken:169.808\n",
      "Epoch:63   Loss:0.003   Val loss:0.229   Accuracy:90.259   Best val loss:0.225   Time taken:169.597\n",
      "Epoch:64   Loss:0.002   Val loss:0.238   Accuracy:89.844   Best val loss:0.225   Time taken:169.251\n",
      "Epoch:65   Loss:0.002   Val loss:0.236   Accuracy:90.269   Best val loss:0.225   Time taken:169.451\n",
      "Epoch:66   Loss:0.002   Val loss:0.233   Accuracy:90.22   Best val loss:0.225   Time taken:170.075\n",
      "Epoch:67   Loss:0.002   Val loss:0.23   Accuracy:90.21   Best val loss:0.225   Time taken:170.15\n",
      "Epoch:68   Loss:0.003   Val loss:0.229   Accuracy:90.18   Best val loss:0.225   Time taken:169.906\n"
     ]
    }
   ],
   "source": [
    "trainer = Net_Trainer(device, 'wrn', 'wrn_dfw_c10', train_dataset_10, test_dataset_10, lr=1.0, optm='dfw', \n",
    "                      loss='svm', batch_size=128, epochs=100, weight_decay=1e-4, smooth=True, mom=0.9)\n",
    "best_epoch, losses, errors, accs = trainer.train()\n",
    "plot_train(losses, errors, accs)\n",
    "print('Best epoch:',best_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 894
    },
    "executionInfo": {
     "elapsed": 5055524,
     "status": "error",
     "timestamp": 1619334614798,
     "user": {
      "displayName": "Mansi Khamkar",
      "photoUrl": "",
      "userId": "07087059808651280926"
     },
     "user_tz": -330
    },
    "id": "SA2o8bBUjRRz",
    "outputId": "91e514b0-2e75-478d-86a1-f4d6cafaf5bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda Tesla T4\n",
      "Epoch:69   Loss:0.002   Val loss:0.238   Accuracy:89.794   Best val loss:0.225   Time taken:161.225\n",
      "Epoch:70   Loss:0.002   Val loss:0.233   Accuracy:89.962   Best val loss:0.225   Time taken:162.912\n",
      "Epoch:71   Loss:0.003   Val loss:0.232   Accuracy:90.042   Best val loss:0.225   Time taken:162.638\n",
      "Epoch:72   Loss:0.002   Val loss:0.232   Accuracy:90.101   Best val loss:0.225   Time taken:162.789\n",
      "Epoch:73   Loss:0.002   Val loss:0.236   Accuracy:89.656   Best val loss:0.225   Time taken:162.737\n",
      "Epoch:74   Loss:0.002   Val loss:0.237   Accuracy:89.557   Best val loss:0.225   Time taken:162.66\n",
      "Epoch:75   Loss:0.002   Val loss:0.229   Accuracy:90.012   Best val loss:0.225   Time taken:162.738\n",
      "Epoch:76   Loss:0.002   Val loss:0.233   Accuracy:89.775   Best val loss:0.225   Time taken:162.519\n",
      "Epoch:77   Loss:0.002   Val loss:0.23   Accuracy:89.903   Best val loss:0.225   Time taken:162.603\n",
      "Epoch:78   Loss:0.002   Val loss:0.234   Accuracy:89.389   Best val loss:0.225   Time taken:162.55\n",
      "Epoch:79   Loss:0.002   Val loss:0.228   Accuracy:89.933   Best val loss:0.225   Time taken:162.509\n",
      "Epoch:80   Loss:0.002   Val loss:0.237   Accuracy:89.508   Best val loss:0.225   Time taken:162.278\n",
      "Epoch:81   Loss:0.002   Val loss:0.23   Accuracy:89.735   Best val loss:0.225   Time taken:162.278\n",
      "Epoch:82   Loss:0.002   Val loss:0.233   Accuracy:89.27   Best val loss:0.225   Time taken:162.286\n",
      "Epoch:83   Loss:0.002   Val loss:0.267   Accuracy:88.014   Best val loss:0.225   Time taken:162.229\n",
      "Epoch:84   Loss:0.003   Val loss:0.23   Accuracy:89.399   Best val loss:0.225   Time taken:162.415\n",
      "Epoch:85   Loss:0.002   Val loss:0.235   Accuracy:89.201   Best val loss:0.225   Time taken:162.033\n",
      "Epoch:86   Loss:0.002   Val loss:0.225   Accuracy:89.537   Best val loss:0.225   Time taken:162.253\n",
      "Epoch:87   Loss:0.002   Val loss:0.234   Accuracy:89.033   Best val loss:0.225   Time taken:162.431\n",
      "Epoch:88   Loss:0.002   Val loss:0.231   Accuracy:89.458   Best val loss:0.225   Time taken:162.135\n",
      "Epoch:89   Loss:0.003   Val loss:0.323   Accuracy:86.294   Best val loss:0.225   Time taken:162.012\n",
      "Epoch:90   Loss:0.003   Val loss:0.222   Accuracy:89.725   Best val loss:0.222   Time taken:162.027\n",
      "Epoch:91   Loss:0.002   Val loss:0.228   Accuracy:89.389   Best val loss:0.222   Time taken:161.7\n",
      "Epoch:92   Loss:0.003   Val loss:0.247   Accuracy:88.311   Best val loss:0.222   Time taken:162.231\n",
      "Epoch:93   Loss:0.002   Val loss:0.242   Accuracy:88.627   Best val loss:0.222   Time taken:161.997\n",
      "Epoch:94   Loss:0.003   Val loss:0.24   Accuracy:88.855   Best val loss:0.222   Time taken:162.465\n",
      "Epoch:95   Loss:0.003   Val loss:0.225   Accuracy:89.191   Best val loss:0.222   Time taken:163.25\n",
      "Epoch:96   Loss:0.003   Val loss:0.234   Accuracy:88.954   Best val loss:0.222   Time taken:162.124\n",
      "Epoch:97   Loss:0.003   Val loss:0.241   Accuracy:88.41   Best val loss:0.222   Time taken:161.967\n",
      "Epoch:98   Loss:0.004   Val loss:0.238   Accuracy:88.835   Best val loss:0.222   Time taken:162.05\n",
      "Epoch:99   Loss:0.004   Val loss:0.252   Accuracy:88.004   Best val loss:0.222   Time taken:162.07\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-c8466e301ce7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNet_Trainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wrn'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wrn_dfw_c10'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataset_10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_dataset_10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_ckpt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'v1_ckpt_wrn_dfw_c10'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'dfw'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'svm'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight_decay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msmooth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mbest_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mplot_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Best epoch:'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbest_epoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/content/drive/My Drive/DL_project/train_utils.py\u001b[0m in \u001b[0;36mplot_train\u001b[0;34m(losses, errors, accs)\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mplot_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Train loss\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'blue'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Val loss\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'red'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Epochs'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'plt' is not defined"
     ]
    }
   ],
   "source": [
    "# Conitnue training by loading checkpoint\n",
    "# Best epoch = epoch with least val loss\n",
    "trainer = Net_Trainer(device, 'wrn', 'wrn_dfw_c10', train_dataset_10, test_dataset_10, load_ckpt='v1_ckpt_wrn_dfw_c10', lr=1.0, optm='dfw', loss='svm', batch_size=128, epochs=100, weight_decay=1e-4, smooth=True)\n",
    "best_epoch, losses, errors, accs = trainer.train()\n",
    "plot_train(losses, errors, accs)\n",
    "print('Best epoch:',best_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 607
    },
    "executionInfo": {
     "elapsed": 5830,
     "status": "ok",
     "timestamp": 1619336460169,
     "user": {
      "displayName": "Mansi Khamkar",
      "photoUrl": "",
      "userId": "07087059808651280926"
     },
     "user_tz": -330
    },
    "id": "64_rC4Gt-JZ2",
    "outputId": "a1af4acf-4b38-497f-afee-62a986cc2f0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda Tesla T4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU5dn48e9NwhK2AAFR9h2LLKIRF0oFtVbASq1LpWhFaa38XpdqVbS1alut+taqr611qXWpVam1LigqdRcVFRQEWUUECYiQKGGHLPfvj3uOmYRJMklmmMyc+3Nd55qZM2fOec5M8tznWc7ziKrinHMuvJqkOgHOOedSywOBc86FnAcC55wLOQ8EzjkXch4InHMu5DwQOOdcyHkgcBlLRF4QkbNTnY76EJEHReT6at6bLCJv7es0uczlgcA1KiKyLWopF5GdUa8n1WVfqjpWVR9KVlprIiJniMhqEZEq67NFZKOInJiKdDkXiwcC16ioautgAT4Hvh+17pFgOxHJTl0q4/I00A44usr6EwAFXtznKXKuGh4IXFoQkdEiUiAi00RkA/CAiLQXkedEZJOIfB153i3qM6+LyE8jzyeLyFsicktk289EZGw1x5omIk9UWfd/InJH1L5WicjWyH72Kqmo6i7gceAnVd76CfCoqpaKyL9FZIOIFIvImyJyUD2/m6NEZG5kP3NF5Kio92KmVUT6icgbkc8Uisi/6nNslxk8ELh0sj/QAegJnIf9/T4Qed0D2An8pYbPHw4sBzoC/wv8vWrVTcR0YJyItAEQkSzgdOBREWkF3AGMVdU2wFHAgmqO9xBwqojkRPaTC3w/sh7gBaA/sB/wIfBIrJ3UREQ6ADMjacoDbgVmikheLWn9PfBfoD3QDfhzXY/tMocHApdOyoFrVXW3qu5U1SJV/Y+q7lDVrcAN7F0VE22Nqv5NVcuwzPgAoHPVjVR1DZYxnxxZdQywQ1XfjUrHYBHJUdUvVHVxrIOp6tvAl1H7OR1YoaoLIu/fr6pbVXU3cB0wLBIs6mI88ImqPqyqpar6GLAMCzg1pbUEC6BdVHWXqnrjc4h5IHDpZFOkygUAEWkpIveIyBoR2QK8CbSLXMHHsiF4oqo7Ik9bV7Pto8DEyPMfR16jqtuBHwHnA1+IyEwRObCGNP+DiuqhsyKvEZEsEblJRD6NpH11ZJuONewrli7Amirr1gBda0nrFYAA74vIYhE5t47HdRnEA4FLJ1WHyv0lMBA4XFXbAt+JrI9V3VNX/wZGR9ocTiYSCABUdZaqfhcrUSwD/lbDfh4GjhWRI4EjqKj++TEwATgOyAV61TPt67Er+2g9gHU1pVVVN6jqz1S1C/Bz4K8i0q+Ox3YZwgOBS2dtsHaBzZG68msTtWNV3QS8jrVBfKaqSwFEpLOITIjUv+8GtmHVL9XtZzXwFvAY8JKqBqWSNpHPFwEtgT/UM6nPAwNE5MeRrqk/AgYBz9WUVhE5Laph/WssyFZ7Hi6zeSBw6ex2IAcoBN4l8V0yH8Wu2B+NWtcEuBS7Ev8Ka5OYWst+HsKu2v8Rte4fWBXOOmAJlv46U9Ui4ESsdFSEVfmcqKqFtaT1MOA9EdkGzAAuVtVV9UmDS3/iE9M451y4eYnAOedCzgOBc86FnAcC55wLOQ8EzjkXco194K69dOzYUXv16pXqZDjnXFr54IMPClW1U6z30i4Q9OrVi3nz5qU6Gc45l1ZEpOod6N/wqiHnnAs5DwTOORdyHgiccy7k0q6NwDmXuUpKSigoKGDXrl21b+xiatGiBd26daNp06Zxf8YDgXOu0SgoKKBNmzb06tWL2HMGuZqoKkVFRRQUFNC7d++4P+dVQ865RmPXrl3k5eV5EKgnESEvL6/OJSoPBM65RsWDQMPU5/sLTyD4+GO4+mooLEx1SpxzrlEJTyBYsQJuuAHWr091SpxzjVBRUREHH3wwBx98MPvvvz9du3b95vWePXtq/Oy8efO46KKL6nS8Xr16UdhILkzD01jctq09btmS2nQ45xqlvLw8FixYAMB1111H69atueyyy755v7S0lOzs2Flmfn4++fn5+ySdyRCeEkEQCIqLU5sO51zamDx5Mueffz6HH344V1xxBe+//z5HHnkkw4cP56ijjmL58uUAvP7665x44omABZFzzz2X0aNH06dPH+64445aj3PrrbcyePBgBg8ezO233w7A9u3bGT9+PMOGDWPw4MH861//AuDKK69k0KBBDB06tFKgaggvETjnGqVf/AIiF+gJc/DBEMln41ZQUMA777xDVlYWW7ZsYfbs2WRnZ/Pyyy/zq1/9iv/85z97fWbZsmW89tprbN26lYEDBzJ16tRq+/V/8MEHPPDAA7z33nuoKocffjhHH300q1atokuXLsycOROA4uJiioqKeOqpp1i2bBkiwubNm+v8HcQSnhJBbq49eiBwztXBaaedRlZWFmCZ8WmnncbgwYO55JJLWLx4cczPjB8/nubNm9OxY0f2228/vvzyy2r3/9Zbb3HyySfTqlUrWrduzQ9/+ENmz57NkCFDeOmll5g2bRqzZ88mNzeX3NxcWrRowZQpU3jyySdp2bJlQs7RSwTOuUaprlfuydKqVatvnv/mN79hzJgxPPXUU6xevZrRo0fH/Ezz5s2/eZ6VlUVpaWmdjztgwAA+/PBDnn/+ea6++mqOPfZYrrnmGt5//31eeeUVnnjiCf7yl7/w6quv1nnfVYWnRNCyJTRp4oHAOVdvxcXFdO3aFYAHH3wwIfscNWoUTz/9NDt27GD79u089dRTjBo1ivXr19OyZUvOPPNMLr/8cj788EO2bdtGcXEx48aN47bbbuOjjz5KSBrCUyIQsVKBBwLnXD1dccUVnH322Vx//fWMHz8+Ifs85JBDmDx5MiNGjADgpz/9KcOHD2fWrFlcfvnlNGnShKZNm3LXXXexdetWJkyYwK5du1BVbr311oSkQVQ1ITvaV/Lz87XeE9P07AnHHAMPPJDYRDnnEmLp0qV861vfSnUy0l6s71FEPlDVmH1cw1M1BFYi8O6jzjlXSfgCgVcNOedcJR4InHMu5DwQOOdcyIUrEOTmeiBwzrkqwhUIvETgnHN7SVogEJH7RWSjiHxczfuTRGShiCwSkXdEZFiy0vKNtm1h+3YoK0v6oZxz6WfMmDHMmjWr0rrbb7+dqVOnVvuZ0aNHE6tLe3XrG6NklggeBE6o4f3PgKNVdQjwe+DeJKbFBMNMbN2a9EM559LPxIkTmT59eqV106dPZ+LEiSlK0b6RtECgqm8CX9Xw/juq+nXk5btAt2Sl5Rs+FLVzrgannnoqM2fO/GYimtWrV7N+/XpGjRrF1KlTyc/P56CDDuLaa6+t034fe+wxhgwZwuDBg5k2bRoAZWVlTJ48mcGDBzNkyBBuu+02AO64445vhpk+44wzEnuC1WgsQ0xMAV5I+lF84Dnn0kcKxqHu0KEDI0aM4IUXXmDChAlMnz6d008/HRHhhhtuoEOHDpSVlXHssceycOFChg4dWush169fz7Rp0/jggw9o3749xx9/PE8//TTdu3dn3bp1fPyx1Z4HQ0rfdNNNfPbZZzRv3jxhw0zXJuWNxSIyBgsE02rY5jwRmSci8zZt2lT/g3kgcM7VIrp6KLpa6PHHH+eQQw5h+PDhLF68mCVLlsS1v7lz5zJ69Gg6depEdnY2kyZN4s0336RPnz6sWrWKCy+8kBdffJG2kfxp6NChTJo0iX/+85/VzoiWaCktEYjIUOA+YKyqFlW3nareS6QNIT8/v/6DI3kgcC59pGgc6gkTJnDJJZfw4YcfsmPHDg499FA+++wzbrnlFubOnUv79u2ZPHkyu3btatBx2rdvz0cffcSsWbO4++67efzxx7n//vuZOXMmb775Js8++yw33HADixYtSnpASFmJQER6AE8CZ6nqin1yUJ+cxjlXi9atWzNmzBjOPffcb0oDW7ZsoVWrVuTm5vLll1/ywgvx12SPGDGCN954g8LCQsrKynjsscc4+uijKSwspLy8nFNOOYXrr7+eDz/8kPLyctauXcuYMWO4+eabKS4uZtu2bck61W8kLcyIyGPAaKCjiBQA1wJNAVT1buAaIA/4q4gAlFY3Ml7CeInAOReHiRMncvLJJ39TRTRs2DCGDx/OgQceSPfu3Rk5cmTc+zrggAO46aabGDNmDKrK+PHjmTBhAh999BHnnHMO5eXlANx4442UlZVx5plnUlxcjKpy0UUX0a5du6ScY7RwDUO9dasFg1tugV/+MrEJc841mA9DnRg+DHVNWrWyCWq8ROCcc98IVyBo0gTatPH7CJxzLkq4AgH4eEPONXLpVl3d2NTn+/NA4JxrNFq0aEFRUZEHg3pSVYqKimjRokWdPtdY7izedzwQONdodevWjYKCAhp042jItWjRgm7d6jZiTzgDgbcRONcoNW3alN69e6c6GaETvqohn5zGOecqCV8g8Koh55yrxAOBc86FXDgDwdatPkuZc85FhDMQAOyDgZyccy4dhDcQePWQc84BHgiccy70PBA451zIhS8Q+OQ0zjlXSfgCgZcInHOukvAGAh9mwjnngDAHAi8ROOccEMZA0Lq1PXogcM45IIyBICvLgoEHAuecA8IYCMDHG3LOuShJCwQicr+IbBSRj6t5X0TkDhFZKSILReSQZKVlLw0NBD57knMugySzRPAgcEIN748F+keW84C7kpiWyhoaCI48Eq69NnHpcc65FEpaIFDVN4GvathkAvAPNe8C7UTkgGSlp5KGTE5TWgrz5sGSJYlNk3POpUgq2wi6AmujXhdE1iVfQ6arXL/ehrD2+xCccxkiLRqLReQ8EZknIvMSMql1Q6qGVq+2x82bG54O55xrBFIZCNYB3aNed4us24uq3quq+aqa36lTp4YfuSGBYM0ae/RA4JzLEKkMBDOAn0R6Dx0BFKvqF/vkyMEsZeXldf+slwiccxkmO1k7FpHHgNFARxEpAK4FmgKo6t3A88A4YCWwAzgnWWnZS9u21gV0+3Zo06Zunw1KBMXFtg+RxKfPOef2oaQFAlWdWMv7CvxPso5fo+jxhuobCPbsgV27ICcnsWlzzrl9LC0aixOuIQPPBVVD4NVDzrmMEM5AEExOU9cuoOXl8Pnn0KNH/T7vnHONUDgDQbdu9hhU88RrwwarEho2zF57icA5lwHCGQgGDIAmTWLfHbxxY/W9iYLAMXSoPXogcM5lgHAGgubNoV8/WLy48vovvrBqn6efjv25IBAcfLA9eiBwzmWAcAYCgEGD9i4RzJ0Lu3fDZ5/F/kzQUBxUDXkbgXMuA4Q7EHzyidX5B+bPt8fqMvg1ayAvD7pGhkTyEoFzLgOEOxCUlsLKlRXragsEq1dDz55270DTph4InHMZIdyBACpXD8VTIujVy+4mbtfOA4FzLiOENxAMHGgZehAIiorsHgGIHQhULRD07Gmvc3O9jcA5lxHCGwhatoQ+fSoCwYIF9piVFTuDLyyEHTsqAoGXCJxzGSK8gQAq9xwKqoUOPTR2IAi6jvbqZY+JCAQlJfDppw3bh3PONZAHguXLrdF4/ny747hv35oDQSJLBPffDwcdZENiO+dcingg2LMHVq2yQDB8ePXzGQf3EAQlgkS0ESxfbvctbNzYsP0451wDeCAAu5Fs+fKKQFBdiaBtWysJQGJKBOsiE7IVFTVsP8451wDhDgQHHmiP06fb+EJBIAjmGogW3EMQaNfOGo9LSup//CAQfPVV/ffhnHMNFO5A0Lq1Ze4vvmivg0AAe5cKgnsIAvUdyjqalwicc41AuAMBVNxh3L69DThXUyCoWiKA+lcPqcL69fbcA4FzLoU8EATtBMOH2w1msQLB7t32ev/9K9Y1NBAUFVWMc+RVQ865FPJAEB0IIHYgCK7YO3asWNfQQBBUC0Xv3znnUiA0gWDNGnjwwRhd9oMhpfPz7TFWICgstMfoQNDQNgIPBM65RiKpgUBEThCR5SKyUkSujPF+DxF5TUTmi8hCERmXrLTMnQvnnBNjqoFDD4WXXoLTTrPX8QaCRJUI9t/fq4accymVtEAgIlnAncBYYBAwUUQGVdnsauBxVR0OnAH8NVnpCfLwTZtivHnccTbGEOzbQCBidxZ7icA5l0LJLBGMAFaq6ipV3QNMByZU2UaBtpHnucD6ZCWmUyd7jBkIorVpY4+1BYLWrW3e44YEgv32sxKBBwLnXAolMxB0BdZGvS6IrIt2HXCmiBQAzwMXxtqRiJwnIvNEZN6mWnPy2IJAEOTp1crKsmAQKxB06FCxrkkTu9O4IW0EXbvajGdeNeScS6FUNxZPBB5U1W7AOOBhEdkrTap6r6rmq2p+pyBHr6MOHawmJq44UnWYicJCqwpq2rTydg0ZZiIIBB062LFKS+u3H+eca6BkBoJ1QPeo190i66JNAR4HUNU5QAugI0mQnW33jNU7EHSMkaxEBIK8PHtd11LBzp3QvTvMmFG/4zvnXEQyA8FcoL+I9BaRZlhjcNVc63PgWAAR+RYWCOpX9xOHTp3iDARVq3wKCysy7Gj1HYF01y5rF2hIIFi9GgoKKuZRcM65ekpaIFDVUuACYBawFOsdtFhEficiJ0U2+yXwMxH5CHgMmKyqmqw0deoURxsB7J3BFxUltkQQDC0RVA0Fx6jPPurZZuKcc4HsZO5cVZ/HGoGj110T9XwJMDKZaYjWqROsWBHHhrm5lWcOKyyEIUP23q6+gSC4h6AhgSDYR1yRzTnnqpfqxuJ9qmPHRtJGEGTiXbrUv2oo2IeXCJxzDZTUEkFj06mTXXiXl1vvz2pFB4IdO2yJFQhyc23Milp3WEV0iSD4XH1LBB4InHMNFKoSQadOUFYWx0V8bq6NOLp7d+wB5wLt2tlw0rGmtqzJunWQk2Ofb9vWujR51ZBzLkVCFwggjovo6GEmYt1VHKjvMBNB11ERWzp0qH/VUGGhBaN43H8/PPJI3Y7jnMt4oQoENY43FG1fBYJAhw717zVUUhJ/ieTGG+H22+t2HOdcxgtVIEh4iaC+Q1GvX185EOTl1S0QlJXBhg02oxrE106wZ48Nvfr553VLq3Mu44UyENRarZ7MEkEwRWXVQFCXqqEvv7RgEMylEE87waef2mc2brS7kp1zLiKuQCAirYIxgERkgIicJCJNa/tcY1PnqqEtWyyTFbHxKaqqTyAoKrJG6IZUDQXtA0EgiKdEsGxZxfOCguq3Ky6GpUvjT4tzLu3FWyJ4E2ghIl2B/wJnAQ8mK1HJkpMDrVrVo2qoQ4eK+QpibVeXQBDddTRQ16qhYB8HH2yP8ZQIli+veL5mTfXbXXMNjBhhbQ/OuVCINxCIqu4Afgj8VVVPAw5KXrKSJ67xhqoGgljVQlW3i1d1gWDnzvirbIKG4rqUCJYvt26qUHM7wRtvwLZtsHBhfGlxzqW9uAOBiBwJTAJmRtbFuERu/OIab6htZK6c2gJBdrZNUNPQEkEwzES87QTr1lkJpXdvaN48/hLB4YdbNVd1gWDLFli0yJ6//358aXHOpb14A8EvgKuApyIDx/UBXktespInrhJBdrbVIdUWCKDuw0wUFFhmfMABFeuCYSbirR5at84+n5UV3wmpWhvBkCH2ueoCwXvv2V3SwXPnXCjENcSEqr4BvAEQaTQuVNWLkpmwZOnYseKit0bBMBOFhXDYYdVv17593bpkLlkCffpUnuSmPiWCoETRsWPtJYLCQvj6axg40LqcVtdG8PbbNuTFt7/tJQLnQiTeXkOPikhbEWkFfAwsEZHLk5u05AguoGu9GTc6ENRUIjj5ZHjlFViwIL4ELFy490im9SkRBIEgnhJB0FA8cCD07Fl94HrnHUvbccdZCaK+03A659JKvFVDg1R1C/AD4AWgN9ZzKO106mTzwuzYUcuGubmW4e7ZE3tSmsAll9i2111X+8F37oSVK2Ho0Mrr6xMIunSx5/GUCKIDQY8esHZtRRVQoKwM3n0XRo60XkOqMG9efOlxzqW1eANB08h9Az8AZqhqCZC0CWSSqU53FwdzEtTWRnDppfDMM/DBBzXvc8kSy4CrlgjqUjW0bZs16ta1RNC8uZUGevSw+xg2bqy8zccf20iqRx1VURXm7QTOhUK8geAeYDXQCnhTRHoCdRxys3Go001lQWZZUyAAuPhiayuorVQQNE5UDQQtW0KLFvGVCKJnNwvStmWLlVyqs2wZ9O9vjcs9e9q6qtVDb79tjyNHWmDq39/bCZwLibgCgareoapdVXWcmjXAmCSnLSnqPN4Q1B4IcnPhssvguedqzjwXLrQMv1+/vd+L96ayqt1P4xk3Y/lyqxaCivGJqgaCd96xHkVBoDj8cCsRJG/mUOdcIxFvY3GuiNwqIvMiy5+w0kHaqfN4Q1B7IAC48ELLzK+5pvptFi2Cgw6KfZdyvENRVw0EQdqiT+jMM+Gee+x5SQmsWlV7IHj7bSsNiNjrESNsYLuahqNwzmWEeKuG7ge2AqdHli3AA8lKVDIlpUQA0KYNXHklzJpld+fGsmhR7LmPoe4lgqCxuOoJFRfbnAOXXmqNwqtWQWlpRSBo185ugovuQrp+Paxebe0DgcMPt0evHnIu48UbCPqq6rWquiqy/Bbok8yEJUvbttaFv9ZAENxdnJVVOSjU5H/+x67Ur7pq7yqVjRtt1NBEBII2bWyBvUsEQTfWHTusuiq6xxDYFX/VLqRz5thjdCAYNgyaNatoMN6924axds5lnHgDwU4R+XbwQkRGArUOjCMiJ4jIchFZKSJXVrPN6SKyREQWi8ijcaan3kTinMQ+yPzz8uKfjzgnxxqM58yBZ5+t/F7QUFy162gg3qqhqkNYVy0RBIFg6lR4/HG4+257HQQCsOqh6EAwe7a1XQwfXrGueXMb1O711+H66y149O9feRRT51xGiDcQnA/cKSKrRWQ18Bfg5zV9QESygDuBscAgYKKIDKqyTX9s6IqRqnoQNpRF0sU13lAQCOKpFoo2eTIMGAC/+pX1zQ9U12MoEJQIqpYkysrgwQftzmCIPbsZVJzQ/Pmw//7wpz9Br17wwgvQuXPFkNlQORCUllrAOO44KwFEGzEC5s6F3/zGgoII3HdfnF+Ecy5dxNtr6CNVHQYMBYaq6nDgmFo+NgJYGalK2gNMByZU2eZnwJ2q+nXkOFU6tydHnUYgrWsgyM6GG26AxYsrzw+8aJEduHPn2J/Ly7NMeevWyutnzoRzzoHvfMdKA9E3kwXH69Ch4oTmz7dMOycHbrvN1kWXBsCu7gsLYft2+O9/4Ysv7BhVXXCBVXMtXgwvvggnnQQPPVRzV1XnXNqp0wxlqrolcocxwKW1bN4VWBv1uiCyLtoAYICIvC0i74rICbF2JCLnBT2WNsUz5HIt6lQ1VNdAAHDKKZCfb6WCYEC6hQurrxaKPs4XX1Re/+qrVk2zerX16vnii8olguCzhYVWj79kSUUVz4QJlpn/5CeVtw96Dq1dCw88YJ8/8cS90zRwIPzhDzAoUpD72c/sOM88U7HN+vXw859XlFicc2mnIVNVSgKOnw30B0YDE4G/iUi7qhup6r2qmq+q+Z2COvEGSGqJAKwK5a67rPvlxRdb9c7ixdVXCwEccYQ9vvpq5fWvvWaDwL32mt1VXFq6dyAITmjxYns/CAQi8Oc/w5QplbcPAsH8+TBjBkyatHe1UCzf/a59NqgeKiuzrqr33mttCc65tNSQQFDbnUbrgO5Rr7tF1kUrIDJkhap+BqzAAkNSdepkvSxrnISrIYEArETw61/DP/4Bt95q4wzVFAgGDrT5BV54oWJdYaGVJEaPtv299RaccAKMqXIvX9DoMX++vQ5mLqtOcNPYzTdbNU+saqFYsrLg3HPhpZeshHLLLRagwMZQcs6lpRoDgYhsFZEtMZatQJeaPgvMBfqLSG8RaQacAcyoss3TWGkAEemIVRWtqs+J1EVcN5W1bw/HHGN18/X161/b1fm0afa6pkAgAmPH2kimu3bZuuB+hCDjHzjQAsVBVSaHC+q6FiywbqV9+9acri5drCfURx9Z+oKZzuIRBI1f/AKuvhpOPdXaN4JxmZxzaafGQKCqbVS1bYyljarWOJeBqpYCFwCzgKXA45FJbX4nIidFNpsFFInIEmyim8tVtQ6T99ZPXOMNZWVZpvy979X/QM2aWYmgaVPL6Ktm4FWNG2f9/2fPttevvWbjENU0HwJUlAg+/NAy9dq6u2ZnV1QvTZ4c16l8o0cP+06eecaGpLj3XhsywwOBc2mrIVVDtVLV51V1gKr2VdUbIuuuUdUZkeeqqpeq6iBVHaKq05OZnsD++9tjMH5bUg0eDH/9qzW0tmxZ87ZjxljDcFA9FLQP1FZ/37GjtQ3MnVt7tVCgRw8LUD/+cXzbR7vkEuuO+vDDVnLq29erhpxLY0kNBI1V/0grxCef7KMDTplSMfZPTVq2tPaA55+3u5CXLNm7PSCWoK6rpKTyTWE1mToVbryxfm0gxx9vxamjj7bXffvafQnerdS5tBTXVJWZpnNnq0pfsSLVKYlh7Firf38gMpRTPIEgOjOPNxBMmlT3tEXLjvrT6dvX5llYs6Yiyjrn0kYoSwQidvPvPisR1MXYsfZ4000WrQ49tPbPBCWC7OyKPv/7UjCstlcPOZeWQhkIwC5cG2WJoH9/u8IuLoZRoypfeVcnKBEcdJC1MexrQS8lbzB2Li2FNhAMGGBd4XfvTnVKqgi6kUJ81UJQUSKIt1oo0Tp3hlatPBA4l6ZCHQhUG2nedfrp1qNn3Lj4tm/ZEs47D84+O7npqo6I9xxyLo2FsrEYLBCAtROkolq9RqNGWdVQTk5824vE1yspmfr29SGqnUtToS0RBJ1bGmU7AcQfBBqLvn1tNrTy8lSnxDlXR6ENBO3aWdV6ow0E6aZvX2twWVd1OCnnXGMX2kAAVj3kgSBBgi6kjbLRxTlXEw8EHggSw7uQOpe2Qh8INmzYe1IwVw/du9s9D95zyLm0E/pAAI30DuN0k51t8yl4icC5tBPqQNDoew6lm759PRA4l4ZCHQiC9k0PBAkS3FSmtU1e55xrTEIdCHJybFh+DwQJ0rcvbH0QpcsAABihSURBVNkCRUmfW8g5l0ChDgTQiEchTUc+CqlzackDQaQLqddmJMCBB9rjggWpTYdzrk5CHwj694fNm2uZyN7Fp18/6NkTXnwx1SlxztVB6ANB0IV0+fLUpiMjBENov/JKIxzf2zlXndAHgmHD7HHOnNSmI2OMGwfbtsFbb6U6Jc65OCU1EIjICSKyXERWisiVNWx3ioioiOQnMz2xdO1qweDZZ/f1kTPUMcdAs2bwwgupTolzLk5JCwQikgXcCYwFBgETRWSvkf9FpA1wMfBestJSm+9/H95+G776KlUpyCCtWsF3vuOBwLk0kswSwQhgpaquUtU9wHRgQoztfg/cDOxKYlpqdOKJNoy+510JMm4cLFkCa9akOiXOuTgkMxB0BdZGvS6IrPuGiBwCdFfVmTXtSETOE5F5IjJv06ZNCU/oYYfZtLtePZQgwZzLHlmdSwspaywWkSbArcAva9tWVe9V1XxVze8UTNSeQE2awPjx1uuxpCThuw+fgQOhVy8PBM6liWQGgnVA96jX3SLrAm2AwcDrIrIaOAKYkYoGY7DqoeJi7+ySECJWPeTdSJ1LC8kMBHOB/iLSW0SaAWcAM4I3VbVYVTuqai9V7QW8C5ykqvOSmKZqffe71tnluedScfQMNHYsbN8OTz2V6pQ452qRtECgqqXABcAsYCnwuKouFpHfichJyTpufbVuDWPGeDtBwhx7LHzrWzBpEvz2t1BWluoUOeeqkdQ2AlV9XlUHqGpfVb0hsu4aVZ0RY9vRqSoNBL7/fRuAzkcjTYCcHHjvPfjxj+G66+C443wcD+caqdDfWRzt+9+3huPrrvNB6BKiTRv4xz/ggQes8eXmm1OdIudcDB4IovToYbUYjz0Gf/97qlOTIURg8mSrd3v++VSnxjkXgweCKq66ymoxLrwQFi1KdWoyyNixfpOZc42UB4IqsrLgn/+Edu3g9NNt/DSXAH6TmXONlgeCGDp3hkcesaGpL7gg1anJEH6TmXONlgeCahxzDPzmN/DQQ/Dww6lOTQbwm8xcJvjDH2DatFSnIuE8ENTgN7+xgTSnTvUupQkR3GQ2e3aqU+Jc/dxzD9x7b8Z1K/RAUIPsbKsiatECfvQj2JWy8VEzxJgxPleBS1/r18Pnn9vctqtWpTo1CeWBoBbdusGDD9p87CefDBs3pjpFaaxVKzj6aO9G6tJT9DSGc+emLh1J4IEgDieeCHffDa+9BkOHwn//m+oUpbFx42DZMli9OtUpca5u5syB5s1tmZfSQRASzgNBnH7+c3j/fcjLg+99D845Bz7+ONWpSkNBN9K//hVKS1ObFufq4p134NBD4eCDPRCE2dChViK89FL4179gyBALCq++muqUpZEBA6xU8Mc/2hf69NMZ1/DmMtDu3fDBB3DUUZCfb88zaCBFDwR11LIl/OlPsHat9SRbtMgG2jz2WHj33VSnLg2I2FjfTz5p84OefDKcdBJs2ZLqlDlXvfnzYc8eOPJICwTbtmVUV0IPBPWUl2fDUaxaBbffbtVERx4Jxx9vF7le61EDEQsAH38Mt95qvYiOOirjemK4RmL3brjjjoaNfhs0FB95pM1tCxlVPeSBoIFatICLL4ZPP7USwpIllsf17Gn3IXzySapT2IhlZ8Mll8CsWdY1b8QIuP9+u6W7vNyWFSvg0Ue9/s3V3y232D/pL2udFbd6c+bYP/UBB8CBB1rVQAb1HBJNs/rZ/Px8ndeII3FpKcycab2MZs2y6u/DD4cf/hAOOsj+hnr1sjGNXJRPPrEqomXL7HXbtjYm+ObNFdtcdhnceKMFEOfi8fnn9k/XtCls3WpX8YccUvf9dOtmd5c++qi9HjXK2gjeeSex6a3Jhx9aOvbbr14fF5EPVDXmVMBeIkiw7GyYMMFqO9auhf/9X9ixw+5KP/FE6NcPOnaE886DN9+0i14H9O9vDS4ffWRjgE+aZHfx3Xef1c9ecIFd2Z1wgk9w4+J32WV2NTZ7NnToUPG6LtauhXXrrFoocNhh9ncZqw54/ny4+mqrI/7664alHywDueIKKzFfd13D9xeDlwj2kcJCq/FYtgxef92m8t2+3Uqahx1mPdKGDbOLlZ49rRrdVfHgg3D++VacOuggWw491IaJredVUqiVlkJRkWWQTZvW/fMbNthVTWMtob3yio0p/7vfWT3tX/5i48s/9xyMHx//fh5/3C5K5s61hmKwksGkSXbhMnRoxbazZ1uvuGDYYhEYPNiuAHv3hkGD4Kyz7A77qnbutCvIWbOsRNy7t03u9NvfWt3zz35mV5bt2tXr66ipROCBIEW2bYNnnrFqpAULKqrFAdq3t7+388+HH/zAakhcxPz5FhAWL7ZlwwbLiMaOtZb61autoebTT60n0tatluGNHGmlieOOq/hHysqySBwrE1S1fS1YYNNuDh4MXbsmPkKXlto5lJfbD928OXTqlLj9b95sd0Lu3m1XlL17Q3GxlbT+/GerOgH7ozvgAOveO3CgldB69LClWze7Kxwsnc8+a13nZs+29YcdZvvu0gVyc23JyrLvsKzMqv3mz7dMs7DQxmrZtcuONXGiTWd6wAHWeWD+fLsC7tfPlmbNYOlS+0137IDRo+3KPCvL7uy86y6rnvnOd6xqccwYO7+CArv637XLPtuiBZSU2O+YlWV1t3PnWpqOOMJuDMrJsXNct87GFNq+3ebdfvllmDHD9hv8raxYYd/TfffBlCm27pVXLA09etjd8wUF9t2//751hFi92jL7Qw+12a/697fPzZljv8WMGXbM3Fz7vYIxbfr1s/GNxoxp0J+CB4I0sHOn1YzMn29VgS+9BJ99Zn+Hl15q7QrZ2fZ/MXiwXTA47J/8oYdsEon16y0jPfBAy2TatbMrqtJSa2yOdQdgVpYVwfr2tX/yXbvsx1i2bO9ifW6uZVg5Oba0bWtX03l5lokXFdlSXGyRfts2ywy7dLEgkpdnadm924LUihWWSZaUVD5Oz56W4R11lL335Zc2tsnmzRbYtm2z/Q0ebKWiwkL7o5k/37bv2NGWTz+1eaOj6x87dbLz27bNhvv4wQ8sLRs3Wsa1YgWsXLl3mlq1svHZS0qsqqRHD/jpT2HTJus3vWDB3p+pek7Dh9t30aKFfddz5lj9KNj3V1s9qYh9n61a2W+7bp2VBI891vazbt3ev+0zz1S++n/mGTvnQMeO9v117gwXXWS/xyOPWABr1qwiMx492jL1QHm5/fannmqB4OWXrbdI//72z9u5897pV7WqgJ/9zP4Gpk2zYPbWWxaITz8dTjvNfpcmTewCoaDAblgKglQDeCBIQ6Wl8MQT1ja6cGHl95o0sf+pUaMsX2rRwpbBg+3irD6l/LRXVmYZQdeu1bfEFxTYP93u3fZPWVpqM6atXGlXbOXlFUMI9O1rV26HHGIZZ1AC2bjRXu/caRloURF89ZUdPy/PMof27S0AtW5tx1m/3o791VeWuTRvbplZ//52Vdmnj0X58nLL6N9+2+oPi4os3SKWYQX7bdnSruSjZ3tr08b+KFq2tIxt0ybLjI4/3pY2bSwovPuu/YFMnWrbx1Jaavtfu9aWgoKKYLRjB5xxBpxySuUqodJS+z6Kiy1glZdbukXsKqZ9+9jH+vxzq3rZssXqRw8+2NL66ae27NplV0Pf+pb94b/2mmW069ZZSeIHP7DvVNWC0Zw5Fuy6dbPvtWqGrGpX423b2j/LfvvBG29YJv7SS5bhTpliV189eth3vHSpBdxevSrv69hjK/dmGzMG/v1v+zuoSUGBVQ+9/jp07269maZMsb+XJEpZIBCRE4D/A7KA+1T1pirvXwr8FCgFNgHnqmqNcxmGJRAEgr/vbdssr9mxw/6f33jD/qerDu3furWVkocPt1qAPn3sf2j//VOTfldP5eVWJGzVqvp6+C1brESUl2eBy+sQG2bFCvsua8vIAy+/bNVko0bZVXxdqvTKyqyEOmjQPrtyS0kgEJEsYAXwXaAAmAtMVNUlUduMAd5T1R0iMhUYrao/qmm/YQsENSkvr6hu3bbNqjxffdUumlasqHwHfOfOFhyilz59PO9wLixqCgTJbO4fAaxU1VWRREwHJgDfBAJVjap0413gzCSmJ+M0aWI1AS1bWo1Ejx5WYgcrqa9da6XrxYut+nj+fLuICXq8ZWfbRUznzlbqHTkSvv1tqw2J1anBOZeZkhkIugJro14XAIfXsP0UIOaMJSJyHnAeQI8ePRKVvoyWnW1VQ717W0eZwO7dFZ0zVq2yat+NG62h+umnbZsWLaz6dORI++wxx3h3VucyWaPoACwiZwL5wNGx3lfVe4F7waqG9mHSMk7z5tYGeuihe7+3YYO1UwbLLbfATTdZm8Of/lTRhdo5l1mSWUO8Duge9bpbZF0lInIc8GvgJFX1Wc1TaP/9rWrp1lutQXrLFps2YOlSKyGcdZbPJ+NcJkpmIJgL9BeR3iLSDDgDmBG9gYgMB+7BgoBPAtnI5ORYL8NPPrEuz//+t3XPv/BC603onMsMSQsEqloKXADMApYCj6vqYhH5nYicFNnsj0Br4N8iskBEZlSzO5dCublWRbRypd2Aeddd1ltx5sxUp8w5lwh+Q5mrs08+sXt55s+3O9/PPTfVKXLO1cZHH3UJ1b+/3atw3HF2Q+QNN/hsk86lMw8Erl5at7abKs86y0bcvfHGVKfIOVdfjaL7qEtPTZvaQKCq8Otf27ApZ52V6lQ55+rKA4FrkCZNbB6Z9eutraBLFxuLyzmXPrxqyDVYs2bw5JM2uN0Pf+jzNDuXbjwQuITIzbXupGVlSZtNzzmXJB4IXMJ07w7/7//B9OleKnAunXggcAn1y1/aeEZ/+EOqU+Kci5cHApdQnTvDeefBww/7uETOpQsPBC7hLr/cZou86abat3XOpZ4HApdwXbtaV9L777fpWZ1zjZsHApcUV15pN5r98Y+pTolzrjYeCFxS9OwJZ54Jf/ubzYDmnGu8PBC4pLnySti1C26/PdUpcc7VxAOBS5qBA+HUU+HOO2Hz5lSnxjlXHQ8ELqmuusqmvLzzzlSnxDlXHQ8ELqmGD4dx46x6aPv2VKfGOReLBwKXdL/6FRQWws03pzolzrlYPBC4pBs5En7yE/j972GGz0rtXKPjgcDtE3ffDfn51qV06dJUp8Y5F80DgdsncnJszoKcHJgwAb7+OtUpcs4FkhoIROQEEVkuIitF5MoY7zcXkX9F3n9PRHolMz0utbp3h//8xwaj69fP7jP4/PNUp8o5l7SpKkUkC7gT+C5QAMwVkRmquiRqsynA16raT0TOAG4GfpSsNLnU+/a3YfZsG3rij3+EW26BESNgyBBbevaEdu1sadsW2rSxpWnTVKfcuYYpKbGec02b2lDt2VVy3/Jy2L0bdu607bZtswVsEMesLBvdd//9E582UdXE7xUQkSOB61T1e5HXVwGo6o1R28yKbDNHRLKBDUAnrSFR+fn5Om/evKSk2e1ba9bAPffAO+/AokXw1VfVb9ukCYjYkp0NLVva0qyZjWmkarOjlZdXPFZH1d4vL7f9ZmfbP1nwXvB+aaktqhXHbtKkYoGK/ZSVVRxXtfJ2wSJSse/gsazMnoO9D5aW4DPBfsvKbH3Tppbe6O8j+FzwHezZY0tZ2d7fYZMmFftp1qzyMaqmIzgGVE5H1d8jWILvorrvNfhNor+PWKJ/w+g0NWkCLVpYJhr87tHnHXwu+neOXkpKKpZmzezvJyenIkMO9hP87tFpjf5eS0sr0paVZWlq0cLSGGTge/bY/lu1sv1v3rx39+ngOwj+lkpKYn8f0aZNq/+oviLygarmx3ovmZPXdwXWRr0uAA6vbhtVLRWRYiAPKIzeSETOA84D6NGjR7LS6/axnj0rJrBRhfXrbdm82ZYtW2DrVlt27678D71zpy27d1f8M4lUXDlVl9FEZ9JQOROHikwtK6siIwsy8OggUVZWcdzgeMFjcJzojD46g4w+RpCW6EytaoYanFOQSZWUVGREVS+ZsrMtkwsCRvR5Rwe4IEMMAkyQluh0RAet4LuIPr+qS5DWICgE6YXKv0eQ4UanLfq3iv4dg+8e7DO7d9uwJXv2VA6C1f3u0YEqCH7Z2fb5HTtsib5oCH737OzK5xF8r82aVT5OkKadO23b1q1tadasYv8lJTaVa/v29l5JiX0m+m9a1QJcTo4FlWA/rVrZcUpL7VgDBuz9N50IyQwECaOq9wL3gpUIUpwclwQiNnx1166pTolz4ZPMxuJ1QPeo190i62JuE6kaygWKkpgm55xzVSQzEMwF+otIbxFpBpwBVL2daAZwduT5qcCrNbUPOOecS7ykVQ1F6vwvAGYBWcD9qrpYRH4HzFPVGcDfgYdFZCXwFRYsnHPO7UNJbSNQ1eeB56usuybq+S7gtGSmwTnnXM38zmLnnAs5DwTOORdyHgiccy7kPBA451zIJW2IiWQRkU3Amnp+vCNV7loOiTCedxjPGcJ53mE8Z6j7efdU1U6x3ki7QNAQIjKvurE2MlkYzzuM5wzhPO8wnjMk9ry9asg550LOA4FzzoVc2ALBvalOQIqE8bzDeM4QzvMO4zlDAs87VG0Ezjnn9ha2EoFzzrkqPBA451zIhSYQiMgJIrJcRFaKyJWpTk8yiEh3EXlNRJaIyGIRuTiyvoOIvCQin0Qe26c6rckgIlkiMl9Enou87i0i70V+839FhkPPGCLSTkSeEJFlIrJURI4Mw28tIpdE/r4/FpHHRKRFJv7WInK/iGwUkY+j1sX8fcXcETn/hSJySF2OFYpAICJZwJ3AWGAQMFFEBqU2VUlRCvxSVQcBRwD/EznPK4FXVLU/8ErkdSa6GFga9fpm4DZV7Qd8DUxJSaqS5/+AF1X1QGAYdu4Z/VuLSFfgIiBfVQdjQ9yfQWb+1g8CJ1RZV93vOxboH1nOA+6qy4FCEQiAEcBKVV2lqnuA6cCEFKcp4VT1C1X9MPJ8K5YxdMXO9aHIZg8BP0hNCpNHRLoB44H7Iq8FOAZ4IrJJRp23iOQC38Hm9EBV96jqZkLwW2PD5+dEZjVsCXxBBv7WqvomNk9LtOp+3wnAP9S8C7QTkQPiPVZYAkFXYG3U64LIuowlIr2A4cB7QGdV/SLy1gagc4qSlUy3A1cAwVTkecBmVY1Mn55xv3lvYBPwQKQ67D4RaUWG/9aqug64BfgcCwDFwAdk9m8drbrft0F5XFgCQaiISGvgP8AvVHVL9HuRqUAzqs+wiJwIbFTVD1Kdln0oGzgEuEtVhwPbqVINlKG/dXvs6rc30AVoxd7VJ6GQyN83LIFgHdA96nW3yLqMIyJNsSDwiKo+GVn9ZVBMjDxuTFX6kmQkcJKIrMaq/Y7B6s/bRaoPIPN+8wKgQFXfi7x+AgsMmf5bHwd8pqqbVLUEeBL7/TP5t45W3e/boDwuLIFgLtA/0rOgGda4NCPFaUq4SL3434Glqnpr1FszgLMjz88GntnXaUsmVb1KVbupai/st31VVScBrwGnRjbLqPNW1Q3AWhEZGFl1LLCEDP+tsSqhI0SkZeTvPTjvjP2tq6ju950B/CTSe+gIoDiqCql2qhqKBRgHrAA+BX6d6vQk6Ry/jRUVFwILIss4rL78FeAT4GWgQ6rTmsTvYDTwXOR5H+B9YCXwb6B5qtOX4HM9GJgX+b2fBtqH4bcGfgssAz4GHgaaZ+JvDTyGtYOUYCXAKdX9voBgPSM/BRZhvariPpYPMeGccyEXlqoh55xz1fBA4JxzIeeBwDnnQs4DgXPOhZwHAuecCzkPBM5FiEiZiCyIWhI2YJuI9IoeRdK5xiS79k2cC42dqnpwqhPh3L7mJQLnaiEiq0Xkf0VkkYi8LyL9Iut7icirkfHfXxGRHpH1nUXkKRH5KLIcFdlVloj8LTKW/n9FJCey/UWROSQWisj0FJ2mCzEPBM5VyKlSNfSjqPeKVXUI8BdspFOAPwMPqepQ4BHgjsj6O4A3VHUYNv7P4sj6/sCdqnoQsBk4JbL+SmB4ZD/nJ+vknKuO31nsXISIbFPV1jHWrwaOUdVVkUH9NqhqnogUAgeoaklk/Req2lFENgHdVHV31D56AS+pTSiCiEwDmqrq9SLyIrANGybiaVXdluRTda4SLxE4Fx+t5nld7I56XkZFG914bJyYQ4C5UaNoOrdPeCBwLj4/inqcE3n+DjbaKcAkYHbk+SvAVPhmHuXc6nYqIk2A7qr6GjANyAX2KpU4l0x+5eFchRwRWRD1+kVVDbqQtheRhdhV/cTIuguxGcIux2YLOyey/mLgXhGZgl35T8VGkYwlC/hnJFgIcIfalJPO7TPeRuBcLSJtBPmqWpjqtDiXDF415JxzIeclAuecCzkvETjnXMh5IHDOuZDzQOCccyHngcA550LOA4FzzoXc/wfyNqInsFVF8gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9fX4/9fJRkjICgRCAoQd2TcBccOFuiJoXWtbbKlba1tr609bN7BqK8Wl2uq37vjRCm5UEaVFQKmClFVEQNYEEgIJkIXs2/n9MQsTsk2WYcjMeT4eeTD3zl3Onatnzrzv+76vqCrGGGOCR4i/AzDGGHNyWeI3xpggY4nfGGOCjCV+Y4wJMpb4jTEmyFjiN8aYIGOJ37RbIvKJiMzwdxzGtDeW+M1JJSJFHn81IlLqMX1jc7alqpeo6jxfxeotEblJRFRErvN3LMZ4Q+wGLuMvIpIO/ExVP63nvTBVrTr5UTWfiKwARgBfqeplJ3G/7eYzMqcWq/jNKUFEJotIpojcIyIHgVdFJEFEPhKRXBHJc75O9VjnMxH5mfP1TSLyhYjMdS67V0QuaWBf94jIuyfM+6uIPOOxrT0icsy5nQZ/iYhIb+Bc4BbgIhHp7vFeqIj8QUR2O7e1XkR6Ot8bKiJLReSoiBwSkT84578mIo+c+Ll4TKc7498MFItImIjc67GPrSJy5Qkx3iwi2zzeHyMid4vIeycs94yI/LWhYzWBwxK/OZV0BxKB3jgSaQjwqnO6F1AK/K2R9ScA3wFdgDnAyyIi9Sw3H7hURGLAkaCBa4F/ikg08AxwiarGAJOATY3s88fAOlV9D9gGeH5J3AXcAFwKxAI/BUqc+/0UWAL0APoDyxrZx4luAC4D4p0V/27gbCAOmA28ISLJzmO7BpjljDMWuAI4ArwBXCwi8c7lwoDrgdebEYdppyzxm1NJDfCQqparaqmqHlHV91S1RFWPAY/iqK4bkqGqL6pqNTAPSAa6nbiQqmYAGwBXZXw+UKKqX3nEMUxEOqpqtqp+28g+fwz80/n6n85pl58B96vqd+rwtaoeAS4HDqrqE6papqrHVHVNI/s40TOqul9VS53H846qHlDVGlVdAOwExnvEMEdV1zpj2KWqGaqaDawErnEudzFwWFXXNyMO005Z4jenklxVLXNNiEiUiPxDRDJEpBBHoop3Vuj1Oeh6oaolzpedGlj2nzgqZ4AfOKdR1WLgOuA2IFtEFovI4Po2ICJnAn1w/IJwbXO4iIxyTvfEUY2fqKH53tp/Qhw/FpFNIpIvIvnAMBy/epra1zzgh87XPwT+rxUxmXbEEr85lZzY0+C3wCBggqrGAuc459fXfNNc7wCTndcMruR41Y6q/ltVp+D4xbAdeLGBbcxwxrLJeV1ijcd8cCTofvWstx/o28A2i4Eoj+nu9Szj/pyc1xheBO4AOqtqPLCF459RQzEA/AsYISLDcPwKebOB5UyAscRvTmUxONr180UkEXiorTasqrnAZziuIexV1W0AItJNRKY52/rLgSIcTT+1iEgkjusCtwCjPP5+CfzA2Wb+EvBHERkgDiNEpDPwEZAsIneKSAcRiRGRCc5Nb8Jx/SHReaH4ziYOJRrHF0GuM66f4Kj4XV4CficiY50x9Hd+WeD8dfUuji+9/6nqPm8/P9O+WeI3p7KngY7AYeArHBdD29I/gQvxqPZx/D9xF3AAOIrjmsLt9aw7HceX0uuqetD1B7wChOFoM38SeBv4D1AIvAx0dF6vmAJMxdE8tRM4z7nd/wO+BtKd6y1o7ABUdSvwBLAaOAQMB770eP8dHNdG/gkcw1HlJ3psYp5zHWvmCSLWj9+YICYivXA0Z3VX1UJ/x2NODqv4jQlSIuL6dTPfkn5wCfN3AMaYk895DeMQkIGjWcoEEWvqMcaYIGNNPcYYE2TaRVNPly5dNC0tzd9hGGNMu7J+/frDqtr1xPntIvGnpaWxbt06f4dhjDHtiohk1DffmnqMMSbIWOI3xpggY4nfGGOCTLto469PZWUlmZmZlJWVNb2w8ZnIyEhSU1MJDw/3dyjGGC+128SfmZlJTEwMaWlp1P+sDeNrqsqRI0fIzMykT58+/g7HGOOldtvUU1ZWRufOnS3p+5GI0LlzZ/vVZUw7024TP2BJ/xRg58CY9qddJ35jgs2WnC08s+YZiiuK/R2Kaccs8bfQeeedx7///e9a855++mluv72+odsdJk+ebDeitZKqklOcw/oD61m4bSEr9q6okwTLq8rJKc6hsLyQ8qpy2tt4VKpKVmEWX2V+xYbsDXyb8y0f7/yYi964iOHPD+fXS37Nma+cSUZ+7XtzjpYe9VPEpr3x6cVdEfk1cDOOx8C9qKpPO5+ktABIw/GwiWtVNc+XcfjCDTfcwPz587nooovc8+bPn8+cOXP8GFXjqqqqCAtrX9fzVZX3tr3HZ+mfsengJjYf2syximO1lgmVUEYnjyY+Mp6dR3ayr2Af6vEUxxAJIbZDLHEd4ojtEEt0RDTR4dHER8bTL6Ef/RP70z+xP30T+pIam0poSCgHjh1gTeYavj70NaWVpVTVVFFZU8mR0iMcKjrEkdIjxEfG0yuuF71iezExdSLn9zmfjuEda8VWWlnKe9veY97X8ygsL2Rs8ljGJo9lRLcR9EnoQ+eOnSmvLufLfV/yn93/YVXmKrbkbCG/LL/OZ9G9U3cePf9RBiQO4OZFN3P6i6fz6rRXySjI4LVNr7H2wFruOP0Onr74aUJDGnossTE+TPzO53jeDIwHKoAlIvIRjkfVLVPVP4vIvcC9wD2+isNXrr76au6//34qKiqIiIggPT2dAwcOcPbZZ3P77bezdu1aSktLufrqq5k9e3aj23r44YdZtGgRpaWlTJo0iX/84x+ICLt27eK2224jNzeX0NBQ3nnnHfr168fjjz/OG2+8QUhICJdccgl//vOfmTx5MnPnzmXcuHEcPnyYcePGkZ6ezmuvvcb7779PUVER1dXVLF68mGnTppGXl0dlZSWPPPII06ZNA+D1119n7ty5iAgjRozgueeeY8SIEezYsYPw8HAKCwsZOXKke7ot5JflExMRU2+i+u7wd9z60a18nvE5MRExjOo+ihkjZzCw80B6xvWkZ2xPDhUf4st9X7IqcxWF5YWc1essBiQOoEtUF8qryymvKqe4spjC8kIKygs4Vn6M4spiiiqKyMrNYtGORVRUV7j3GRYSRnxkPIdLDgMgCB3COhAWEkZYSBidO3YmKTqJlJgU8sryWLF3BVnHsqjRGqLCo5jSdwopMSnufS7fu5yC8gL6xPehd3xv3vzmTZ5f97x7f50iOlFdU01pVSnhIeGcnnI61w+9nmFJw0iLT6Oqpory6nIiwyK5qN9FdAjrAMCIbiO4Yv4VXP7W5e7p64Zex9/W/o0DRQd448o3CA0JZd6meby08SXOSD2D2ZNnExcZB8AX+77gziV3klmYSWLHRBI7JhIdEe0+zg6hHdxfkMOShvHT0T8lIjSi0XNZUllCTnEONVpD34SGHilsTgW+LP9OA9aoagmAiHwOXAVMAyY7l5mH47mnrUr8dy65k00HN7VmE3WM6j6Kpy9+usH3ExMTGT9+PJ988gnTpk1j/vz5XHvttYgIjz76KImJiVRXV3PBBRewefNmRowY0eC27rjjDh588EEAfvSjH/HRRx8xdepUbrzxRu69916uvPJKysrKqKmp4ZNPPuGDDz5gzZo1REVFcfRo0z/vN2zYwObNm0lMTKSqqoqFCxcSGxvL4cOHmThxIldccQVbt27lkUceYdWqVXTp0oWjR48SExPD5MmTWbx4MdOnT2f+/PlcddVVrU76FdUVLNy2kBc2vMDyvcsJDwmnb0Jf+if2p2t0V2IjYqmsqeTljS8TFR7Fi1Nf5Kejf0qI1N8yeemAS1scS3VNNfsL97Pr6C725u1lb/5ecopzGJ40nAmpExjVfRSRYZGNbqOsqozP0j9j0XeL+HjXx3yx7wt30rx84OXMHD2Tc9POJURCqNEadh3dxbbcbezN38vevL2ESAgX9r2Qc9POpVNEJ6/iHtRlEGt+toYFWxa44wSYmDqRu/59F2fnn01OcQ77C/czqPMgnlnzDPO3zOexCx5j1f5VvLzxZXrF9WLqwKnkleVxpPQIBWUFVGs1VTVVlFWVUVzh+ILMK8vjidVPMOfCOUwbPI1vc77l84zP2Zi9kX2F+9hXsI+swiyKK483uf1oxI946qKn6BzVuU7sq/ev5rEvHmNw58HcMPwGRncfjYhwqOgQ2w5vY1T3UcRHxtdaZ9meZcR0iGF8ynivPh/TOF8m/i3Ao86HS5cClwLrgG6qmu1c5iDQrb6VReQWHL8O6NWrlw/DbDlXc48r8b/88ssAvP3227zwwgtUVVWRnZ3N1q1bG038K1asYM6cOZSUlHD06FGGDh3K5MmTycrK4sorrwQcN0oBfPrpp/zkJz8hKioKcHwBNWXKlCnu5VSVP/zhD6xcuZKQkBCysrI4dOgQy5cv55prrqFLly61tvuzn/2MOXPmMH36dF599VVefPHFFn1W+WX5LN29lI93fcziHYvJLcmld1xv7j/7fiqqK9h5dCe783az+dBmCsoLKKks4eohV/PURU/RvVP3Fu3TG6EhoaTFp5EWn9bibUSGRXJx/4u5uH/TzzMJkRAGdh7IwM4DW7w/l/jIeG4dd2uteXdOvJMeMT348cIfM7bHWF6Y+gIX9buIDdkb+PnHP2fmhzMJCwnjnjPv4YFzHiA6IrrRfagqS3Yt4XdLf8dVb19FVHgUJZUlAHSL7kZafBojuo3g0v6X0q1TN5Kik9hxZAdPrH6CJbuW8NRFT3Fx/4vpHNWZksoSHlj+AE995fhCWLJrCXNXz6VPfB9Kq0o5WHQQgJ6xPZl/9Xwm9ZxEVU0V9y27jzmrHE2oPxv9Mx6f8jiJHRPJPpbNwu0LiYmI4cYRNzZYGLSlBVsWkBqbypm9zvT5vnzJZ4lfVbeJyOM4HhhdDGwCqk9YRkWk3itvqvoC8ALAuHHjGr0611hl7kvTpk3jN7/5DRs2bKCkpISxY8eyd+9e5s6dy9q1a0lISOCmm25qtJ97WVkZP//5z1m3bh09e/Zk1qxZLeoXHxYWRk1NjXubnqKjj//P/eabb5Kbm8v69esJDw8nLS2t0f2deeaZpKen89lnn1FdXc2wYcOaHduazDWc+9q5lFeXkxCZwEX9L2LGyBlM6TulwbZoVbWuoi107dBrmTpwKpFhke7PcGyPsayeuZr3t73PaV1OY2jSUK+2JSJcMuASpvSbwisbX2FD9gbOSD2Dc9PObfTL8gfDf8DMD2fyw4U/BCAhMoHw0HByinO4fdzt/PnCP1NVU8X7295n0Y5FJHZMZFS3UfSI6cG9y+7lnFfP4aFzH+LzjM9ZtncZt469lZiIGJ766in+9d2/GNR5EKv2r3Jfy3ll0yu8csUr9EmoeyOhqlKt1YSFHE93NVrDh999yOIdi7nnrHvon9i/1ntrMtcwIXVCrS+TnOIcrn/vegAmp03mgXMe4Ly089rnf6eqelL+gMeAnwPfAcnOecnAd02tO3bsWD3R1q1b68zzh2uvvVZHjhypDz74oKqqbtq0SUeMGKHV1dV68OBBTUpK0ldffVVVVc8991xdu3ZtrfXz8vI0KSlJS0pK9NixYzp06FB96KGHVFV1woQJunDhQlVVLSsr0+LiYv3kk0/0jDPO0OLiYlVVPXLkiKqqzpw5U5977jlVVX3qqae0d+/eqqr66quv6i9+8Qv3/p5++mm94447VFV1+fLlCujevXt1y5YtOmDAAD18+HCt7aqqzp07V5OTk93bP1FT52LmBzM15rEY/SLjC62srmx0WRM4Kqsr9eMdH+sTq57Q2z+6Xa9acJUu37O8yfXyS/P1mrevUWahHf7YQV/Z8Ir7va8Pfq3nzztfRz4/Umd/Nlu3HNqiL61/SWMei9FOj3XSeZvm1dnes2ue1bCHw/TcV8/Vx794XOdtmqfDnxuuzEKZhXZ6rJO+uflNVVXdmL1RJ7w4QZmFe57Lxzs+VmahN394sybPTVZmofcuvdfrz6OiqkKLyou8Xr4tAOu0vnxc38y2+gOSnP/2ArYD8cBfgHud8+8F5jS1nVM58S9cuFAB3bZtm3vejBkzdMCAAXr++efrlVde2WjiV1W97777tG/fvjpp0iS96aab3Il/x44det555+nw4cN1zJgxunv3blVV/dOf/qSnnXaajhw5Un//+9+rquq2bdt0+PDhOmrUKL3vvvsaTPy5ubk6ceJEHTZsmN500006ePBg3bt3r6qqvvbaazp06FAdMWKEzpgxw71Odna2RkZGal5eXr2fQWPnoqKqQhMfT9Qb37ux0c/RGE81NTX6zrfv6NcHv/Zq+Yz8DD3n1XM0dHaobs/d7p5fVF6kXed01cF/G6wjnh/hTvaDnh2kr296Xfcc3aNnvXKWMgs9+5WzNWR2iCb9JUmjHo3S2xbdVmsff/z8j8ostKCsQEsrS/XHC3+sIbND9H+Z/2s0tuKKYn1q9VPa44keGv5wuF7+z8v1ja/f0MKywuZ/MM3kr8T/X2Ar8DVwgXNeZ2AZsBP4FEhsajuncuIPBu+8847+8Ic/bPD9xs7F0t1LlVno+1vf90VoxrgdPHZQOz3WSa9acJV73l++/IsyC/1y35eqqppZkKkr01dqVXWVe5nK6kp9YPkDGv5wuN626DY9WnJUp7w+RUf/v9G1tj/trWk66NlB7un80nzt8UQPHfH8CK2oqqi1bE1NjX5z6Bt9YPkD2nVOV2UWOvm1yfqbJb/Rnk/2VGahXeZ00WV7ltVab1P2Jn169dO14msNvyT+tvqzxO8/d9xxh/br10+/++67Bpdp7FzcuuhWjX40WksqSnwRnjG1PPzZw8os9IuML7SovEiT/pKkU16f4tW6nsn7/mX3a+js0FpNM6lPpuoP3vtBrXUWbluozEIfW/mYqqoeLj6sj3z+iA7+22BlFiqzRC9+42L9b8Z/3etU11TryvSVOuTvQzRkdoj+5cu/aGFZod615C4NnR2qzELvWnJXaz4Gt4YSf/u6m8ecdM8++2yL162uqWbh9oVcNvCyOjc2GeMLd51xF8+ve567l97NVaddRU5xDg+d+5BX64aHHu+mPDF1ItVazfrs9ZzT+xwOFR0iszCTsclja60zffB0vn/a95n9+WzS89N545s3KKksYXLaZH41/ldcedqVdXqlhUgIZ/c+m69mfsVPP/wpdy+9m1mfzaK4sphbxtyCiPDkV08ypOsQZo6Z2foPpR7tOvGrWs8Pf3MUFfX7Yt8X5BTn8P3Tvn8SIzLBLDoimofPe5ibF93MhuwNXNj3whZ1vZyQOgGArzK/4pze57A+ez0A43qMq7Pss5c8y6d7PuXVTa/yg+E/4HeTfsewpKZ7v8V0iOHtq9/mydVPsnjnYh49/1HO6HkGVTVV7Mnbw+2Lb2dg54Gc3fvsZsfflHY7Vk9kZCRHjhxpNPEY31J1jMfvusfgRO9ufZfIsMhW3WBlTHPdNOomhnQdQnl1udfV/om6RHWhX0I/1mStAWD9gfUIwujuo+ssmxyTzMZbN5J+ZzqvTX/Nq6TvIiL8dtJvWT5jOWf0PANw3D2+4OoF9Enow1VvX0V6fnqLjqEx7bbiT01NJTMzk9zcXH+HEtRcT+A6UY3W8N6297ik/yVe341qTFsICwnj9emvszJjJWf1OqvF25mYOpHle5ejqqzLXsegLoOI6RBT77L13T/QGgkdE1h0wyLuXHJnk3eOt0S7Tfzh4eH21KdT2Or9q8kuyrZmHuMXY3uMZWyPsU0v2IiJqRN585s3ySzMZP2B9UxOm9w2wXlpYOeBfHzjxz7Zdrtt6jGntufWPUeniE5MHTTV36EY0yITUhzt/P/a/i+yjmXVubDbnlniN21ub95eFmxZwK1jbyW2Q6y/wzGmRUZ2H0mH0A48t+45oP4Lu+2VJX7T5p5Y/QQhEsJvJv7G36EY02IRoRGM7TGW7Ye3Oy7sJte9sNteWeI3bSqnOIeXN77Mj0b8iJTYFH+HY0yrTEyZCMDgLoMDqpOCJX7Tpp5d8yzlVeXcfebd/g7FmFZz9edv7YXiU40lftNmjpUf429r/8b0wdMZ3GWwv8MxptUm9ZxEqIQyKXWSv0NpU+22O6c59cz7eh75Zfncc2a7e5KmMfVKjU1l8+2bGZA4wN+htClL/KbNfPDdB5zW5TT3z2NjAsGQrkP8HUKbs6Ye0yaKKor4PP1zLhtwmb9DMcY0wRK/aRPL9iyjsqbSxuUxph2wxG/axOKdi4mJiGn3D6E2JhhY4jetpqp8vPNjvtfve0SERvg7HGNMEyzxm1b7Jucbso5lWTOPMe2EJX7Taot3LAbgkv6X+DkSY4w3LPGbVvt418eMSR5Dckyyv0MxxnjBEr9plbzSPFbtX8Wl/a2Zx5j2whK/aZV/7/43NVpj7fvGtCOW+E2r/Dfjv8RExDA+Zby/QzHGeMkSv2mVXXm7GNh5IKEhof4OxRjjJUv8plV2H91Nv8R+/g7DGNMMlvhNi1XVVJFRkEH/hP7+DsUY0wyW+E2L7SvYR1VNlVX8xrQzlvhNi+06uguAfgmW+I1pTyzxmxbbfXQ3gFX8xrQzlvhNi+3O201kWCQ9Ynr4OxRjTDNY4jcttjtvN30T+hIi9p+RMe2J/R9rWmzX0V3Wvm9MO2SJ37SIqrInbw/9E60rpzHtjSV+0yIHiw5SUlliFb8x7ZAlftMiu/OsR48x7ZVPE7+I/EZEvhWRLSLylohEikgfEVkjIrtEZIGI2LP62iHrw29M++WzxC8iKcCvgHGqOgwIBa4HHgeeUtX+QB4w01cxGN/ZfXQ3oRJK7/je/g7FGNNMvm7qCQM6ikgYEAVkA+cD7zrfnwdM93EMAWlN5hpO+/tpHC096pf9787bTa+4XvZwdWPaIZ8lflXNAuYC+3Ak/AJgPZCvqlXOxTKBFF/F0J7ct+w+rnjrCq+X/yrzK7Yf3s4X+77wYVQN251no3Ia0175sqknAZgG9AF6ANHAxc1Y/xYRWSci63Jzc30U5alj4faFLNqxiPT8dK+WzynOARyVvz9YH35j2i9fNvVcCOxV1VxVrQTeB84E4p1NPwCpQFZ9K6vqC6o6TlXHde3a1Ydh+t+x8mNsP7wdgHe3vtvE0g7uxJ918hN/flk+R0uPWh9+Y9opXyb+fcBEEYkSEQEuALYCK4CrncvMAD7wYQztwvrs9ShKh9AO3if+Ekfi/1/W/6iuqW5y+aqaKq5acBVfZX7VqljBY3A2q/iNaZd82ca/BsdF3A3AN859vQDcA9wlIruAzsDLvoqhvVh3YB0Ad4y/gzVZa9hXsK/JdVwV/7GK478WGpORn8HC7QuZu2pu64LF+vAb0975tFePqj6kqoNVdZiq/khVy1V1j6qOV9X+qnqNqpb7MgZ/eWnDS2zN3erVsmsPrCUtPo3bxt0GwHtb32tynUNFhxjXYxzgXXNP1jFHi9rinYspLC/0Kq5HVj7CQyseokZras23PvzGtG92564PrNq/ipsX3cxfVv3Fq+XXHVjHuB7j6J/Yn1HdR/HO1neaXCenOIezep5FfGS8Vxd4swodib+sqowPtjfduvblvi95YMUDPLzyYW5ddKs7+e/N28tbW94iJSaF6IjoJrdjjDn1WOL3gYc+ewg43oTTmCMlR9iTt4fTe5wOwNWnXc3qzNVkFmY2uE5xRTHFlcV079Sd8Snjm1Xxd4vuxltb3mp02eqaau745A5SY1O5e9LdvLTxJX7ywU9459t3GP2P0ewv2M8/Lv9Hk/s0xpyaLPG3sZUZK/l0z6ekxqayNXcrxRXFjS7v+nJwNdtcM/QaABZsWcDCbQuZ8n9TOOuVs1BV9zq5JY7urUnRSUxImcA3Od9QVFHU6H6yCrOIDo/mplE3sXTPUg6XHG5w2X+s/webDm7iye89yZwpc5g9eTavf/061757LQM7D2TjrRu5bOBlTX8YxphTkiX+NqSqPLjiQbp36s7cKXOp0Ro2HdzU6DquxD82eSwAAzsPZHjScH639Hdc9fZVrMxYyZf7vySvLM+9juvCrivx12gN6w+sb3Q/B4oOkBKbwg3DbqCqpqrB3kO5xbnct/w+zu9zPlcPcXS+evDcB3nu0ud46NyH+OKnX9AnoY93H4gx5pRkib8NrUhfwecZn/P7s37P2b3PBhwXbhuz9sBaBnUeRFxknHve/efcz/TB0/nXdf/i9emvA7C/YL/7fc/EPz5lPND0Bd6swix6xPRgRLcRDO4ymPlb5te73H3L76OooohnL3kWRy9ch9tPv51Zk2fZEA3GBABL/G2korqC+5ffT4+YHtwy9hZ6xPSgR0yPJtv51x5Y627mcbl26LUsvG4h0wZPc1fXnl08PRN/1+iu9Evo13TiP5ZFSkwKIsINw25gZcZK9wVfl9ziXF7b9Bq3jr2VIV2HeH3sxpj2xRJ/G8g+ls0Fr1/A6szVPHLeI0SGRQKOdvvGEv+BYwc4cOyA+8JufXrF9QJgf2Hdir9rtOOO5gmpExq9MUtVOXDsACkxjmGRbhh2A4ryxuY3ai037+t5VNZU8vPTf97Y4Rpj2jlL/B6yCrP403//VOtCalNW7V/F2BfGsiF7A299/y1+Mvon7vdO73E63x35joKygnrXPfHCbn2SopMIDwmvVfEfKjpEp4hORIVHATAhZQIHjh3gkZWPsDF7Y51+94dLDlNRXUFKrCPxD+g8gAv7XsiTXz3pvvisqry44UXO7HmmVfvGBDhL/B7e3fouf1j+BzIKMrxa/sCxA5w/73yiwqNYPXM11w+7vtb7roS+IXtDveuvO7COUAlldPLoBvcRIiH0jOtZu6mnJIdu0d3c01cMuoJR3UfxwIoHGPPCGFKeTGHnkZ214gToEdPDPW/25NnkFOfw/LrnAUdvpB1HdnDzmJu9OnZjTPtlid9Dflk+4KiovfHhdx9SXl3OB9d/wIhuI+q870r8DTX3rM9ez5CuQ9yVe0N6xvas09STFJ3knk6LT2PjrYszl5sAABjMSURBVBs5cNcBnrv0OQ4WHeSz9M/c77v68LuaegAm9ZzE9/p9j8e/fJyiiiJe3PAicR3i3N1JjTGByxK/B3fiL/Yu8S/asYi+CX0bbBrpEtWFtPg01mXXn/gzCzO96hrZK65XnYu7nonfJTkmmVvH3UpUeBTf5n7rnu+6iOtq6nGZPXk2h0sO88fP/8i7W9/lhyN+2OSXkDGm/bPE7yG/3PuKv7iimGV7ljF14NRa3R5PNK7HONZm1d+l81DRIZKi6ibwE/WK60VWYZZ7FM6GEj84moaGdB1Sa5wgV8Wf3Cm51rITUydycf+LmbNqDuXV5dbMY0yQsMTvwXUR1puK/9M9n1JeXc4Vgxp/atbpPU5nb/5ejpQcqTW/uqaa3JJcunXq1sCax/WM7Um1VpNdlE2N1pBbnNtg4gcY2nVonYo/KTqJ8NDwOsvOnjzbHefI7iObjMUY0/5Z4vfQnDb+RTsWEdchjrN7nd3ocq52/vXZte+sPVJ6hBqtqXWRtiGuLp37CvaRV5pHtVY3mviHdB3CgWMH3MdzoOhArfZ9T+NTxvPXi//K0xc/3WQcxpjAYInfg7dt/DVaw0c7PuLi/hfXW0V7GpM8BqBOc4+rL75XFX9cT8Bx967nzVsNGdp1KIC7uSerMKtO+76nX034FZN6TmoyDmNMYLDE76Gg3LumnrVZazlUfIipA6c2uc34yHjS4tPYerj22PyuXxWNJXAXz4rfFVujiT/Jkfi/zXE097ju2jXGGLDEX4u3TT2LdiwiVEK5ZMAlXm03NTa1zvAIrgTuTVNPbIdYYjvEsr/Qu4q/V1wvd8+e8qpyDpccrtWH3xgT3CzxO6mq1009i3Ys4qxeZ5HYMdGrbafEpLh71ri4vly8aeqB4106vUn8nj17souy3TEYYwxY4ncrqiiiRmuI6xBHflk+5VX1PxEyszCTzYc2e9XM4+Kq+D2HgsgpziEsJIyEyASvttErrpe74g+REDp37Nzo8q6ePQ314TfGBC9L/E6u9v1BXQYBxy++nig9Px2g3jt1G5ISk0JpVan7FwU4flUkRSc1eg+Ap56xPd0Vf5eoLoSGhDa6vKtnj6tbp1X8xhgXS/xOrqQ8sPNAoOHEn1fqeCBKQkfvKnU4Xm17NvccKj7kVfu+S6+4XhwuOUx6frpXF4RdPXuW7llaKwZjjLHE7+RO/ImOxN9QO7/rSVjeNtHA8Wrb8wJvTnGO1+374Kj4wTHgm1eJ39mzZ9meZXQI7dCseI0xgc0Sv5Mr8buaehrq2dNmFX9R8yt+cDxv19suoFHhUeSV5ZESm+J1k5IxJvBZ4ndyDdcwqLMz8TdR8cd1iKv3/fq4ulK6Kn5Vdbfxe8uV+AGvxvdx9ewBa983xtRmid/JVfEnxyTTKaJTgxV/flk+sR1im7y46ikiNIKuUV3dFX9BeQEV1RXNqvhTYlMQHFW7t18YrnZ+a983xnhqMvGLyFQRCfgvCFfij+sQR7fobo1W/C1pL0+JPd6XvznDNbhEhEbQvVN3wPvE76r4e3Sym7eMMcd5k9CvA3aKyBwRGezrgPwlvyyfjmEd6RDWgW6dGkn8pXnNat93SYlJcTf1uG/eakbFD8fH7LGK3xjTGk0mflX9ITAa2A28JiKrReQWEYnxeXQnUUF5AXGRjnb7btHdGr6429KKPyaFzMJMAK/G26mPq53f2/XG9RhHfGR8o8/0NcYEH6+acFS1EHgXmA8kA1cCG0Tklz6M7aTKL8snPjIeoPGmnpZW/LEp5JbkUl5V3uzhGlxcXTq9TfzdOnUj7548zul9TvOCNcYENG/a+K8QkYXAZ0A4MF5VLwFGAr/1bXgnj2fiT4pO4kjJEapqquos15qKHyC7KJuc4hwEoUtUl2ZtY0zyGOIj40mOSW56YWOMaUCYF8t8H3hKVVd6zlTVEhGZ6ZuwTr78snw6RznGv+nWqRuKklucWyfJ5pW2/OIuOLp0Hio+RJeoLoSFePPxH3fj8Bv5/mnfp2N4x2bv3xhjXLxp6pkF/M81ISIdRSQNQFWX+SQqPygoL3D3zXdddD2xuae8qpzSqlL3L4PmcN+9eyyr2X34XUTEkr4xptW8SfzvADUe09XOeQGlVhu/s+39xAu87uEaWtjGD86Kv+hQs9v3jTGmrXiT+MNUtcI14Xwd4buQTj7XWPyeF3ehbsXvHq6hBU09CZEJRIZFknUsyzFOTzO7chpjTFvxJvHnisgVrgkRmQYc9l1IJ19ZVRkV1RV1Kv4TR+h03eTVkopfRNwPZGnuyJzGGNOWvLm6eBvwpoj8DRBgP/Bjn0Z1krnG4ne18cdExBAZFtlwU08LR7pMjU1l55GdFFUUtaiN3xhj2oI3N3DtVtWJwBDgNFWdpKq7mlpPRAaJyCaPv0IRuVNEEkVkqYjsdP7r9/GCXZW8q+IXkXr78rdkZE5PKbEpfJPzDdD8PvzGGNNWvOpPKCKXAUOBSNfwvqr6cGPrqOp3wCjn+qFAFrAQuBdYpqp/FpF7ndP3tPQA2sKJiR+od9iG1lb8KTEpVFQ7LpdYU48xxl+8uYHr/+EYr+eXOJp6rgF6N3M/FwC7VTUDmAbMc86fB0xv5rbanGtIZteQDVD/sA2uir8l3Tmh9vDIVvEbY/zFm4u7k1T1x0Ceqs4GzgAGNnM/1wNvOV93U9Vs5+uDQL0Z0Dke0DoRWZebm9vM3TVPfRV/UnRSvRV/dHg04aHhLdqP52Bp1sZvjPEXbxJ/mfPfEhHpAVTiGK/HKyISAVxBPX3/VVUBrW89VX1BVcep6riuXbt6u7sWqbepJ7obucW51OjxWxjyylo2To+LZ8Vvid8Y4y/eJP5FIhIP/AXYAKQD/2zGPi4BNqiqq3w+JCLJAM5/63+q+UnUUBt/tVZzpOSIe15Lh2twcVX8cR3iiAyLbPF2jDGmNRpN/M4HsCxT1XxVfQ9H2/5gVX2wGfu4gePNPAAfAjOcr2cAHzRjWz5RUF5AWEgYHcOOD4fgeuhJdlG2e15rK/7kTskIYtW+McavGk38qloD/N1julxVC7zduIhEA1OA9z1m/xmYIiI7gQud037lumvX84HkfeL7ALA3b697Xmsr/vDQcJKik+zCrjHGr7zpzrlMRL4PvO9sk/eaqhYDnU+YdwRHL59ThudwDS79EvsBsCdvj3teayt+gEk9J5EWn9aqbRhjTGt4k/hvBe4CqkSkDEeXTlXVWJ9GdhLll+W779p1SYhMIK5DHLvzdtdarjUVP8D7173f9ELGGOND3ty5G6OqIaoaoaqxzul2nfTf2PwGU9+aiusHTEF5QZ2KX0Tom9DXXfFXVldSVFHU6sRvjDH+1mTFLyL1PrfvxAeztCcf7/yYj3Z8xM6jOxnYeSD5Zfkkd6nbQ7VvQl+25GwB6u/5Y4wx7ZE3TT13e7yOBMYD64HzfRLRSZBRkAHAf3b/x53460vo/RL6sWjHImq0plVj8RtjzKnEm6aeqR5/U4BhQJ7vQ/OdjHxH4l+6ZylQfxs/OCr+iuoKDhw70Kqx+I0x5lTSvIe+OmQCp7V1ICeLK5GHSAgr9q6gpLKEksqSeiv+vgl9Adh9dDelVaWAVfzGmPbPmzb+Zzk+rEIIjhE3N/gyKF/KLMxEUS4fcDmLdiziP7v/A9Tfdu9K/Hvy9rjvtLWK3xjT3nlT8a/zeF0FvKWqX/ooHp9Lz08H4KZRN7F452Le2eoYQqi+xN8rrhehEsqevD0kxzgu/lrFb4xp77xJ/O8CZapaDY6x9UUkSlVLfBuab7ja90d1H8X4lPF8+N2HQO0hmV3CQ8PpFdeLPflW8RtjAoc3g7QtAzp6THcEPvVNOL6XUZCBIKTGpjKl7xSKKoqAhrtpuvry55Xl0TGsIx3COpzMcI0xps15k/gjVbXINeF8HeW7kHwroyCDHjE9iAiN4Hv9vuee31ji3310d4NdPo0xpr3xJvEXi8gY14SIjAVKfReSb2XkZ9A73vEAsQkpE4iJiAGotzsnOPry55bksr9wv7XvG2MCgjeJ/07gHRH5r4h8ASwA7vBtWL6Tnp9O7zhH4g8PDWdy2mSg8YofYP2B9da+b4wJCE1e3FXVtSIyGBjknPWdqlb6NizfqK6pZn/hfq6Lu8497+YxN1NaVUpMh5h613El/iOlR6ziN8YEBG8etv4LIFpVt6jqFqCTiPzc96G1veyibKpqqmoNizx10FSW/mgpIVL/R+FK/GA9eowxgcGbpp6bVTXfNaGqecDNvgvJd1xdOV1t/N5I6JjgTviW+I0xgcCbxB8qHo+mEpFQIMJ3IfmOa3A2Vxu/t1xVvzX1GGMCgTeJfwmwQEQuEJELcDw/9xPfhuUbroq/V1yvZq3nTvxW8RtjAoA3d+7eA9wC3Oac3gx091lEPpSen06XqC5ER0Q3az1X4rd+/MaYQODNsMw1wBogHcdY/OcD23wblm9kFGQ0u5kHHH35wZp6jDGBocGKX0QGAjc4/w7j6L+Pqp53ckJrexkFGQztOrTZ643qPgpBavXwMcaY9qqxin87jur+clU9S1WfBapPTlhtT1Udd+22oOI/PeV0Dv7uIMOShvkgMmOMObkaS/xXAdnAChF50XlhVxpZ/pR2uOQwpVWlzerK6SkpOqmNIzLGGP9oMPGr6r9U9XpgMLACx9ANSSLyvIh8r6H1TlWucfhbUvEbY0wg8ebibrGq/lNVpwKpwEYcPX3aFXcf/hZW/MYYEyi86cfvpqp5qvqCql7gq4B8xX3XrlX8xpgg16zE355lFGQQExFjffGNMUEvaBJ/bkku3Tt1x2P0CWOMCUpBk/iLK4qbfceuMcYEouBJ/JXFRIdb4jfGmOBJ/FbxG2MMEEyJ3yp+Y4wBginxW8VvjDFAMCV+q/iNMQYIpsRfYYnfGGMgSBK/qlJSWWJNPcYYg48Tv4jEi8i7IrJdRLaJyBkikigiS0Vkp/Nfnz/dpLSqFEWt4jfGGHxf8f8VWKKqg4GROJ7cdS+wTFUHAMuc0z5VXFEMYBW/Mcbgw8QvInHAOcDLAKpaoar5wDRgnnOxecB0X8XgUlzpTPxW8RtjjE8r/j5ALvCqiGwUkZdEJBropqrZzmUOAt3qW1lEbhGRdSKyLjc3t1WBWMVvjDHH+TLxhwFjgOdVdTRQzAnNOqqqgNa3snP453GqOq5r166tCsQqfmOMOc6XiT8TyFTVNc7pd3F8ERwSkWQA5785PowBOF7xR4VH+XpXxhhzyvNZ4lfVg8B+ERnknHUBsBX4EJjhnDcD+MBXMbi4K35r6jHGGMJ8vP1fAm+KSASwB/gJji+bt0VkJpABXOvjGI638VtTjzHG+Dbxq+omYFw9b53URzdaxW+MMccFxZ27VvEbY8xxwZH4reI3xhi34Ej8FcUIQsewjv4OxRhj/C44En9lMVHhUfagdWOMIVgSvz2ExRhj3IIi8ZdUldiFXWOMcQqKxG8VvzHGHBccid8eu2iMMW7Bkfit4jfGGLfgSPxW8RtjjFtwJH6r+I0xxi04Er9V/MYY4xYcib/CEr8xxrgER+KvtKYeY4xxCfjEX1FdQVVNlVX8xhjjFPCJ3x60bowxtQV+4rcHrRtjTC2Bn/it4jfGmFoCP/E7K/6o8Cg/R2KMMaeGwE/89thFY4ypJfATvz120Rhjagn8xG8VvzHG1BL4id8qfmOMqSXwE79V/MYYU0vgJ36r+I0xppbAT/xW8RtjTC2Bn/gri+kQ2oHQkFB/h2KMMaeEwE/89hAWY4ypJfATvz2ExRhjagmOxG8VvzHGuAV84i+pLLGK3xhjPAR84rc2fmOMqS3wE7+18RtjTC2Bn/it4jfGmFoCP/FbxW+MMbWE+XLjIpIOHAOqgSpVHSciicACIA1IB65V1TxfxVBcYYnfGGM8nYyK/zxVHaWq45zT9wLLVHUAsMw57TPWndMYY2rzR1PPNGCe8/U8YLqvdlRdU01ZVZlV/MYY48HXiV+B/4jIehG5xTmvm6pmO18fBLr5aucllSWAjcxpjDGefNrGD5ylqlkikgQsFZHtnm+qqoqI1rei84viFoBevXq1aOfuIZmt4jfGGDefVvyqmuX8NwdYCIwHDolIMoDz35wG1n1BVcep6riuXbu2aP+uIZmjwqNatL4xxgQinyV+EYkWkRjXa+B7wBbgQ2CGc7EZwAe+isEewmKMMXX5sqmnG7BQRFz7+aeqLhGRtcDbIjITyACu9VUA9hAWY4ypy2eJX1X3ACPrmX8EuMBX+/VkFb8xxtQV0HfuWsVvjDF1BXbit4rfGGPqCOzEbxW/McbUEdiJ3yp+Y4ypI7ATv1X8xhhTR2An/spiQiWUiNAIf4dijDGnjMBO/M6HsDjvJTDGGEOgJ357CIsxxtQR+InfLuwaY0wtgZ347elbxhhTh6+HZfarM1LPYEjXIf4OwxhjTikBnfh/f/bv/R2CMcaccgK6qccYY0xdlviNMSbIWOI3xpggY4nfGGOCjCV+Y4wJMpb4jTEmyFjiN8aYIGOJ3xhjgoyoqr9jaJKI5AIZLVy9C3C4DcNpL4LxuIPxmCE4j9uO2Tu9VbXriTPbReJvDRFZp6rj/B3HyRaMxx2MxwzBedx2zK1jTT3GGBNkLPEbY0yQCYbE/4K/A/CTYDzuYDxmCM7jtmNuhYBv4zfGGFNbMFT8xhhjPFjiN8aYIBPQiV9ELhaR70Rkl4jc6+94fEFEeorIChHZKiLfisivnfMTRWSpiOx0/pvg71jbmoiEishGEfnIOd1HRNY4z/cCEYnwd4xtTUTiReRdEdkuIttE5IxAP9ci8hvnf9tbROQtEYkMxHMtIq+ISI6IbPGYV++5FYdnnMe/WUTGNGdfAZv4RSQU+DtwCTAEuEFEAvE5jFXAb1V1CDAR+IXzOO8FlqnqAGCZczrQ/BrY5jH9OPCUqvYH8oCZfonKt/4KLFHVwcBIHMcfsOdaRFKAXwHjVHUYEApcT2Ce69eAi0+Y19C5vQQY4Py7BXi+OTsK2MQPjAd2qeoeVa0A5gPT/BxTm1PVbFXd4Hx9DEciSMFxrPOci80DpvsnQt8QkVTgMuAl57QA5wPvOhcJxGOOA84BXgZQ1QpVzSfAzzWOR8R2FJEwIArIJgDPtaquBI6eMLuhczsNeF0dvgLiRSTZ230FcuJPAfZ7TGc65wUsEUkDRgNrgG6qmu186yDQzU9h+crTwP8H1DinOwP5qlrlnA7E890HyAVedTZxvSQi0QTwuVbVLGAusA9Hwi8A1hP459qloXPbqvwWyIk/qIhIJ+A94E5VLfR8Tx19dgOm366IXA7kqOp6f8dykoUBY4DnVXU0UMwJzToBeK4TcFS3fYAeQDR1m0OCQlue20BO/FlAT4/pVOe8gCMi4TiS/puq+r5z9iHXTz/nvzn+is8HzgSuEJF0HE145+No+453NgdAYJ7vTCBTVdc4p9/F8UUQyOf6QmCvquaqaiXwPo7zH+jn2qWhc9uq/BbIiX8tMMB59T8CxwWhD/0cU5tztm2/DGxT1Sc93voQmOF8PQP44GTH5iuq+ntVTVXVNBzndbmq3gisAK52LhZQxwygqgeB/SIyyDnrAmArAXyucTTxTBSRKOd/665jDuhz7aGhc/sh8GNn756JQIFHk1DTVDVg/4BLgR3AbuA+f8fjo2M8C8fPv83AJuffpTjavJcBO4FPgUR/x+qj458MfOR83Rf4H7ALeAfo4O/4fHC8o4B1zvP9LyAh0M81MBvYDmwB/g/oEIjnGngLx3WMShy/7mY2dG4BwdFrcTfwDY5eT17vy4ZsMMaYIBPITT3GGGPqYYnfGGOCjCV+Y4wJMpb4jTEmyFjiN8aYIGOJ3wQtEakWkU0ef202uJmIpHmOsmjMqSSs6UWMCVilqjrK30EYc7JZxW/MCUQkXUTmiMg3IvI/EenvnJ8mIsud458vE5FezvndRGShiHzt/Jvk3FSoiLzoHEv+PyLS0bn8r5zPT9gsIvP9dJgmiFniN8Gs4wlNPdd5vFegqsOBv+EYCRTgWWCeqo4A3gSecc5/BvhcVUfiGDvnW+f8AcDfVXUokA983zn/XmC0czu3+ergjGmI3blrgpaIFKlqp3rmpwPnq+oe5wB4B1W1s4gcBpJVtdI5P1tVu4hILpCqquUe20gDlqrjARqIyD1AuKo+IiJLgCIcQy78S1WLfHyoxtRiFb8x9dMGXjdHucfrao5fU7sMxzgrY4C1HqNMGnNSWOI3pn7Xefy72vl6FY7RQAFuBP7rfL0MuB3czwGOa2ijIhIC9FTVFcA9QBxQ51eHMb5klYYJZh1FZJPH9BJVdXXpTBCRzTiq9huc836J4+lXd+N4EtZPnPN/DbwgIjNxVPa34xhlsT6hwBvOLwcBnlHH4xONOWmsjd+YEzjb+Mep6mF/x2KML1hTjzHGBBmr+I0xJshYxW+MMUHGEr8xxgQZS/zGGBNkLPEbY0yQscRvjDFB5v8HcasOVTbUt6wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best epoch: 90\n"
     ]
    }
   ],
   "source": [
    "# Conitnue training by loading checkpoint\n",
    "# Best epoch = epoch with least val loss\n",
    "# This cell is used only to plot the graphs\n",
    "trainer = Net_Trainer(device, 'wrn', 'wrn_dfw_c10', train_dataset_10, test_dataset_10, load_ckpt='v1_ckpt_wrn_dfw_c10', lr=1.0, optm='dfw', loss='svm', \n",
    "                      batch_size=128, epochs=100, weight_decay=1e-4, smooth=True)\n",
    "best_epoch, losses, errors, accs = trainer.train()\n",
    "plot_train(losses, errors, accs)\n",
    "print('Best epoch:',best_epoch)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "wrn_dfw_c10.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}