mobile_network_with_triplet_digiFace.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d0cefa12",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libaries\n",
    "\n",
    "from PIL import Image\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "import random\n",
    "import sys\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler\n",
    "from torchinfo import summary\n",
    "from torchvision import transforms\n",
    "from torch import optim\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "022d45fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.9.0 (tags/v3.9.0:9cf6752, Oct  5 2020, 15:34:40) [MSC v.1927 64 bit (AMD64)]\n",
      "CUDA Available: True\n",
      "GPU Name: NVIDIA GeForce RTX 2070\n",
      "GPU Properties:\n",
      " _CudaDeviceProperties(name='NVIDIA GeForce RTX 2070', major=7, minor=5, total_memory=8191MB, multi_processor_count=36)\n",
      "Sat Feb 18 11:47:29 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 526.98       Driver Version: 526.98       CUDA Version: 12.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ... WDDM  | 00000000:01:00.0  On |                  N/A |\n",
      "|  0%   40C    P2    48W / 175W |    966MiB /  8192MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A       252    C+G   ...e31\\Documents\\AnyDesk.exe    N/A      |\n",
      "|    0   N/A  N/A      3668    C+G   ...2txyewy\\TextInputHost.exe    N/A      |\n",
      "|    0   N/A  N/A      4424    C+G   ...artMenuExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A      4464    C+G   ...perience\\NVIDIA Share.exe    N/A      |\n",
      "|    0   N/A  N/A      4904    C+G   ...5n1h2txyewy\\SearchApp.exe    N/A      |\n",
      "|    0   N/A  N/A      5368    C+G   ...e\\PhoneExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A      5736    C+G   ...in7x64\\steamwebhelper.exe    N/A      |\n",
      "|    0   N/A  N/A      7336      C   ...ython\\Python39\\python.exe    N/A      |\n",
      "|    0   N/A  N/A      9212    C+G   ...y\\ShellExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A     10136    C+G   C:\\Windows\\explorer.exe         N/A      |\n",
      "|    0   N/A  N/A     10572    C+G   ...perience\\NVIDIA Share.exe    N/A      |\n",
      "|    0   N/A  N/A     12060    C+G   ...8bbwe\\WindowsTerminal.exe    N/A      |\n",
      "|    0   N/A  N/A     15060    C+G   ...wekyb3d8bbwe\\Video.UI.exe    N/A      |\n",
      "|    0   N/A  N/A     16620    C+G   ...8wekyb3d8bbwe\\GameBar.exe    N/A      |\n",
      "|    0   N/A  N/A     16828    C+G   ...ge\\Application\\msedge.exe    N/A      |\n",
      "|    0   N/A  N/A     16972      C   ...ython\\Python39\\python.exe    N/A      |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "print('Python version:', sys.version)\n",
    "print('CUDA Available:', torch.cuda.is_available())\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('GPU Name:', torch.cuda.get_device_name())\n",
    "    print('GPU Properties:\\n', torch.cuda.get_device_properties('cuda'))\n",
    "    device = \"cuda\"\n",
    "    torch.cuda.set_per_process_memory_fraction(0.95, 0)\n",
    "    torch.cuda.empty_cache()\n",
    "else:\n",
    "    print(\"Cuda is not available, please use cpu instead\")\n",
    "    device = \"cpu\"\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d03cfbb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 41\n",
    "# Define custom dataset\n",
    "class FaceDataset(Dataset):\n",
    "    def __init__(self, df = None, num_sample = None, transform = None, num_img_pool = 10):\n",
    "        # set random seed for FaceDataset\n",
    "        np.random.seed(seed)\n",
    "        random.seed(seed)\n",
    "        # create constructors\n",
    "        self.unique_img_name = None\n",
    "        self.data = dict()\n",
    "        self.images = list()\n",
    "        # label to indices\n",
    "        self.label_to_indices = dict()\n",
    "        self.labels = list()\n",
    "        # read csv file\n",
    "        self.df = df\n",
    "        # set the transformation\n",
    "        self.transform = transform\n",
    "        # drop last n row from dataframe\n",
    "        self.df = self.df.head(num_sample)\n",
    "        #get the length of entire dataset\n",
    "        self.len_ = len(self.df)\n",
    "        # load imgs\n",
    "        self.load_imgs(self.df, num_imgs = num_img_pool, max = num_sample)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.len_\n",
    "\n",
    "    # get each pair of images -> 1: same identity, 0: different identity \n",
    "    # if index is even -> same pair \n",
    "    # if index is odd -> random identity\n",
    "    def __getitem__(self, idx):\n",
    "        anchor_img = self.images[idx]\n",
    "        anchor_label = self.labels[idx]\n",
    "\n",
    "        pos_idx = np.random.choice(np.arange(len(self.images))[self.labels == anchor_label])\n",
    "        neg_idx = np.random.choice(np.arange(len(self.images))[self.labels != anchor_label])\n",
    "\n",
    "        pos_img = self.images[pos_idx]\n",
    "        neg_img = self.images[neg_idx]\n",
    "\n",
    "        pos_label = self.labels[pos_idx]\n",
    "        neg_label = self.labels[neg_idx]\n",
    "\n",
    "        if self.transform is None:\n",
    "            img_to_tensor = transforms.ToTensor()\n",
    "            anchor_img = img_to_tensor(anchor_img)\n",
    "            pos_img = img_to_tensor(pos_img)\n",
    "            neg_img = img_to_tensor(neg_img)\n",
    "        else:\n",
    "            anchor_img = self.transform(anchor_img)\n",
    "            pos_img = self.transform(pos_img)\n",
    "            neg_img = self.transform(neg_img)\n",
    "\n",
    "        return anchor_img, pos_img, neg_img\n",
    "\n",
    "    # load imgs from pandas to memory and define the maximum number of images\n",
    "    def load_imgs(self, df, num_imgs, max):\n",
    "        # iterate thought each row\n",
    "        for i, row in tqdm(df.iterrows(), total = max):\n",
    "            # get identity of each row\n",
    "            row_identity = row['identity']\n",
    "            # append each identity to numberical value\n",
    "            self.label_to_indices[int(row_identity)] = i\n",
    "            count_img = 0\n",
    "            # loop imgs in each identity\n",
    "            for img_name in row['path']:\n",
    "                if count_img > num_imgs:\n",
    "                    break\n",
    "                # concatenate the directoru and image name\n",
    "                # path_to_image = self.dir+img_name\n",
    "                path_to_image = img_name\n",
    "                # open image and convert to RGB\n",
    "                img = Image.open(path_to_image).convert('RGB')\n",
    "\n",
    "                self.images.append(img)\n",
    "                self.labels.append(i)\n",
    "                count_img += 1\n",
    "            # print('Added img '+ str(row_identity))\n",
    "        self.labels = np.array(self.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e85575a",
   "metadata": {},
   "source": [
    "# Preperation of dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0a0c9d80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>identity</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[digiFace1M\\subjects_0-1999_72_imgs\\0\\20.png, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[digiFace1M\\subjects_0-1999_72_imgs\\1\\66.png, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[digiFace1M\\subjects_0-1999_72_imgs\\2\\29.png, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[digiFace1M\\subjects_0-1999_72_imgs\\3\\42.png, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[digiFace1M\\subjects_0-1999_72_imgs\\4\\33.png, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72661</th>\n",
       "      <td>199994</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1999...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72662</th>\n",
       "      <td>199995</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1999...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72663</th>\n",
       "      <td>199996</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1999...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72664</th>\n",
       "      <td>199997</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1999...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72665</th>\n",
       "      <td>199998</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1999...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>72666 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       identity                                               path\n",
       "0             0  [digiFace1M\\subjects_0-1999_72_imgs\\0\\20.png, ...\n",
       "1             1  [digiFace1M\\subjects_0-1999_72_imgs\\1\\66.png, ...\n",
       "2             2  [digiFace1M\\subjects_0-1999_72_imgs\\2\\29.png, ...\n",
       "3             3  [digiFace1M\\subjects_0-1999_72_imgs\\3\\42.png, ...\n",
       "4             4  [digiFace1M\\subjects_0-1999_72_imgs\\4\\33.png, ...\n",
       "...         ...                                                ...\n",
       "72661    199994  [digiFace1M\\subjects_166666-199998_5_imgs\\1999...\n",
       "72662    199995  [digiFace1M\\subjects_166666-199998_5_imgs\\1999...\n",
       "72663    199996  [digiFace1M\\subjects_166666-199998_5_imgs\\1999...\n",
       "72664    199997  [digiFace1M\\subjects_166666-199998_5_imgs\\1999...\n",
       "72665    199998  [digiFace1M\\subjects_166666-199998_5_imgs\\1999...\n",
       "\n",
       "[72666 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_df = pd.read_csv('./digiface_csv_files/digi_all.csv')\n",
    "ds_df = ds_df.groupby('identity')['path'].apply(list).reset_index()\n",
    "ds_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "84a5f52d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Size: 50866\n",
      "Val Size: 13080\n",
      "Test Size: 8720\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>identity</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26344</th>\n",
       "      <td>120344</td>\n",
       "      <td>[digiFace1M\\subjects_100000-133332_5_imgs\\1203...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61833</th>\n",
       "      <td>189166</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1891...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46921</th>\n",
       "      <td>174254</td>\n",
       "      <td>[digiFace1M\\subjects_166666-199998_5_imgs\\1742...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4873</th>\n",
       "      <td>8873</td>\n",
       "      <td>[digiFace1M\\subjects_8000-9999_72_imgs\\8873\\54...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19875</th>\n",
       "      <td>113875</td>\n",
       "      <td>[digiFace1M\\subjects_100000-133332_5_imgs\\1138...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       identity                                               path\n",
       "26344    120344  [digiFace1M\\subjects_100000-133332_5_imgs\\1203...\n",
       "61833    189166  [digiFace1M\\subjects_166666-199998_5_imgs\\1891...\n",
       "46921    174254  [digiFace1M\\subjects_166666-199998_5_imgs\\1742...\n",
       "4873       8873  [digiFace1M\\subjects_8000-9999_72_imgs\\8873\\54...\n",
       "19875    113875  [digiFace1M\\subjects_100000-133332_5_imgs\\1138..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# splitting each dataset\n",
    "train_df, eval_df = train_test_split(ds_df, test_size= 0.3, shuffle = True, random_state = seed)\n",
    "val_df, test_df = train_test_split(eval_df, test_size = 0.4, shuffle = True, random_state = seed)\n",
    "\n",
    "# print to check size of each dataset\n",
    "print(f'Train Size: {len(train_df)}')\n",
    "print(f'Val Size: {len(val_df)}')\n",
    "print(f'Test Size: {len(test_df)}')\n",
    "\n",
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7799a1c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Started Loading Train Set------------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "def796abab534fc58e81918dfb36bfd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Train Set:  6000\n",
      "-----------Finished Loading Train Set------------\n",
      "\n",
      "\n",
      "------------Started Loading Validation Set------------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c5e07ffba9e47668d1678b41e1ecb97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Train Set:  3000\n",
      "-----------Finished Loading Validation Set------------\n"
     ]
    }
   ],
   "source": [
    "# define image size\n",
    "img_size = 112\n",
    "\n",
    "# define transformation for test set\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.Resize((256, 256)),\n",
    "    transforms.CenterCrop(img_size),\n",
    "    transforms.RandomHorizontalFlip(p=0.6),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                         std=[0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "\n",
    "# define transformation for validation set\n",
    "val_transform = transforms.Compose([\n",
    "    transforms.Resize((256, 256)),\n",
    "    transforms.CenterCrop(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                         std=[0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "\n",
    "# define batch size\n",
    "train_batch_size = 64\n",
    "val_batch_size = 64\n",
    "print('------------Started Loading Train Set------------')\n",
    "# create dataloader for train set\n",
    "train_triplet_dataset = FaceDataset(df = train_df, num_sample = 6000, transform = train_transform)\n",
    "train_triplet_dataloader = DataLoader(train_triplet_dataset, batch_size=train_batch_size, shuffle=True, pin_memory=True)\n",
    "print('Total Train Set: ', train_triplet_dataset.__len__())\n",
    "print('-----------Finished Loading Train Set------------')\n",
    "\n",
    "print('\\n')\n",
    "\n",
    "print('------------Started Loading Validation Set------------')\n",
    "# create dataloader for validation set\n",
    "val_triplet_dataset = FaceDataset(df = val_df,num_sample = 3000, transform = val_transform)\n",
    "val_triplet_dataloader = DataLoader(val_triplet_dataset, batch_size=val_batch_size, shuffle=True, pin_memory=True)\n",
    "print('Total Train Set: ', val_triplet_dataset.__len__())\n",
    "print('-----------Finished Loading Validation Set------------')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "32f89ec3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(val_triplet_dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ec1c881",
   "metadata": {},
   "source": [
    "# Triplet Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aa2ff8da",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn import Linear, Conv2d, BatchNorm1d, BatchNorm2d, PReLU, ReLU, ReLU6, Sigmoid, Dropout2d\\\n",
    "    ,Dropout, AvgPool2d, MaxPool2d, AdaptiveAvgPool2d, Sequential, Module, Parameter\n",
    "\n",
    "class Flatten(Module):\n",
    "    def forward(self, input):\n",
    "        return input.view(input.size(0), -1)\n",
    "\n",
    "def l2_norm(input,axis=1):\n",
    "    norm = torch.norm(input,2,axis,True)\n",
    "    output = torch.div(input, norm)\n",
    "    return output\n",
    "\n",
    "class h_sigmoid(Module):\n",
    "    def __init__(self, inplace=True):\n",
    "        super(h_sigmoid, self).__init__()\n",
    "        self.relu = ReLU6(inplace=inplace)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.relu(x + 3) / 6\n",
    "\n",
    "\n",
    "class h_swish(Module):\n",
    "    def __init__(self, inplace=True):\n",
    "        super(h_swish, self).__init__()\n",
    "        self.sigmoid = h_sigmoid(inplace=inplace)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x * self.sigmoid(x)\n",
    "\n",
    "\n",
    "class SELayer(Module):\n",
    "    def __init__(self, channel, reduction=4):\n",
    "        super(SELayer, self).__init__()\n",
    "        self.avg_pool = AdaptiveAvgPool2d(1)\n",
    "        self.fc = Sequential(\n",
    "            Linear(channel, channel // reduction),\n",
    "            ReLU(inplace=True),\n",
    "            Linear(channel // reduction, channel),\n",
    "            h_sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        b, c, _, _ = x.size()\n",
    "        y = self.avg_pool(x).view(b, c)\n",
    "        y = self.fc(y).view(b, c, 1, 1)\n",
    "        return x * y\n",
    "\n",
    "class PermutationBlock(Module):\n",
    "    def __init__(self, groups):\n",
    "        super(PermutationBlock, self).__init__()\n",
    "        self.groups = groups\n",
    "\n",
    "    def forward(self, input):\n",
    "        n, c, h, w = input.size()\n",
    "        G = self.groups\n",
    "        output = input.view(n, G, c // G, h, w).permute(0, 2, 1, 3, 4).contiguous().view(n, c, h, w)\n",
    "        return output\n",
    "\n",
    "class Conv_block(Module):\n",
    "    def __init__(self, in_c, out_c, kernel=(1, 1), stride=(1, 1), padding=(0, 0), groups=1):\n",
    "        super(Conv_block, self).__init__()\n",
    "        self.conv = Conv2d(in_c, out_channels=out_c, kernel_size=kernel, groups=groups, stride=stride, padding=padding, bias=False)\n",
    "        self.bn = BatchNorm2d(out_c)\n",
    "        self.prelu = PReLU(out_c)\n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = self.bn(x)\n",
    "        x = self.prelu(x)\n",
    "        return x\n",
    "\n",
    "class Linear_block(Module):\n",
    "    def __init__(self, in_c, out_c, kernel=(1, 1), stride=(1, 1), padding=(0, 0), groups=1):\n",
    "        super(Linear_block, self).__init__()\n",
    "        self.conv = Conv2d(in_c, out_channels=out_c, kernel_size=kernel, groups=groups, stride=stride, padding=padding, bias=False)\n",
    "        self.bn = BatchNorm2d(out_c)\n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = self.bn(x)\n",
    "        return x\n",
    "\n",
    "class Depth_Wise(Module):\n",
    "    def __init__(self, in_c, out_c, residual = False, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=1):\n",
    "        super(Depth_Wise, self).__init__()\n",
    "        self.conv = Conv_block(in_c, out_c=groups, kernel=(1, 1), padding=(0, 0), stride=(1, 1))\n",
    "        self.conv_dw = Conv_block(groups, groups, groups=groups, kernel=kernel, padding=padding, stride=stride)\n",
    "        self.project = Linear_block(groups, out_c, kernel=(1, 1), padding=(0, 0), stride=(1, 1))\n",
    "        self.residual = residual\n",
    "    def forward(self, x):\n",
    "        if self.residual:\n",
    "            short_cut = x\n",
    "        x = self.conv(x)\n",
    "        x = self.conv_dw(x)\n",
    "        x = self.project(x)\n",
    "        if self.residual:\n",
    "            output = short_cut + x\n",
    "        else:\n",
    "            output = x\n",
    "        return output\n",
    "\n",
    "class Residual(Module):\n",
    "    def __init__(self, c, num_block, groups, kernel=(3, 3), stride=(1, 1), padding=(1, 1)):\n",
    "        super(Residual, self).__init__()\n",
    "        modules = []\n",
    "        for _ in range(num_block):\n",
    "            modules.append(Depth_Wise(c, c, residual=True, kernel=kernel, padding=padding, stride=stride, groups=groups))\n",
    "        self.model = Sequential(*modules)\n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "\n",
    "class LinearScheduler(nn.Module):\n",
    "    def __init__(self, dropblock, start_value, stop_value, nr_steps):\n",
    "        super(LinearScheduler, self).__init__()\n",
    "        self.dropblock = dropblock\n",
    "        self.i = 0\n",
    "        self.drop_values = np.linspace(start=start_value, stop=stop_value, num=nr_steps)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.dropblock(x)\n",
    "\n",
    "    def step(self):\n",
    "        if self.i < len(self.drop_values):\n",
    "            self.dropblock.drop_prob = self.drop_values[self.i]\n",
    "\n",
    "        self.i += 1\n",
    "\n",
    "class DropBlock2D(nn.Module):\n",
    "    r\"\"\"Randomly zeroes 2D spatial blocks of the input tensor.\n",
    "    As described in the paper\n",
    "    `DropBlock: A regularization method for convolutional networks`_ ,\n",
    "    dropping whole blocks of feature map allows to remove semantic\n",
    "    information as compared to regular dropout.\n",
    "    Args:\n",
    "        drop_prob (float): probability of an element to be dropped.\n",
    "        block_size (int): size of the block to drop\n",
    "    Shape:\n",
    "        - Input: `(N, C, H, W)`\n",
    "        - Output: `(N, C, H, W)`\n",
    "    .. _DropBlock: A regularization method for convolutional networks:\n",
    "       https://arxiv.org/abs/1810.12890\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, drop_prob, block_size):\n",
    "        super(DropBlock2D, self).__init__()\n",
    "\n",
    "        self.drop_prob = drop_prob\n",
    "        self.block_size = block_size\n",
    "\n",
    "    def forward(self, x):\n",
    "        # shape: (bsize, channels, height, width)\n",
    "\n",
    "        assert x.dim() == 4, \\\n",
    "            \"Expected input with 4 dimensions (bsize, channels, height, width)\"\n",
    "\n",
    "        if not self.training or self.drop_prob == 0.:\n",
    "            return x\n",
    "        else:\n",
    "            # get gamma value\n",
    "            gamma = self._compute_gamma(x)\n",
    "\n",
    "            # sample mask\n",
    "            mask = (torch.rand(x.shape[0], *x.shape[2:]) < gamma).float()\n",
    "\n",
    "            # place mask on input device\n",
    "            mask = mask.to(x.device)\n",
    "\n",
    "            # compute block mask\n",
    "            block_mask = self._compute_block_mask(mask)\n",
    "\n",
    "            # apply block mask\n",
    "            out = x * block_mask[:, None, :, :]\n",
    "\n",
    "            # scale output\n",
    "            out = out * block_mask.numel() / block_mask.sum()\n",
    "\n",
    "            return out\n",
    "\n",
    "    def _compute_block_mask(self, mask):\n",
    "        block_mask = F.max_pool2d(input=mask[:, None, :, :],\n",
    "                                  kernel_size=(self.block_size, self.block_size),\n",
    "                                  stride=(1, 1),\n",
    "                                  padding=self.block_size // 2)\n",
    "\n",
    "        if self.block_size % 2 == 0:\n",
    "            block_mask = block_mask[:, :, :-1, :-1]\n",
    "\n",
    "        block_mask = 1 - block_mask.squeeze(1)\n",
    "\n",
    "        return block_mask\n",
    "\n",
    "    def _compute_gamma(self, x):\n",
    "        return self.drop_prob / (self.block_size ** 2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2b43d1d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class MobileFaceNet(Module):\n",
    "    def __init__(self, embedding_size=512):\n",
    "        super(MobileFaceNet, self).__init__()\n",
    "        self.conv1 = Conv_block(3, 64, kernel=(3, 3), stride=(2, 2), padding=(1, 1))\n",
    "        self.conv2_dw = Conv_block(64, 64, kernel=(3, 3), stride=(1, 1), padding=(1, 1), groups=64)\n",
    "        self.conv_23 = Depth_Wise(64, 64, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=128)\n",
    "        self.conv_3 = Residual(64, num_block=4, groups=128, kernel=(3, 3), stride=(1, 1), padding=(1, 1))\n",
    "        self.conv_34 = Depth_Wise(64, 128, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=256)\n",
    "        self.conv_4 = Residual(128, num_block=6, groups=256, kernel=(3, 3), stride=(1, 1), padding=(1, 1))\n",
    "        self.conv_45 = Depth_Wise(128, 128, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=512)\n",
    "        self.conv_5 = Residual(128, num_block=2, groups=256, kernel=(3, 3), stride=(1, 1), padding=(1, 1))\n",
    "        self.conv_6_sep = Conv_block(128, 512, kernel=(1, 1), stride=(1, 1), padding=(0, 0))\n",
    "        self.conv_6_dw = Linear_block(512, 512, groups=512, kernel=(7,7), stride=(1, 1), padding=(0, 0))\n",
    "        self.conv_6_flatten = Flatten()\n",
    "        self.linear = Linear(512, embedding_size, bias=False)\n",
    "        self.bn = BatchNorm1d(embedding_size)\n",
    "        self.dropblock = DropBlock2D(block_size=3, drop_prob=0.3)\n",
    "        self.dropout = Dropout2d(0.3)\n",
    "\n",
    "    def forward_once(self,x):\n",
    "        out = self.conv1(x)\n",
    "\n",
    "        out = self.conv2_dw(out)\n",
    "\n",
    "        out = self.conv_23(out)\n",
    "\n",
    "        out = self.conv_3(out)\n",
    "\n",
    "        out = self.conv_34(out)\n",
    "\n",
    "        out = self.conv_4(out)\n",
    "\n",
    "        # out = self.dropblock(out)\n",
    "\n",
    "        out = self.conv_45(out)\n",
    "\n",
    "        out = self.conv_5(out)\n",
    "\n",
    "        out = self.conv_6_sep(out)\n",
    "\n",
    "        out = self.dropblock(out)\n",
    "\n",
    "        out = self.conv_6_dw(out)\n",
    "\n",
    "        out = self.dropblock(out)\n",
    "\n",
    "        out = self.conv_6_flatten(out)\n",
    "\n",
    "        out = self.linear(out)\n",
    "\n",
    "        out = self.bn(out)\n",
    "        return l2_norm(out)\n",
    "\n",
    "    def forward(self, anchor_img, positive_img, negative_img):\n",
    "        anchor = self.forward_once(anchor_img)\n",
    "        positive = self.forward_once(positive_img)\n",
    "        negative = self.forward_once(negative_img)\n",
    "        return anchor, positive, negative\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aee11b9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============================================================================================\n",
      "Layer (type:depth-idx)                        Output Shape              Param #\n",
      "===============================================================================================\n",
      "MobileFaceNet                                 [32, 512]                 --\n",
      "├─Conv_block: 1-1                             [32, 64, 56, 56]          --\n",
      "│    └─Conv2d: 2-1                            [32, 64, 56, 56]          1,728\n",
      "│    └─BatchNorm2d: 2-2                       [32, 64, 56, 56]          128\n",
      "│    └─PReLU: 2-3                             [32, 64, 56, 56]          64\n",
      "├─Conv_block: 1-2                             [32, 64, 56, 56]          --\n",
      "│    └─Conv2d: 2-4                            [32, 64, 56, 56]          576\n",
      "│    └─BatchNorm2d: 2-5                       [32, 64, 56, 56]          128\n",
      "│    └─PReLU: 2-6                             [32, 64, 56, 56]          64\n",
      "├─Depth_Wise: 1-3                             [32, 64, 28, 28]          --\n",
      "│    └─Conv_block: 2-7                        [32, 128, 56, 56]         --\n",
      "│    │    └─Conv2d: 3-1                       [32, 128, 56, 56]         8,192\n",
      "│    │    └─BatchNorm2d: 3-2                  [32, 128, 56, 56]         256\n",
      "│    │    └─PReLU: 3-3                        [32, 128, 56, 56]         128\n",
      "│    └─Conv_block: 2-8                        [32, 128, 28, 28]         --\n",
      "│    │    └─Conv2d: 3-4                       [32, 128, 28, 28]         1,152\n",
      "│    │    └─BatchNorm2d: 3-5                  [32, 128, 28, 28]         256\n",
      "│    │    └─PReLU: 3-6                        [32, 128, 28, 28]         128\n",
      "│    └─Linear_block: 2-9                      [32, 64, 28, 28]          --\n",
      "│    │    └─Conv2d: 3-7                       [32, 64, 28, 28]          8,192\n",
      "│    │    └─BatchNorm2d: 3-8                  [32, 64, 28, 28]          128\n",
      "├─Residual: 1-4                               [32, 64, 28, 28]          --\n",
      "│    └─Sequential: 2-10                       [32, 64, 28, 28]          --\n",
      "│    │    └─Depth_Wise: 3-9                   [32, 64, 28, 28]          18,432\n",
      "│    │    └─Depth_Wise: 3-10                  [32, 64, 28, 28]          18,432\n",
      "│    │    └─Depth_Wise: 3-11                  [32, 64, 28, 28]          18,432\n",
      "│    │    └─Depth_Wise: 3-12                  [32, 64, 28, 28]          18,432\n",
      "├─Depth_Wise: 1-5                             [32, 128, 14, 14]         --\n",
      "│    └─Conv_block: 2-11                       [32, 256, 28, 28]         --\n",
      "│    │    └─Conv2d: 3-13                      [32, 256, 28, 28]         16,384\n",
      "│    │    └─BatchNorm2d: 3-14                 [32, 256, 28, 28]         512\n",
      "│    │    └─PReLU: 3-15                       [32, 256, 28, 28]         256\n",
      "│    └─Conv_block: 2-12                       [32, 256, 14, 14]         --\n",
      "│    │    └─Conv2d: 3-16                      [32, 256, 14, 14]         2,304\n",
      "│    │    └─BatchNorm2d: 3-17                 [32, 256, 14, 14]         512\n",
      "│    │    └─PReLU: 3-18                       [32, 256, 14, 14]         256\n",
      "│    └─Linear_block: 2-13                     [32, 128, 14, 14]         --\n",
      "│    │    └─Conv2d: 3-19                      [32, 128, 14, 14]         32,768\n",
      "│    │    └─BatchNorm2d: 3-20                 [32, 128, 14, 14]         256\n",
      "├─Residual: 1-6                               [32, 128, 14, 14]         --\n",
      "│    └─Sequential: 2-14                       [32, 128, 14, 14]         --\n",
      "│    │    └─Depth_Wise: 3-21                  [32, 128, 14, 14]         69,632\n",
      "│    │    └─Depth_Wise: 3-22                  [32, 128, 14, 14]         69,632\n",
      "│    │    └─Depth_Wise: 3-23                  [32, 128, 14, 14]         69,632\n",
      "│    │    └─Depth_Wise: 3-24                  [32, 128, 14, 14]         69,632\n",
      "│    │    └─Depth_Wise: 3-25                  [32, 128, 14, 14]         69,632\n",
      "│    │    └─Depth_Wise: 3-26                  [32, 128, 14, 14]         69,632\n",
      "├─Depth_Wise: 1-7                             [32, 128, 7, 7]           --\n",
      "│    └─Conv_block: 2-15                       [32, 512, 14, 14]         --\n",
      "│    │    └─Conv2d: 3-27                      [32, 512, 14, 14]         65,536\n",
      "│    │    └─BatchNorm2d: 3-28                 [32, 512, 14, 14]         1,024\n",
      "│    │    └─PReLU: 3-29                       [32, 512, 14, 14]         512\n",
      "│    └─Conv_block: 2-16                       [32, 512, 7, 7]           --\n",
      "│    │    └─Conv2d: 3-30                      [32, 512, 7, 7]           4,608\n",
      "│    │    └─BatchNorm2d: 3-31                 [32, 512, 7, 7]           1,024\n",
      "│    │    └─PReLU: 3-32                       [32, 512, 7, 7]           512\n",
      "│    └─Linear_block: 2-17                     [32, 128, 7, 7]           --\n",
      "│    │    └─Conv2d: 3-33                      [32, 128, 7, 7]           65,536\n",
      "│    │    └─BatchNorm2d: 3-34                 [32, 128, 7, 7]           256\n",
      "├─Residual: 1-8                               [32, 128, 7, 7]           --\n",
      "│    └─Sequential: 2-18                       [32, 128, 7, 7]           --\n",
      "│    │    └─Depth_Wise: 3-35                  [32, 128, 7, 7]           69,632\n",
      "│    │    └─Depth_Wise: 3-36                  [32, 128, 7, 7]           69,632\n",
      "├─Conv_block: 1-9                             [32, 512, 7, 7]           --\n",
      "│    └─Conv2d: 2-19                           [32, 512, 7, 7]           65,536\n",
      "│    └─BatchNorm2d: 2-20                      [32, 512, 7, 7]           1,024\n",
      "│    └─PReLU: 2-21                            [32, 512, 7, 7]           512\n",
      "├─DropBlock2D: 1-10                           [32, 512, 7, 7]           --\n",
      "├─Linear_block: 1-11                          [32, 512, 1, 1]           --\n",
      "│    └─Conv2d: 2-22                           [32, 512, 1, 1]           25,088\n",
      "│    └─BatchNorm2d: 2-23                      [32, 512, 1, 1]           1,024\n",
      "├─DropBlock2D: 1-12                           [32, 512, 1, 1]           --\n",
      "├─Flatten: 1-13                               [32, 512]                 --\n",
      "├─Linear: 1-14                                [32, 512]                 262,144\n",
      "├─BatchNorm1d: 1-15                           [32, 512]                 1,024\n",
      "├─Conv_block: 1-16                            [32, 64, 56, 56]          (recursive)\n",
      "│    └─Conv2d: 2-24                           [32, 64, 56, 56]          (recursive)\n",
      "│    └─BatchNorm2d: 2-25                      [32, 64, 56, 56]          (recursive)\n",
      "│    └─PReLU: 2-26                            [32, 64, 56, 56]          (recursive)\n",
      "├─Conv_block: 1-17                            [32, 64, 56, 56]          (recursive)\n",
      "│    └─Conv2d: 2-27                           [32, 64, 56, 56]          (recursive)\n",
      "│    └─BatchNorm2d: 2-28                      [32, 64, 56, 56]          (recursive)\n",
      "│    └─PReLU: 2-29                            [32, 64, 56, 56]          (recursive)\n",
      "├─Depth_Wise: 1-18                            [32, 64, 28, 28]          (recursive)\n",
      "│    └─Conv_block: 2-30                       [32, 128, 56, 56]         (recursive)\n",
      "│    │    └─Conv2d: 3-37                      [32, 128, 56, 56]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-38                 [32, 128, 56, 56]         (recursive)\n",
      "│    │    └─PReLU: 3-39                       [32, 128, 56, 56]         (recursive)\n",
      "│    └─Conv_block: 2-31                       [32, 128, 28, 28]         (recursive)\n",
      "│    │    └─Conv2d: 3-40                      [32, 128, 28, 28]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-41                 [32, 128, 28, 28]         (recursive)\n",
      "│    │    └─PReLU: 3-42                       [32, 128, 28, 28]         (recursive)\n",
      "│    └─Linear_block: 2-32                     [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Conv2d: 3-43                      [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─BatchNorm2d: 3-44                 [32, 64, 28, 28]          (recursive)\n",
      "├─Residual: 1-19                              [32, 64, 28, 28]          (recursive)\n",
      "│    └─Sequential: 2-33                       [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-45                  [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-46                  [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-47                  [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-48                  [32, 64, 28, 28]          (recursive)\n",
      "├─Depth_Wise: 1-20                            [32, 128, 14, 14]         (recursive)\n",
      "│    └─Conv_block: 2-34                       [32, 256, 28, 28]         (recursive)\n",
      "│    │    └─Conv2d: 3-49                      [32, 256, 28, 28]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-50                 [32, 256, 28, 28]         (recursive)\n",
      "│    │    └─PReLU: 3-51                       [32, 256, 28, 28]         (recursive)\n",
      "│    └─Conv_block: 2-35                       [32, 256, 14, 14]         (recursive)\n",
      "│    │    └─Conv2d: 3-52                      [32, 256, 14, 14]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-53                 [32, 256, 14, 14]         (recursive)\n",
      "│    │    └─PReLU: 3-54                       [32, 256, 14, 14]         (recursive)\n",
      "│    └─Linear_block: 2-36                     [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Conv2d: 3-55                      [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-56                 [32, 128, 14, 14]         (recursive)\n",
      "├─Residual: 1-21                              [32, 128, 14, 14]         (recursive)\n",
      "│    └─Sequential: 2-37                       [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-57                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-58                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-59                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-60                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-61                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-62                  [32, 128, 14, 14]         (recursive)\n",
      "├─Depth_Wise: 1-22                            [32, 128, 7, 7]           (recursive)\n",
      "│    └─Conv_block: 2-38                       [32, 512, 14, 14]         (recursive)\n",
      "│    │    └─Conv2d: 3-63                      [32, 512, 14, 14]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-64                 [32, 512, 14, 14]         (recursive)\n",
      "│    │    └─PReLU: 3-65                       [32, 512, 14, 14]         (recursive)\n",
      "│    └─Conv_block: 2-39                       [32, 512, 7, 7]           (recursive)\n",
      "│    │    └─Conv2d: 3-66                      [32, 512, 7, 7]           (recursive)\n",
      "│    │    └─BatchNorm2d: 3-67                 [32, 512, 7, 7]           (recursive)\n",
      "│    │    └─PReLU: 3-68                       [32, 512, 7, 7]           (recursive)\n",
      "│    └─Linear_block: 2-40                     [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─Conv2d: 3-69                      [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─BatchNorm2d: 3-70                 [32, 128, 7, 7]           (recursive)\n",
      "├─Residual: 1-23                              [32, 128, 7, 7]           (recursive)\n",
      "│    └─Sequential: 2-41                       [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─Depth_Wise: 3-71                  [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─Depth_Wise: 3-72                  [32, 128, 7, 7]           (recursive)\n",
      "├─Conv_block: 1-24                            [32, 512, 7, 7]           (recursive)\n",
      "│    └─Conv2d: 2-42                           [32, 512, 7, 7]           (recursive)\n",
      "│    └─BatchNorm2d: 2-43                      [32, 512, 7, 7]           (recursive)\n",
      "│    └─PReLU: 2-44                            [32, 512, 7, 7]           (recursive)\n",
      "├─DropBlock2D: 1-25                           [32, 512, 7, 7]           --\n",
      "├─Linear_block: 1-26                          [32, 512, 1, 1]           (recursive)\n",
      "│    └─Conv2d: 2-45                           [32, 512, 1, 1]           (recursive)\n",
      "│    └─BatchNorm2d: 2-46                      [32, 512, 1, 1]           (recursive)\n",
      "├─DropBlock2D: 1-27                           [32, 512, 1, 1]           --\n",
      "├─Flatten: 1-28                               [32, 512]                 --\n",
      "├─Linear: 1-29                                [32, 512]                 (recursive)\n",
      "├─BatchNorm1d: 1-30                           [32, 512]                 (recursive)\n",
      "├─Conv_block: 1-31                            [32, 64, 56, 56]          (recursive)\n",
      "│    └─Conv2d: 2-47                           [32, 64, 56, 56]          (recursive)\n",
      "│    └─BatchNorm2d: 2-48                      [32, 64, 56, 56]          (recursive)\n",
      "│    └─PReLU: 2-49                            [32, 64, 56, 56]          (recursive)\n",
      "├─Conv_block: 1-32                            [32, 64, 56, 56]          (recursive)\n",
      "│    └─Conv2d: 2-50                           [32, 64, 56, 56]          (recursive)\n",
      "│    └─BatchNorm2d: 2-51                      [32, 64, 56, 56]          (recursive)\n",
      "│    └─PReLU: 2-52                            [32, 64, 56, 56]          (recursive)\n",
      "├─Depth_Wise: 1-33                            [32, 64, 28, 28]          (recursive)\n",
      "│    └─Conv_block: 2-53                       [32, 128, 56, 56]         (recursive)\n",
      "│    │    └─Conv2d: 3-73                      [32, 128, 56, 56]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-74                 [32, 128, 56, 56]         (recursive)\n",
      "│    │    └─PReLU: 3-75                       [32, 128, 56, 56]         (recursive)\n",
      "│    └─Conv_block: 2-54                       [32, 128, 28, 28]         (recursive)\n",
      "│    │    └─Conv2d: 3-76                      [32, 128, 28, 28]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-77                 [32, 128, 28, 28]         (recursive)\n",
      "│    │    └─PReLU: 3-78                       [32, 128, 28, 28]         (recursive)\n",
      "│    └─Linear_block: 2-55                     [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Conv2d: 3-79                      [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─BatchNorm2d: 3-80                 [32, 64, 28, 28]          (recursive)\n",
      "├─Residual: 1-34                              [32, 64, 28, 28]          (recursive)\n",
      "│    └─Sequential: 2-56                       [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-81                  [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-82                  [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-83                  [32, 64, 28, 28]          (recursive)\n",
      "│    │    └─Depth_Wise: 3-84                  [32, 64, 28, 28]          (recursive)\n",
      "├─Depth_Wise: 1-35                            [32, 128, 14, 14]         (recursive)\n",
      "│    └─Conv_block: 2-57                       [32, 256, 28, 28]         (recursive)\n",
      "│    │    └─Conv2d: 3-85                      [32, 256, 28, 28]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-86                 [32, 256, 28, 28]         (recursive)\n",
      "│    │    └─PReLU: 3-87                       [32, 256, 28, 28]         (recursive)\n",
      "│    └─Conv_block: 2-58                       [32, 256, 14, 14]         (recursive)\n",
      "│    │    └─Conv2d: 3-88                      [32, 256, 14, 14]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-89                 [32, 256, 14, 14]         (recursive)\n",
      "│    │    └─PReLU: 3-90                       [32, 256, 14, 14]         (recursive)\n",
      "│    └─Linear_block: 2-59                     [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Conv2d: 3-91                      [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-92                 [32, 128, 14, 14]         (recursive)\n",
      "├─Residual: 1-36                              [32, 128, 14, 14]         (recursive)\n",
      "│    └─Sequential: 2-60                       [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-93                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-94                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-95                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-96                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-97                  [32, 128, 14, 14]         (recursive)\n",
      "│    │    └─Depth_Wise: 3-98                  [32, 128, 14, 14]         (recursive)\n",
      "├─Depth_Wise: 1-37                            [32, 128, 7, 7]           (recursive)\n",
      "│    └─Conv_block: 2-61                       [32, 512, 14, 14]         (recursive)\n",
      "│    │    └─Conv2d: 3-99                      [32, 512, 14, 14]         (recursive)\n",
      "│    │    └─BatchNorm2d: 3-100                [32, 512, 14, 14]         (recursive)\n",
      "│    │    └─PReLU: 3-101                      [32, 512, 14, 14]         (recursive)\n",
      "│    └─Conv_block: 2-62                       [32, 512, 7, 7]           (recursive)\n",
      "│    │    └─Conv2d: 3-102                     [32, 512, 7, 7]           (recursive)\n",
      "│    │    └─BatchNorm2d: 3-103                [32, 512, 7, 7]           (recursive)\n",
      "│    │    └─PReLU: 3-104                      [32, 512, 7, 7]           (recursive)\n",
      "│    └─Linear_block: 2-63                     [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─Conv2d: 3-105                     [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─BatchNorm2d: 3-106                [32, 128, 7, 7]           (recursive)\n",
      "├─Residual: 1-38                              [32, 128, 7, 7]           (recursive)\n",
      "│    └─Sequential: 2-64                       [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─Depth_Wise: 3-107                 [32, 128, 7, 7]           (recursive)\n",
      "│    │    └─Depth_Wise: 3-108                 [32, 128, 7, 7]           (recursive)\n",
      "├─Conv_block: 1-39                            [32, 512, 7, 7]           (recursive)\n",
      "│    └─Conv2d: 2-65                           [32, 512, 7, 7]           (recursive)\n",
      "│    └─BatchNorm2d: 2-66                      [32, 512, 7, 7]           (recursive)\n",
      "│    └─PReLU: 2-67                            [32, 512, 7, 7]           (recursive)\n",
      "├─DropBlock2D: 1-40                           [32, 512, 7, 7]           --\n",
      "├─Linear_block: 1-41                          [32, 512, 1, 1]           (recursive)\n",
      "│    └─Conv2d: 2-68                           [32, 512, 1, 1]           (recursive)\n",
      "│    └─BatchNorm2d: 2-69                      [32, 512, 1, 1]           (recursive)\n",
      "├─DropBlock2D: 1-42                           [32, 512, 1, 1]           --\n",
      "├─Flatten: 1-43                               [32, 512]                 --\n",
      "├─Linear: 1-44                                [32, 512]                 (recursive)\n",
      "├─BatchNorm1d: 1-45                           [32, 512]                 (recursive)\n",
      "===============================================================================================\n",
      "Total params: 1,200,512\n",
      "Trainable params: 1,200,512\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (G): 21.23\n",
      "===============================================================================================\n",
      "Input size (MB): 14.45\n",
      "Forward/backward pass size (MB): 2347.96\n",
      "Params size (MB): 4.80\n",
      "Estimated Total Size (MB): 2367.21\n",
      "===============================================================================================\n"
     ]
    }
   ],
   "source": [
    "# triplet_model = TripletNetwork()\n",
    "triplet_model = MobileFaceNet()\n",
    "print(summary(triplet_model, input_size=[(32,3,112,112),(32,3,112,112),(32,3,112,112)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f9e2b3e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "class Arcface(Module):\n",
    "    # implementation of additive margin softmax loss in https://arxiv.org/abs/1801.05599\n",
    "    def __init__(self, embedding_size=512,  classnum=51332,  s=64., m=0.5):\n",
    "        super(Arcface, self).__init__()\n",
    "        self.classnum = classnum\n",
    "        self.kernel = Parameter(torch.Tensor(embedding_size,classnum))\n",
    "        # initial kernel\n",
    "        self.kernel.data.uniform_(-1, 1).renorm_(2,1,1e-5).mul_(1e5)\n",
    "        self.m = m # the margin value, default is 0.5\n",
    "        self.s = s # scalar value default is 64, see normface https://arxiv.org/abs/1704.06369\n",
    "        self.cos_m = math.cos(m)\n",
    "        self.sin_m = math.sin(m)\n",
    "        self.mm = self.sin_m * m  # issue 1\n",
    "        self.threshold = math.cos(math.pi - m)\n",
    "        print('Arcface head')\n",
    "\n",
    "    def forward(self, embbedings, label):\n",
    "        # weights norm\n",
    "        nB = len(embbedings)\n",
    "        kernel_norm = l2_norm(self.kernel,axis=0)\n",
    "        # cos(theta+m)\n",
    "        cos_theta = torch.mm(embbedings,kernel_norm)\n",
    "#         output = torch.mm(embbedings,kernel_norm)\n",
    "        cos_theta = cos_theta.clamp(-1,1) # for numerical stability\n",
    "        cos_theta_2 = torch.pow(cos_theta, 2)\n",
    "        sin_theta_2 = 1 - cos_theta_2\n",
    "        sin_theta = torch.sqrt(sin_theta_2)\n",
    "        cos_theta_m = (cos_theta * self.cos_m - sin_theta * self.sin_m)\n",
    "        # this condition controls the theta+m should in range [0, pi]\n",
    "        #      0<=theta+m<=pi\n",
    "        #     -m<=theta<=pi-m\n",
    "        cond_v = cos_theta - self.threshold\n",
    "        cond_mask = cond_v <= 0\n",
    "        keep_val = (cos_theta - self.mm) # when theta not in [0,pi], use cosface instead\n",
    "        cos_theta_m[cond_mask] = keep_val[cond_mask]\n",
    "        output = cos_theta * 1.0 # a little bit hacky way to prevent in_place operation on cos_theta\n",
    "        idx_ = torch.arange(0, nB, dtype=torch.long)\n",
    "        output[idx_, label] = cos_theta_m[idx_, label]\n",
    "        output *= self.s # scale up in order to make softmax work, first introduced in normface\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "69562b44",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TripletLoss(nn.Module):\n",
    "    def __init__(self, margin):\n",
    "        super(TripletLoss, self).__init__()\n",
    "        self.margin = margin\n",
    "\n",
    "    def forward(self, anchor, positive, negative, size_average=True):\n",
    "        # calculate Euclidean's distance\n",
    "        distance_positive = (anchor - positive).pow(2).sum(1)\n",
    "        distance_negative = (anchor - negative).pow(2).sum(1)\n",
    "        # distance_positive = F.pairwise_distance(anchor, positive).view((-1, 1))\n",
    "        # distance_negative = F.pairwise_distance(anchor, negative).view((-1, 1))\n",
    "        losses = F.relu(distance_positive - distance_negative + self.margin)\n",
    "        return torch.mean(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "94a3dfbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping:\n",
    "    def __init__(self, patient=5, min_delta=0):\n",
    "\n",
    "        self.patient = patient\n",
    "        self.min_delta = min_delta\n",
    "        self.counter = 0\n",
    "        self.early_stop = False\n",
    "\n",
    "    def __call__(self, train_loss, validation_loss):\n",
    "        if (validation_loss - train_loss) > self.min_delta:\n",
    "            self.counter +=1\n",
    "            if self.counter >= self.patient:\n",
    "                self.early_stop = True\n",
    "\n",
    "    def check_early_stop(self):\n",
    "        return self.early_stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "65b3ccf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_accuracy(threshold, dist, actual_issame):\n",
    "    predict_issame = np.less(dist, threshold)\n",
    "    tp = np.sum(np.logical_and(predict_issame, actual_issame))\n",
    "    tn = np.sum(np.logical_and(np.logical_not(predict_issame), np.logical_not(actual_issame)))\n",
    "\n",
    "    acc = float(tp + tn) / dist.size\n",
    "\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "74c46123",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = {\n",
    "    'margin': 1.2,\n",
    "    'lr': 1e-3,\n",
    "    'patience': 5,\n",
    "    'factor': 0.1,\n",
    "    'min_lr': 1e-10,\n",
    "    'threshold': 1e-3\n",
    "}\n",
    "criterion = TripletLoss(margin = model_config['margin'])\n",
    "optimizer = optim.Adam(triplet_model.parameters(), lr=model_config['lr'])\n",
    "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=model_config['patience'], factor=model_config['factor'], min_lr=model_config['min_lr'], threshold=model_config['threshold'], verbose=True)\n",
    "\n",
    "# early_stop = EarlyStopping(patient=5, min_delta=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "70f53461",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "split = 5\n",
    "k_fold = KFold(n_splits = 5, shuffle = True)\n",
    "# history_fold = {\n",
    "#     avg\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a6a8ac6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn(load_weight_pth = None, save_weight_pth = None, num_epochs = 200 , load_weight = False):\n",
    "    load_weight_path  = './weights/triplet_mobilenetwork_100_epochs_.pth' if (load_weight_pth == None) else load_weight_pth\n",
    "    save_weight_path = './weights/triplet_mobilenetwork_200_epochs_.pth' if (load_weight_pth == None) else save_weight_pth\n",
    "\n",
    "    # throw model to compute in the device\n",
    "    triplet_model.to(device=device)\n",
    "    total_train_loss = list()\n",
    "    total_val_loss = list()\n",
    "\n",
    "    total_train_acc = list()\n",
    "    total_val_acc = list()\n",
    "\n",
    "    best_val_loss = 0\n",
    "    best_train_loss = 0\n",
    "    previous_lr = 0\n",
    "\n",
    "    # load weight\n",
    "    if bool(load_weight_path) & load_weight:\n",
    "        triplet_model.load_state_dict(torch.load(load_weight_path))\n",
    "        print('Congratulations, Weight has been loaded!')\n",
    "\n",
    "    # for fold, (train_ids, valid_ids) in enumerate(k_fold.splits(train_triplet_dataset)):\n",
    "    #     print(f'Fold: {fold}')\n",
    "    #     train_subsampler = SubsetRandomSampler(train_ids)\n",
    "    #     val_subsampler = SubsetRandomSampler(valid_ids)\n",
    "    #\n",
    "    #     train_triplet_dataloader = DataLoader(train_triplet_dataset, batch_size=train_batch_size, shuffle=True, pin_memory=True)\n",
    "    # iterate though each epoch\n",
    "    for epoch in tqdm(range(num_epochs)):\n",
    "        print(f'Epoch:{epoch+1}/{num_epochs}')\n",
    "        # epoch_train_loss = list()\n",
    "        epoch_train_acc = np.array([])\n",
    "        # epoch_val_loss = list()\n",
    "        epoch_val_acc = np.array([])\n",
    "\n",
    "        train_loss = 0\n",
    "        val_loss = 0\n",
    "\n",
    "        # switch to train mode\n",
    "        triplet_model.train()\n",
    "        with torch.set_grad_enabled(True):\n",
    "            # loop though each item in dataloader\n",
    "            for anchor_img, pos_img, neg_img in tqdm(train_triplet_dataloader):\n",
    "\n",
    "                threshold_list = np.full(anchor_img.shape[0], model_config['margin'])\n",
    "                is_same = np.full(anchor_img.shape[0], 1)\n",
    "\n",
    "                # throw img to compute in the device\n",
    "                anchor_img = anchor_img.to(device)\n",
    "                pos_img = pos_img.to(device)\n",
    "                neg_img = neg_img.to(device)\n",
    "                # clear gradient to prevent gradient vanish\n",
    "                optimizer.zero_grad()\n",
    "                # train model\n",
    "                output1, output2, output3 = triplet_model(anchor_img, pos_img, neg_img)\n",
    "\n",
    "                # cal distance positive\n",
    "                distance_positive = (output1 - output2).pow(2).sum(1)\n",
    "                # compute loss from criterion\n",
    "                loss = criterion(output1, output2, output3)\n",
    "                # backward propagate\n",
    "                loss.backward()\n",
    "                # update parameters\n",
    "                optimizer.step()\n",
    "                # append train loss to epoch_train_loss\n",
    "                train_loss += loss.item() * anchor_img.size(0)\n",
    "\n",
    "                epoch_train_acc = np.append(epoch_train_acc,[calculate_accuracy(threshold_list,distance_positive.cpu().data.numpy(),is_same)])\n",
    "\n",
    "        # calculate loss\n",
    "        current_train_loss = train_loss / len(train_triplet_dataloader.sampler)\n",
    "        total_train_loss.append(current_train_loss)\n",
    "\n",
    "        # switch mode to eval\n",
    "        triplet_model.eval()\n",
    "        with torch.no_grad():\n",
    "            for anchor_img, pos_img, neg_img in val_triplet_dataloader:\n",
    "\n",
    "                threshold_list = np.full(anchor_img.shape[0], model_config['margin'])\n",
    "                is_same = np.full(anchor_img.shape[0], 1)\n",
    "\n",
    "                # throw img to compute in the device\n",
    "                anchor_img = anchor_img.to(device)\n",
    "                pos_img = pos_img.to(device)\n",
    "                neg_img = neg_img.to(device)\n",
    "                # clear gradient to prevent gradient vanish\n",
    "                optimizer.zero_grad()\n",
    "                # compute the out in eval mode\n",
    "                output1, output2, output3 = triplet_model(anchor_img, pos_img, neg_img)\n",
    "                # cal distance positive\n",
    "                distance_positive = (output1 - output2).pow(2).sum(1)\n",
    "                # compute loss in eval mode\n",
    "                loss = criterion(output1, output2, output3)\n",
    "                # calculate loss for each feature\n",
    "                val_loss += loss.item() * anchor_img.size(0)\n",
    "\n",
    "                # epoch_val_acc.append(calculate_accuracy(threshold_list,distance_positive.cpu().data.numpy(),is_same)[0])\n",
    "                epoch_val_acc = np.append(epoch_val_acc,[calculate_accuracy(threshold_list,distance_positive.cpu().data.numpy(),is_same)])\n",
    "\n",
    "        # calculate loss for each batch\n",
    "        current_val_loss = val_loss / len(val_triplet_dataloader.sampler)\n",
    "        total_val_loss.append(current_val_loss)\n",
    "\n",
    "        total_train_acc.append(epoch_train_acc.mean())\n",
    "        total_val_acc.append(epoch_val_acc.mean())\n",
    "    # get learning rate from model\n",
    "        optim_lr = optimizer.param_groups[0]['lr']\n",
    "\n",
    "        # append current validation loss to list\n",
    "        if (best_val_loss == 0) | (best_train_loss == 0):\n",
    "            best_val_loss = current_val_loss\n",
    "            best_train_loss = current_train_loss\n",
    "\n",
    "        # find the best val loss\n",
    "        if best_val_loss >= current_val_loss:\n",
    "            best_val_loss = current_val_loss\n",
    "\n",
    "        # find the best train loss\n",
    "        if best_train_loss >= current_train_loss:\n",
    "            best_train_loss = current_train_loss\n",
    "\n",
    "        print(f'train loss: {current_train_loss}')\n",
    "        print(f'val loss: {current_val_loss}')\n",
    "        print(f'Learning rate: {optim_lr:.8f}')\n",
    "        print(f'acc train: {epoch_train_acc.mean()}')\n",
    "        print(f'acc val: {epoch_val_acc.mean()}')\n",
    "        if (optim_lr < previous_lr) | (optim_lr > previous_lr):\n",
    "            print('LEARNING RATE HAS CHANGED!')\n",
    "        print('-------------------------------------------------------------')\n",
    "\n",
    "        # replace the previous lr with the current lr\n",
    "        previous_lr = optim_lr\n",
    "        scheduler.step(current_train_loss)\n",
    "\n",
    "    print('Best Validation loss',best_val_loss)\n",
    "    print('Best Train loss', best_train_loss)\n",
    "\n",
    "    # save model weights\n",
    "    torch.save(triplet_model.state_dict(), save_weight_path)\n",
    "    torch.cuda.empty_cache()\n",
    "    return total_train_loss, total_val_loss, total_train_acc, total_val_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26b30ad6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "69eb88ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1c0c5d972df4d0e8d8280bde8063aab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/120 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:1/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28c70113745a4a7cad9fb7bc3341c34b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.7801557566324869\n",
      "val loss: 0.5722194215456645\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.42774822695035464\n",
      "acc val: 0.65625\n",
      "LEARNING RATE HAS CHANGED!\n",
      "-------------------------------------------------------------\n",
      "Epoch:2/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d34905f09db647408890c3bf8afd29ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.6131229261557262\n",
      "val loss: 0.5439039573669434\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.6346963652482269\n",
      "acc val: 0.7066394376899695\n",
      "-------------------------------------------------------------\n",
      "Epoch:3/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "573f5ef7ed714603a9c7d94343533ca7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.5333196069399516\n",
      "val loss: 0.526369498093923\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.6869459219858156\n",
      "acc val: 0.7301006838905776\n",
      "-------------------------------------------------------------\n",
      "Epoch:4/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf15db969f2f4b86a435dee3c6841d50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.5197906551361084\n",
      "val loss: 0.4718895117441813\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.7077238475177305\n",
      "acc val: 0.7616831306990882\n",
      "-------------------------------------------------------------\n",
      "Epoch:5/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8472dab491454fce8053e2f65b0a101b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.47430268017450966\n",
      "val loss: 0.4468213476339976\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.7461768617021277\n",
      "acc val: 0.7604958206686929\n",
      "-------------------------------------------------------------\n",
      "Epoch:6/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d6ac6017ee54f65a17c751bb5690cae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.4280830322901408\n",
      "val loss: 0.450054496606191\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.770279255319149\n",
      "acc val: 0.7809650455927052\n",
      "-------------------------------------------------------------\n",
      "Epoch:7/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb2f83d93145416b893233c73937bac9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.4136760705312093\n",
      "val loss: 0.4082140560944875\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.7750443262411348\n",
      "acc val: 0.8205262158054711\n",
      "-------------------------------------------------------------\n",
      "Epoch:8/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f33c320058ce43fc8917dfff820d11ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.4035920976003011\n",
      "val loss: 0.3732845861911774\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.7711103723404256\n",
      "acc val: 0.8189114741641337\n",
      "-------------------------------------------------------------\n",
      "Epoch:9/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9bfb50a1de2545b7a358a410ee6fa5f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3923598749637604\n",
      "val loss: 0.37618178041776024\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.7889516843971631\n",
      "acc val: 0.8372435410334347\n",
      "-------------------------------------------------------------\n",
      "Epoch:10/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03ff0b9e411c43edaf3522b455d32b28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3610591322580973\n",
      "val loss: 0.35707572929064435\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8211436170212766\n",
      "acc val: 0.823518237082067\n",
      "-------------------------------------------------------------\n",
      "Epoch:11/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe7d810fe6264f249a28fbbc1c0be954",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.36612494723002115\n",
      "val loss: 0.36302090613047283\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8145500886524822\n",
      "acc val: 0.8251329787234043\n",
      "-------------------------------------------------------------\n",
      "Epoch:12/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "697fe4ccfe814395a7dd6ce4cc2a1d59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.35498908440272015\n",
      "val loss: 0.35379976717631023\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8186502659574468\n",
      "acc val: 0.8247530395136777\n",
      "-------------------------------------------------------------\n",
      "Epoch:13/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e7aece6bab74da3a473682f31772d87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3432457230091095\n",
      "val loss: 0.3434874864419301\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8265181737588652\n",
      "acc val: 0.8375284954407295\n",
      "-------------------------------------------------------------\n",
      "Epoch:14/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7b0234f78b74203863fe100f2c8c315",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.34718528620402017\n",
      "val loss: 0.320593691666921\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8254654255319149\n",
      "acc val: 0.859375\n",
      "-------------------------------------------------------------\n",
      "Epoch:15/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39607bc5d56c423eafffc11110ca5d9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3303584086894989\n",
      "val loss: 0.3423962609767914\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.836824024822695\n",
      "acc val: 0.8560980243161095\n",
      "-------------------------------------------------------------\n",
      "Epoch:16/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "104cc95019c3448ba190b4a90e4a97f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3172310891946157\n",
      "val loss: 0.34814342482884725\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8353834219858156\n",
      "acc val: 0.8301196808510638\n",
      "-------------------------------------------------------------\n",
      "Epoch:17/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83a9a9633f994e5c846467e2854b9926",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.32293004242579143\n",
      "val loss: 0.30840102744102477\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8410904255319149\n",
      "acc val: 0.8598024316109423\n",
      "-------------------------------------------------------------\n",
      "Epoch:18/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40eb91290308429ebf844c6a7c8f5ef9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3130691033999125\n",
      "val loss: 0.3184336893558502\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8489583333333333\n",
      "acc val: 0.8543408054711246\n",
      "-------------------------------------------------------------\n",
      "Epoch:19/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a4a8744a6d94d46aee89297c2560917",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3122010091940562\n",
      "val loss: 0.31607683261235553\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8519503546099291\n",
      "acc val: 0.8491166413373861\n",
      "-------------------------------------------------------------\n",
      "Epoch:20/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0279c6c4049f4b8c888e4e277ca664e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.3026063058376312\n",
      "val loss: 0.3045142794450124\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8541666666666667\n",
      "acc val: 0.8592800151975684\n",
      "-------------------------------------------------------------\n",
      "Epoch:21/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bb47e617e8b4829b6d9840dea054343",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.29773126606146494\n",
      "val loss: 0.32324108068148294\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8537234042553191\n",
      "acc val: 0.8580452127659575\n",
      "-------------------------------------------------------------\n",
      "Epoch:22/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "727f116222a94251accf6b58005fc552",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2955508728027344\n",
      "val loss: 0.3105145506064097\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8620899822695035\n",
      "acc val: 0.8410904255319149\n",
      "-------------------------------------------------------------\n",
      "Epoch:23/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bce9a4a75ae447a885c3ff917f5c52e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.29151117340723676\n",
      "val loss: 0.30178650037447613\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8601507092198583\n",
      "acc val: 0.8540558510638298\n",
      "-------------------------------------------------------------\n",
      "Epoch:24/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa60d2d213374d90a7e60ff428cccf1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2851400605837504\n",
      "val loss: 0.3076414773464203\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8674645390070922\n",
      "acc val: 0.854768237082067\n",
      "-------------------------------------------------------------\n",
      "Epoch:25/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b210b62075bd477cb421263c917d86d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.28840517059961956\n",
      "val loss: 0.2985789713859558\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8650265957446809\n",
      "acc val: 0.853295972644377\n",
      "-------------------------------------------------------------\n",
      "Epoch:26/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ff24afdf3f046ca8d06837766320997",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.27479042037328083\n",
      "val loss: 0.31264560349782305\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8764406028368795\n",
      "acc val: 0.8539608662613982\n",
      "-------------------------------------------------------------\n",
      "Epoch:27/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc34c4039f044c4fbf65a46249a4d6a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2767826127211253\n",
      "val loss: 0.28728236214319863\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8747783687943262\n",
      "acc val: 0.8612746960486323\n",
      "-------------------------------------------------------------\n",
      "Epoch:28/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d042bbb282a1441fa768e15c1fe4cbcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.26632141455014546\n",
      "val loss: 0.3073602488040924\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8802637411347518\n",
      "acc val: 0.8394281914893617\n",
      "-------------------------------------------------------------\n",
      "Epoch:29/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62b1efd7959b4e00add72b35b3e0de5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2724195481141408\n",
      "val loss: 0.291362619638443\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8762743794326242\n",
      "acc val: 0.8644091945288754\n",
      "-------------------------------------------------------------\n",
      "Epoch:30/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88cc2acdf9bf4bfa9326b160353607cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2593663984934489\n",
      "val loss: 0.28360525333881376\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8837544326241136\n",
      "acc val: 0.8760448328267478\n",
      "-------------------------------------------------------------\n",
      "Epoch:31/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6d46f8ded6545afad371a0cdf79cbca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.255851322889328\n",
      "val loss: 0.2715095990896225\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.883699024822695\n",
      "acc val: 0.8707731762917933\n",
      "-------------------------------------------------------------\n",
      "Epoch:32/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "030306b30feb45539abaaa3277870577",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2439209189414978\n",
      "val loss: 0.2691017309029897\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8909574468085106\n",
      "acc val: 0.8783719604863223\n",
      "-------------------------------------------------------------\n",
      "Epoch:33/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "079c940c189f46d3bd9454dc2104f355",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.24962741748491923\n",
      "val loss: 0.27018874017397565\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8916777482269503\n",
      "acc val: 0.8797967325227964\n",
      "-------------------------------------------------------------\n",
      "Epoch:34/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00602a176ca04807a1dd5b4aeb7ee519",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.25260178462664284\n",
      "val loss: 0.291359926700592\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8846409574468085\n",
      "acc val: 0.8528685410334347\n",
      "-------------------------------------------------------------\n",
      "Epoch:35/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0a8bfd9dda84adfbd7c4f7348f8a325",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.24892479451497396\n",
      "val loss: 0.28545624073346454\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8897938829787234\n",
      "acc val: 0.8837386018237082\n",
      "-------------------------------------------------------------\n",
      "Epoch:36/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd1cd9edc5644a6d8797e33885963ce7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2401934560139974\n",
      "val loss: 0.28557446265220643\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8931737588652482\n",
      "acc val: 0.8692534194528876\n",
      "-------------------------------------------------------------\n",
      "Epoch:37/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b29010491c374c98b39cda7371cbca82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2430729694366455\n",
      "val loss: 0.280309988339742\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8953900709219859\n",
      "acc val: 0.8789893617021277\n",
      "-------------------------------------------------------------\n",
      "Epoch:38/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53bf372f26284cd2aa5533c0dc97d116",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2449277412891388\n",
      "val loss: 0.27125961701075235\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8962211879432624\n",
      "acc val: 0.8619395896656535\n",
      "-------------------------------------------------------------\n",
      "Epoch:39/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e02b250dae8a40b78be5ae823b48dcc2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.24820294125874837\n",
      "val loss: 0.2894558397134145\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8850288120567376\n",
      "acc val: 0.8635543313069909\n",
      "-------------------------------------------------------------\n",
      "Epoch:40/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c375ed885bf840a38afe9ae172864f98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2544147544701894\n",
      "val loss: 0.2627808581988017\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8925642730496455\n",
      "acc val: 0.8648366261398177\n",
      "-------------------------------------------------------------\n",
      "Epoch:41/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f36bcb77e44b438eb078e8f5ea94100f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.23593605558077493\n",
      "val loss: 0.25868074933687846\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9065824468085106\n",
      "acc val: 0.8883453647416414\n",
      "-------------------------------------------------------------\n",
      "Epoch:42/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cd18acd531248eaac7a70443a8290fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.22932169950008394\n",
      "val loss: 0.2588848242759705\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.8986591312056738\n",
      "acc val: 0.8927621580547113\n",
      "-------------------------------------------------------------\n",
      "Epoch:43/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "773fd820b496431c88b43818cb195277",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.231653586268425\n",
      "val loss: 0.26069284784793856\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9073581560283689\n",
      "acc val: 0.8718180091185411\n",
      "-------------------------------------------------------------\n",
      "Epoch:44/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e81a7a1b295e48a5ae5606a237fef922",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.22808004983266195\n",
      "val loss: 0.2688785813649495\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9064162234042553\n",
      "acc val: 0.8787044072948329\n",
      "-------------------------------------------------------------\n",
      "Epoch:45/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5679e6c79393454da923ea2f7fe5d5f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.22462950547536215\n",
      "val loss: 0.25960835750897726\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9130651595744681\n",
      "acc val: 0.8856857902735563\n",
      "-------------------------------------------------------------\n",
      "Epoch:46/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c6b47afea2447aba94a42142decd3e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.23121954902013142\n",
      "val loss: 0.2495121881167094\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.907635195035461\n",
      "acc val: 0.8882978723404256\n",
      "-------------------------------------------------------------\n",
      "Epoch:47/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "338e29c02e5648679f0a03d1d185a566",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2224255588054657\n",
      "val loss: 0.2560146822532018\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9141179078014184\n",
      "acc val: 0.8781819908814589\n",
      "-------------------------------------------------------------\n",
      "Epoch:48/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c04444418324147a79c465bd16ded05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.21256602875391642\n",
      "val loss: 0.2488074045976003\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9185505319148937\n",
      "acc val: 0.879606762917933\n",
      "-------------------------------------------------------------\n",
      "Epoch:49/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c62c74b632c042259e0ff833e1516172",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.21297340734799702\n",
      "val loss: 0.2507017068862915\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9167220744680851\n",
      "acc val: 0.8841185410334347\n",
      "-------------------------------------------------------------\n",
      "Epoch:50/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "417d6edb35764e6e9638b116631cc350",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2072847591638565\n",
      "val loss: 0.24568081831932068\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9206006205673758\n",
      "acc val: 0.8856382978723404\n",
      "-------------------------------------------------------------\n",
      "Epoch:51/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38fd05aed9454f71a1ce69b85dca1650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.21244868040084838\n",
      "val loss: 0.23553311125437418\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9163896276595744\n",
      "acc val: 0.8882028875379939\n",
      "-------------------------------------------------------------\n",
      "Epoch:52/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bbbbf71f37740738b6744f7256cc404",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.21467927559216818\n",
      "val loss: 0.2537734122276306\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9182734929078014\n",
      "acc val: 0.8905775075987842\n",
      "-------------------------------------------------------------\n",
      "Epoch:53/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9d9d55517824203b40b659e113f429d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.19681189767519633\n",
      "val loss: 0.2499333922068278\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9260859929078014\n",
      "acc val: 0.8929521276595744\n",
      "-------------------------------------------------------------\n",
      "Epoch:54/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0b4b5328d304a8ca718cf4046974778",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.20737860298156738\n",
      "val loss: 0.23931235019365946\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9233710106382979\n",
      "acc val: 0.879891717325228\n",
      "-------------------------------------------------------------\n",
      "Epoch:55/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba86ab7115af4a11bda13f757577ce37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.21489121627807617\n",
      "val loss: 0.25658149806658426\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9171099290780141\n",
      "acc val: 0.8863506838905776\n",
      "-------------------------------------------------------------\n",
      "Epoch:56/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a9816e805284954a48c042b91804e2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.20337903412183125\n",
      "val loss: 0.24794044196605683\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9253102836879433\n",
      "acc val: 0.8854958206686929\n",
      "-------------------------------------------------------------\n",
      "Epoch:57/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6e6d0f9923f42e79c660bc13e34cee8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.2112399121125539\n",
      "val loss: 0.24698905404408772\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9229831560283689\n",
      "acc val: 0.8716280395136777\n",
      "-------------------------------------------------------------\n",
      "Epoch:58/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a15fedcdd9434cc095eb5b03c14fe24c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.20976261115074157\n",
      "val loss: 0.23586210882663727\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9317375886524822\n",
      "acc val: 0.8807465805471124\n",
      "-------------------------------------------------------------\n",
      "Epoch:59/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59948937a02740d0a877f7c5c1901c65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.19854905052979788\n",
      "val loss: 0.2508163997332255\n",
      "Learning rate: 0.00100000\n",
      "acc train: 0.9281914893617021\n",
      "acc val: 0.8872530395136777\n",
      "-------------------------------------------------------------\n",
      "Epoch 00059: reducing learning rate of group 0 to 1.0000e-04.\n",
      "Epoch:60/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b07f65286e4a49998b6e78f7d6025aca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.19250505713621777\n",
      "val loss: 0.23988618683815002\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9316267730496455\n",
      "acc val: 0.8919547872340425\n",
      "LEARNING RATE HAS CHANGED!\n",
      "-------------------------------------------------------------\n",
      "Epoch:61/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "597486a324e8471c9d173a2fb50af2d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1770215159257253\n",
      "val loss: 0.23326049304008484\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9438718971631205\n",
      "acc val: 0.8943294072948329\n",
      "-------------------------------------------------------------\n",
      "Epoch:62/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cfdd2c29667429d90541afd4a886e5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.17107592503229777\n",
      "val loss: 0.22434008185068766\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9463652482269503\n",
      "acc val: 0.9030205167173252\n",
      "-------------------------------------------------------------\n",
      "Epoch:63/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d50aa01f8794293aa924258e7dc5db4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.17104584272702536\n",
      "val loss: 0.2364407768646876\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9390514184397163\n",
      "acc val: 0.8954217325227964\n",
      "-------------------------------------------------------------\n",
      "Epoch:64/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "814625b49903403fbb95e52b002b7982",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.17267760622501374\n",
      "val loss: 0.23632939370473227\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9450908687943262\n",
      "acc val: 0.8970364741641337\n",
      "-------------------------------------------------------------\n",
      "Epoch:65/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "355b803b36b94b8c8a0295e50a87d6bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.16230430567264556\n",
      "val loss: 0.22061705301205317\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9556183510638298\n",
      "acc val: 0.9031155015197568\n",
      "-------------------------------------------------------------\n",
      "Epoch:66/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26a2a49259ed410cb393060a2bc52cfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.16189398284753165\n",
      "val loss: 0.2010437012910843\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9541777482269503\n",
      "acc val: 0.9141337386018237\n",
      "-------------------------------------------------------------\n",
      "Epoch:67/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2643591ac0bc415497142600515e466e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.16536820348103842\n",
      "val loss: 0.22164574166138967\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9534574468085106\n",
      "acc val: 0.9051101823708206\n",
      "-------------------------------------------------------------\n",
      "Epoch:68/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5fa07f25d484e50a0d8510d0456e9c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.16509063919385275\n",
      "val loss: 0.20495227134227753\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9510195035460992\n",
      "acc val: 0.8991261398176291\n",
      "-------------------------------------------------------------\n",
      "Epoch:69/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3eaeb42822d14da0b42fa89e0a65ee86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.15609053854147592\n",
      "val loss: 0.21778800225257874\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9566156914893617\n",
      "acc val: 0.8977963525835866\n",
      "-------------------------------------------------------------\n",
      "Epoch:70/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de5fe261d4514908bb934e3c05fa07cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.15393070888519286\n",
      "val loss: 0.21835726861159008\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9519614361702128\n",
      "acc val: 0.8930471124620061\n",
      "-------------------------------------------------------------\n",
      "Epoch:71/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94e38f62fdaf4bbd87b24b77fa60c5af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1595595552921295\n",
      "val loss: 0.22334634387493132\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9588874113475178\n",
      "acc val: 0.9043503039513677\n",
      "-------------------------------------------------------------\n",
      "Epoch:72/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae401839a1504ab3aa4cae9c6022b190",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.15518471912542978\n",
      "val loss: 0.20173777683575947\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9559507978723404\n",
      "acc val: 0.9086246200607904\n",
      "-------------------------------------------------------------\n",
      "Epoch:73/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d3d8747715e4caaa564b0b58f4d6176",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.15491523790359496\n",
      "val loss: 0.22559235537052155\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9573914007092199\n",
      "acc val: 0.9012632978723404\n",
      "-------------------------------------------------------------\n",
      "Epoch:74/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2806806bfb84c0baa43fc214152ed44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1547888853152593\n",
      "val loss: 0.2170264309644699\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9583333333333333\n",
      "acc val: 0.9041603343465046\n",
      "-------------------------------------------------------------\n",
      "Epoch:75/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d17e090afe0477ea2a9c238841c649b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1529438248872757\n",
      "val loss: 0.21728524180253347\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9603834219858156\n",
      "acc val: 0.9034004559270518\n",
      "-------------------------------------------------------------\n",
      "Epoch:76/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "225dc41d9b8b4a83a27ffc5b119009f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14566258792082468\n",
      "val loss: 0.1961624682744344\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9589428191489362\n",
      "acc val: 0.9064399696048632\n",
      "-------------------------------------------------------------\n",
      "Epoch:77/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1787373cd13a4e19aafb4da1158576ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.15130239164829254\n",
      "val loss: 0.22052762873967488\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9633754432624114\n",
      "acc val: 0.9069623860182372\n",
      "-------------------------------------------------------------\n",
      "Epoch:78/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b03afa988744f9993fb8d7ce0041420",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1488514971335729\n",
      "val loss: 0.21722455660502116\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9638741134751774\n",
      "acc val: 0.8970364741641337\n",
      "-------------------------------------------------------------\n",
      "Epoch:79/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "edc41c0e4d08473cbcacabe5207ce1b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1519785489241282\n",
      "val loss: 0.21594360335667928\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9619902482269503\n",
      "acc val: 0.9033054711246201\n",
      "-------------------------------------------------------------\n",
      "Epoch:80/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74e889899fac4f07ad47ddd31a9a55df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1540045437812805\n",
      "val loss: 0.21892395480473836\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.961436170212766\n",
      "acc val: 0.9040178571428571\n",
      "-------------------------------------------------------------\n",
      "Epoch:81/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a22aebf17fb249579261c8cd38bc7567",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1499582303762436\n",
      "val loss: 0.20686034969488779\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9637632978723404\n",
      "acc val: 0.8950892857142858\n",
      "-------------------------------------------------------------\n",
      "Epoch:82/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ff5087ab2c44f8e880ebff747a47521",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1426263287862142\n",
      "val loss: 0.2088456264336904\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9657025709219859\n",
      "acc val: 0.9063924772036475\n",
      "-------------------------------------------------------------\n",
      "Epoch:83/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4061acdafdd449b8b96e95b3bac7a8af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.15232189377148947\n",
      "val loss: 0.20152466265360514\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9626551418439716\n",
      "acc val: 0.9020706686930091\n",
      "-------------------------------------------------------------\n",
      "Epoch:84/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a409bca691b74561afdbec63c7161783",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14266568950812022\n",
      "val loss: 0.229364430586497\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9666999113475178\n",
      "acc val: 0.9049202127659575\n",
      "-------------------------------------------------------------\n",
      "Epoch:85/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "832f6cce015e4371bbbe2ff5d3e2fc0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14831985513369242\n",
      "val loss: 0.2228535816669464\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9588874113475178\n",
      "acc val: 0.9009308510638298\n",
      "-------------------------------------------------------------\n",
      "Epoch:86/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1ab1f7a6796452da1e91dd0157b94bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14674822195370993\n",
      "val loss: 0.21747272046407065\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9693040780141844\n",
      "acc val: 0.9045877659574468\n",
      "-------------------------------------------------------------\n",
      "Epoch:87/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02ec9d74182e433e8fa4a8abc4e6092c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14045418540636698\n",
      "val loss: 0.23255637979507446\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9688608156028369\n",
      "acc val: 0.899411094224924\n",
      "-------------------------------------------------------------\n",
      "Epoch:88/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b47000747354b5c8d10905bd2190fb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1434863291978836\n",
      "val loss: 0.21656831097602844\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9622118794326242\n",
      "acc val: 0.9034479483282675\n",
      "-------------------------------------------------------------\n",
      "Epoch:89/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1c703a2938442ed8b66923bf3de04fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14008234409491221\n",
      "val loss: 0.2143229788541794\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9694148936170213\n",
      "acc val: 0.9129939209726443\n",
      "-------------------------------------------------------------\n",
      "Epoch:90/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e93b57c3998f48b28d45d0c93647f547",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13996817382176718\n",
      "val loss: 0.20752743832270304\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9644281914893617\n",
      "acc val: 0.9058700607902735\n",
      "-------------------------------------------------------------\n",
      "Epoch:91/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcdafe423cc94a22bb40f7847622a1fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14725146730740865\n",
      "val loss: 0.22561786234378814\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9677526595744681\n",
      "acc val: 0.8986512158054711\n",
      "-------------------------------------------------------------\n",
      "Epoch:92/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02014da5b8d043b1bf11df633a210456",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14893117805322012\n",
      "val loss: 0.21956080595652264\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9646498226950354\n",
      "acc val: 0.901358282674772\n",
      "-------------------------------------------------------------\n",
      "Epoch:93/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d23d98f90e474cc4b082eeda322b2521",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13696626243988672\n",
      "val loss: 0.2122950202226639\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9693040780141844\n",
      "acc val: 0.9017857142857142\n",
      "-------------------------------------------------------------\n",
      "Epoch:94/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11a8e12b006949339edd6a78337e5124",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.14206753432750702\n",
      "val loss: 0.21152834343910218\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9649268617021277\n",
      "acc val: 0.8962765957446809\n",
      "-------------------------------------------------------------\n",
      "Epoch:95/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "339a396dc8a34c878931e6fd47f5a2b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13473985032240549\n",
      "val loss: 0.2187055337826411\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.969248670212766\n",
      "acc val: 0.8999810030395138\n",
      "-------------------------------------------------------------\n",
      "Epoch:96/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2124300ddf11499b849254fa236a83be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13608372290929158\n",
      "val loss: 0.21182714947064718\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9684175531914894\n",
      "acc val: 0.8985087386018237\n",
      "-------------------------------------------------------------\n",
      "Epoch:97/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99317bd29e694dfc9724ebb5dad1d127",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13426758007208506\n",
      "val loss: 0.2065368587176005\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9666445035460992\n",
      "acc val: 0.9010258358662614\n",
      "-------------------------------------------------------------\n",
      "Epoch:98/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fc934e6b3f44dfe9d3faa76152ac928",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13135182007153828\n",
      "val loss: 0.22552072044213614\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9738475177304965\n",
      "acc val: 0.8930946048632219\n",
      "-------------------------------------------------------------\n",
      "Epoch:99/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f263aaf0a04b4cc2af1d4f0b87b636e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1401053864955902\n",
      "val loss: 0.20903072893619537\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9640403368794327\n",
      "acc val: 0.9069148936170213\n",
      "-------------------------------------------------------------\n",
      "Epoch:100/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0762b954fe0641fbad326281dfce63d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13036665320396423\n",
      "val loss: 0.21187920264403026\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9722960992907801\n",
      "acc val: 0.9076747720364742\n",
      "-------------------------------------------------------------\n",
      "Epoch:101/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e8dbb1376e142aea9f6654bfba84957",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13564899023373922\n",
      "val loss: 0.2138513727982839\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.965093085106383\n",
      "acc val: 0.9035904255319149\n",
      "-------------------------------------------------------------\n",
      "Epoch:102/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffd121985a2142d48780966056718b70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13469798580805462\n",
      "val loss: 0.22179412786165872\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9677526595744681\n",
      "acc val: 0.8960866261398177\n",
      "-------------------------------------------------------------\n",
      "Epoch:103/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3fa453d70cd47a2a9f7f41dbe654d86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1322344266374906\n",
      "val loss: 0.21589537874857584\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9740137411347518\n",
      "acc val: 0.9048727203647415\n",
      "-------------------------------------------------------------\n",
      "Epoch:104/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebd7a1456bdd4837a2bb7456cb9bfc90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1327405216296514\n",
      "val loss: 0.2161375521818797\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9715757978723404\n",
      "acc val: 0.9059650455927052\n",
      "-------------------------------------------------------------\n",
      "Epoch:105/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5be9c957f4534f889cc32aa09681b3c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13098375348250071\n",
      "val loss: 0.20897219729423522\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9699689716312057\n",
      "acc val: 0.9087196048632219\n",
      "-------------------------------------------------------------\n",
      "Epoch:106/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe7437ccfb424fc6b92b675e69da1aac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13594292724132537\n",
      "val loss: 0.20878277071317036\n",
      "Learning rate: 0.00010000\n",
      "acc train: 0.9679188829787234\n",
      "acc val: 0.9020706686930091\n",
      "-------------------------------------------------------------\n",
      "Epoch 00106: reducing learning rate of group 0 to 1.0000e-05.\n",
      "Epoch:107/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5689c910b1c04c68b00a4d5db972fc7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13654227737585703\n",
      "val loss: 0.20323600268363953\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9691932624113476\n",
      "acc val: 0.9079122340425532\n",
      "LEARNING RATE HAS CHANGED!\n",
      "-------------------------------------------------------------\n",
      "Epoch:108/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e32da0c4ea3f4ade8c5fd85d0a22f6e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1316444536447525\n",
      "val loss: 0.21221111166477202\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9730718085106383\n",
      "acc val: 0.9063924772036475\n",
      "-------------------------------------------------------------\n",
      "Epoch:109/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a299cb4a87ae4e9cadae6dd4e1de60a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.12631152248382568\n",
      "val loss: 0.21449980926513673\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9710771276595744\n",
      "acc val: 0.9013107902735563\n",
      "-------------------------------------------------------------\n",
      "Epoch:110/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46fa25a04abf457bbd0f256efe75a635",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1349815359910329\n",
      "val loss: 0.21308527910709382\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9701906028368795\n",
      "acc val: 0.9082446808510638\n",
      "-------------------------------------------------------------\n",
      "Epoch:111/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46f18d48c4e746e785f914b4d746854a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13281896730264028\n",
      "val loss: 0.20780731956164042\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9715757978723404\n",
      "acc val: 0.913563829787234\n",
      "-------------------------------------------------------------\n",
      "Epoch:112/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0f2ef2b0bac416483d9507adedc6179",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1299887033700943\n",
      "val loss: 0.20972837241490683\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9670323581560284\n",
      "acc val: 0.8984612462006079\n",
      "-------------------------------------------------------------\n",
      "Epoch:113/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87822ba6c8514168841a435a5b66406f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.12848350421587626\n",
      "val loss: 0.20261641613642375\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9741799645390071\n",
      "acc val: 0.8993636018237082\n",
      "-------------------------------------------------------------\n",
      "Epoch:114/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bff320015466468b99a5373e1f7d8e75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.1342000142733256\n",
      "val loss: 0.2190963464975357\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.969248670212766\n",
      "acc val: 0.8987462006079028\n",
      "-------------------------------------------------------------\n",
      "Epoch:115/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c351930e675f41a687d4d12f55d9869d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13457337681452433\n",
      "val loss: 0.21489464290936788\n",
      "Learning rate: 0.00001000\n",
      "acc train: 0.9686945921985816\n",
      "acc val: 0.9073423252279635\n",
      "-------------------------------------------------------------\n",
      "Epoch 00115: reducing learning rate of group 0 to 1.0000e-06.\n",
      "Epoch:116/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "599fc2586a4d4212abf5845f5ce461d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.12420788327852884\n",
      "val loss: 0.20848475182056428\n",
      "Learning rate: 0.00000100\n",
      "acc train: 0.9744015957446809\n",
      "acc val: 0.9050626899696049\n",
      "LEARNING RATE HAS CHANGED!\n",
      "-------------------------------------------------------------\n",
      "Epoch:117/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03e38ed9aaba40ce9056af0c1c07b859",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.12868470108509064\n",
      "val loss: 0.21369548002878824\n",
      "Learning rate: 0.00000100\n",
      "acc train: 0.9689162234042553\n",
      "acc val: 0.901358282674772\n",
      "-------------------------------------------------------------\n",
      "Epoch:118/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33cfbdd1c7f141ddb70fa3eb9375124b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13523921783765158\n",
      "val loss: 0.21297263443470002\n",
      "Learning rate: 0.00000100\n",
      "acc train: 0.9746232269503545\n",
      "acc val: 0.9031155015197568\n",
      "-------------------------------------------------------------\n",
      "Epoch:119/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5212704df8384fb58f42dd9e45656c2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.12839624698956809\n",
      "val loss: 0.20803495879968006\n",
      "Learning rate: 0.00000100\n",
      "acc train: 0.9709663120567376\n",
      "acc val: 0.9059650455927052\n",
      "-------------------------------------------------------------\n",
      "Epoch:120/120\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "604bdf67d9624ac7acdf3e44d849eb06",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/94 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss: 0.13050558439890544\n",
      "val loss: 0.20668043200174968\n",
      "Learning rate: 0.00000100\n",
      "acc train: 0.976063829787234\n",
      "acc val: 0.9092895136778116\n",
      "-------------------------------------------------------------\n",
      "Best Validation loss 0.1961624682744344\n",
      "Best Train loss 0.12420788327852884\n"
     ]
    }
   ],
   "source": [
    "total_train_loss, total_val_loss, total_train_acc, total_val_acc = train_nn(num_epochs=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5d16cf72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+XklEQVR4nO3dd3hUVfrA8e9JhxQIKbQEQu8lEECqFAug0lQEVEBFVlzFtnZXWFx33ZXdtYtYsPxQRFBEiiggShMIGEqAUBMSWhqQRtrM+f1xhpBAyqQxTHg/z5PH3P7eXHznzLmnKK01QgghnJ+LowMQQghRNSShCyFEDSEJXQghaghJ6EIIUUNIQhdCiBrCzVEXDgwM1GFhYY66vBBCOKXt27cna62DittmV0JXSg0F3gRcgY+01q9dsr0J8BlQ17bPc1rrFaWdMywsjMjISHsuL4QQwkYpFVfStjKrXJRSrsC7wDCgPTBeKdX+kt1eAhZqrcOBccB7FQ9XCCFERdhTh94TOKS1PqK1zgUWACMv2UcDfrbf6wAnqi5EIYQQ9rAnoTcG4gstJ9jWFTYTuEcplQCsAB4t7kRKqalKqUilVGRSUlIFwhVCCFGSqnopOh74VGv9H6VUb+ALpVRHrbW18E5a67nAXICIiAgZc0CIKywvL4+EhASys7MdHYoog5eXFyEhIbi7u9t9jD0J/TgQWmg5xLausAeAoQBa681KKS8gEEi0OxIhRLVLSEjA19eXsLAwlFKODkeUQGtNSkoKCQkJNGvWzO7j7Kly2Qa0Uko1U0p5YF56Lr1kn2PAEAClVDvAC5A6FSGuMtnZ2QQEBEgyv8oppQgICCj3N6kyE7rWOh94BFgF7MO0ZolWSs1SSo2w7fYU8KBSaifwFTBZyzCOQlyVJJk7h4o8J7vq0G1tyldcsu7lQr/vBfqW++oVsC02lXUxiTx5YxtcXeQfphBCXOB0Xf93xp/l3V8Ok5mb7+hQhBDllJKSQteuXenatSsNGjSgcePGBcu5ubmlHhsZGcn06dPLdb2wsDCSk5MrE7JTcVjX/4ry9jQhZ+bk4+dl/9tfIYTjBQQEEBUVBcDMmTPx8fHhL3/5S8H2/Px83NyKT0sRERFERERciTCdltOV0AsndCGE85s8eTIPPfQQvXr14plnnmHr1q307t2b8PBw+vTpQ0xMDADr1q3j1ltvBcyHwf3338/AgQNp3rw5b731lt3Xi42NZfDgwXTu3JkhQ4Zw7NgxAL755hs6duxIly5dGDBgAADR0dH07NmTrl270rlzZw4ePFjFd1+1nK6E7uPpCkBGjsXBkQjh3P72QzR7T6RV6TnbN/Jjxm0dyn1cQkICmzZtwtXVlbS0NNavX4+bmxurV6/mhRdeYPHixZcds3//fn755RfS09Np06YN06ZNs6vN9qOPPsqkSZOYNGkSn3zyCdOnT2fJkiXMmjWLVatW0bhxY86ePQvAnDlzeOyxx7j77rvJzc3FYrm6847TJXRvDymhC1HT3Hnnnbi6msLauXPnmDRpEgcPHkQpRV5eXrHH3HLLLXh6euLp6UlwcDCnT58mJCSkzGtt3ryZb7/9FoB7772XZ555BoC+ffsyefJkxo4dy5gxYwDo3bs3r776KgkJCYwZM4ZWrVpVxe1WG+dL6LYqlwxJ6EJUSkVK0tXF29u74Pe//vWvDBo0iO+++47Y2FgGDhxY7DGenp4Fv7u6upKfX7mcMGfOHLZs2cLy5cvp3r0727dvZ8KECfTq1Yvly5czfPhwPvjgAwYPHlyp61Qnp6tD95E6dCFqtHPnztG4sRku6tNPP63y8/fp04cFCxYAMH/+fPr37w/A4cOH6dWrF7NmzSIoKIj4+HiOHDlC8+bNmT59OiNHjmTXrl1VHk9VcrqELi9FhajZnnnmGZ5//nnCw8MrXeoG6Ny5MyEhIYSEhPDkk0/y9ttvM2/ePDp37swXX3zBm2++CcDTTz9Np06d6NixI3369KFLly4sXLiQjh070rVrV/bs2cPEiRMrHU91Uo7q0BkREaErMsHF+VwL7V7+kWeHtmXawBbVEJkQNde+ffto166do8MQdirueSmltmuti22/6XQldC93F1yUlNCFEOJSTpfQlVJ4e7rJS1EhhLiE0yV0MC9GpYQuhBBFOWVC9/Z0k7FchBDiEs6Z0D1cpaeoEEJcwjkTulS5CCHEZSShCyGumEGDBrFq1aoi69544w2mTZtW4jEDBw7kQhPn4cOHF4yzUtjMmTOZPXt2qddesmQJe/fuLVh++eWXWb16dTmiL17hQcMczSkTuo+0chHCKY0fP76gl+YFCxYsYPz48XYdv2LFCurWrVuha1+a0GfNmsUNN9xQoXNdrZwyoXt7ukoJXQgndMcdd7B8+fKCySxiY2M5ceIE/fv3Z9q0aURERNChQwdmzJhR7PGFJ6x49dVXad26Nf369SsYYhfgww8/pEePHnTp0oXbb7+drKwsNm3axNKlS3n66afp2rUrhw8fZvLkySxatAiANWvWEB4eTqdOnbj//vvJyckpuN6MGTPo1q0bnTp1Yv/+/aXeX2pqKqNGjaJz585cd911BUMF/PrrrwUTeYSHh5Oens7JkycZMGAAXbt2pWPHjqxfv75yf1yccHAuuFDlIi9FhaiUlc/Bqd1Ve84GnWDYayVurlevHj179mTlypWMHDmSBQsWMHbsWJRSvPrqq9SrVw+LxcKQIUPYtWsXnTt3LvY827dvZ8GCBURFRZGfn0+3bt3o3r07AGPGjOHBBx8E4KWXXuLjjz/m0UcfZcSIEdx6663ccccdRc6VnZ3N5MmTWbNmDa1bt2bixIm8//77PP744wAEBgayY8cO3nvvPWbPns1HH31U4v3NmDGD8PBwlixZwtq1a5k4cSJRUVHMnj2bd999l759+5KRkYGXlxdz587l5ptv5sUXX8RisZCVlVWev3SxnLKE7uPhRq7FSm6+1dGhCCHKqXC1S+HqloULF9KtWzfCw8OJjo4uUj1yqfXr1zN69Ghq166Nn58fI0aMKNi2Z88e+vfvT6dOnZg/fz7R0dGlxhMTE0OzZs1o3bo1AJMmTeK3334r2H5hKN3u3bsTGxtb6rk2bNjAvffeC8DgwYNJSUkhLS2Nvn378uSTT/LWW29x9uxZ3Nzc6NGjB/PmzWPmzJns3r0bX1/fUs9tD6ctoYPp/u/h5uHgaIRwUqWUpKvTyJEjeeKJJ9ixYwdZWVl0796do0ePMnv2bLZt24a/vz+TJ08mOzu7QuefPHkyS5YsoUuXLnz66aesW7euUvFeGKa3MkP0Pvfcc9xyyy2sWLGCvn37smrVKgYMGMBvv/3G8uXLmTx5Mk8++WSlB/+yq4SulBqqlIpRSh1SSj1XzPb/KaWibD8HlFJnKxVVGXxkTHQhnJaPjw+DBg3i/vvvLyidp6Wl4e3tTZ06dTh9+jQrV64s9RwDBgxgyZIlnD9/nvT0dH744YeCbenp6TRs2JC8vDzmz59fsN7X15f09PTLztWmTRtiY2M5dOgQAF988QXXX399he6tf//+Bddct24dgYGB+Pn5cfjwYTp16sSzzz5Ljx492L9/P3FxcdSvX58HH3yQKVOmsGPHjgpds7AyS+hKKVfgXeBGIAHYppRaqrUu+D6ktX6i0P6PAuGVjqwUBSV06S0qhFMaP348o0ePLqh66dKlC+Hh4bRt25bQ0FD69u1b6vHdunXjrrvuokuXLgQHB9OjR4+Cba+88gq9evUiKCiIXr16FSTxcePG8eCDD/LWW28VvAwF8PLyYt68edx5553k5+fTo0cPHnrooQrd14W5Tjt37kzt2rX57LPPANM085dffsHFxYUOHTowbNgwFixYwOuvv467uzs+Pj58/vnnFbpmYWUOn6uU6g3M1FrfbFt+HkBr/c8S9t8EzNBa/1zaeSs6fC7AuphEJs/bxuJpvenetF6FziHEtUiGz3Uu1TF8bmMgvtBygm3dZZRSTYFmwNoStk9VSkUqpSKTkpLsuHTxLla5SEsXIYS4oKpbuYwDFmmti820Wuu5WusIrXVEUFBQhS8isxYJIcTl7Enox4HQQsshtnXFGQd8VdmgyiIvRYWoOEfNUibKpyLPyZ6Evg1opZRqppTywCTtpZfupJRqC/gDm8sdRTlJCV2IivHy8iIlJUWS+lVOa01KSgpeXl7lOq7MVi5a63yl1CPAKsAV+ERrHa2UmgVEaq0vJPdxwAJ9Bf6leHu6ApLQhSivkJAQEhISqMw7LHFleHl5ERISUq5j7OpYpLVeAay4ZN3LlyzPLNeVK8HTzRV3VyUvRYUoJ3d3d5o1a+boMEQ1ccqu/yBD6AohxKWcN6F7SEIXQojCnDahy5joQghRlNMmdG9PV+n6L4QQhThxQneTl6JCCFGI0yZ0H3kpKoQQRThtQpdWLkIIUZTTJnR5KSqEEEU5bUK/MFG0dGEWQgjDiRO6G1YN2Xkyr6gQQoATJ3QZcVEIIYpy2oTu7SEjLgohRGHOm9ClhC6EEEU4bUL3kTHRhRCiCKdN6AVjokv3fyGEAJw4octE0UIIUZTTJnSZhk4IIYqShC6EEDWE8yZ0D1OHLq1chBDCcNqE7ubqgpe7i5TQhRDCxmkTOoCPpzvp2ZLQhRAC7EzoSqmhSqkYpdQhpdRzJewzVim1VykVrZT6smrDLF6wryeJ6TlX4lJCCHHVcytrB6WUK/AucCOQAGxTSi3VWu8ttE8r4Hmgr9b6jFIquLoCLqxBHS9Oncu+EpcSQoirnj0l9J7AIa31Ea11LrAAGHnJPg8C72qtzwBorROrNszi1ffz4nSaJHQhhAD7EnpjIL7QcoJtXWGtgdZKqY1Kqd+VUkOrKsDSNPDzIiUzl5x86VwkhBBV9VLUDWgFDATGAx8qpepeupNSaqpSKlIpFZmUlFTpizas4wVAYprUowshhD0J/TgQWmg5xLausARgqdY6T2t9FDiASfBFaK3naq0jtNYRQUFBFY25QH1bQj8l1S5CCGFXQt8GtFJKNVNKeQDjgKWX7LMEUzpHKRWIqYI5UnVhFq+Bny2hy4tRIYQoO6FrrfOBR4BVwD5godY6Wik1Syk1wrbbKiBFKbUX+AV4WmudUl1BX3AhocuLUSGEsKPZIoDWegWw4pJ1Lxf6XQNP2n6uGL9abtRyd+WklNCFEMK5e4oqpUxbdCmhCyGEkyZ0rQt+re/nyWkpoQshhBMm9O2fwltdwZIHmHp0KaELIYQzJvTaAXAmFo5tBqBBnVqcTsvGatWlHyeEEDWc8yX05oPA1QNifgSggZ8neRZNalaugwMTQgjHcr6E7ukDzQbAAVtCryNt0YUQApwxoQO0HgqphyH5IPWlLboQQgBOm9BvNv+NWUnDOrUA6f4vhBDOmdDrNoH6HeHAKgJ9PHBRUuUihBDOmdDBVLsc24xb7jmCfD0loQshrnnOndC1BQ6ulrboQgiBMyf0xt3Bpz7sWkCDOjJzkRBCOG9Cd3GBXn+CQ6sJd4uVAbqEENc8503oAD0eBK863JTyf6Rn55OVm+/oiIQQwmGcO6F7+UGvh2ie/AutVTw74885OiIhhHAY507oAL0eQnt485day5i5NJo8i9XREQkhhEM4f0KvXQ/VYwo36o2cOX2Mj9YfdXREQgjhEM6f0AFaD0NpK/eEnePNNQc4lpLl6IiEEOKKqxkJPbA1AJPb5OHm4sLsn2IcHJAQQlx5NSOhewdArXr4ZRxlUNtgtsedcXREQghxxdWMhA6mlJ58kPYN/Th+9jznsvIcHZEQQlxRNSiht4LkA7Rv5AfA3pNpDg5ICCGuLLsSulJqqFIqRil1SCn1XDHbJyulkpRSUbafKVUfahkCW0NmEu3rms5FktCFENcat7J2UEq5Au8CNwIJwDal1FKt9d5Ldv1aa/1INcRoH9uL0aCceAJ9PNknCV0IcY2xp4TeEziktT6itc4FFgAjqzesCghsZf5rq3bZe0ISuhDi2mJPQm8MxBdaTrCtu9TtSqldSqlFSqnQ4k6klJqqlIpUSkUmJSVVINxS1G1qJo9OPkD7hn4cTEwnN196jQohrh1V9VL0ByBMa90Z+Bn4rLidtNZztdYRWuuIoKCgKrq0jasb1GsByQdp19CXPIvmUGJG1V5DCCGuYvYk9ONA4RJ3iG1dAa11itY6x7b4EdC9asIrp8BWkHyQDraWLlKPLoS4ltiT0LcBrZRSzZRSHsA4YGnhHZRSDQstjgD2VV2I5RDYGs4cpZm/J17uLtLSRQhxTSmzlYvWOl8p9QiwCnAFPtFaRyulZgGRWuulwHSl1AggH0gFJldjzCULbAXWfFzPxtKmgbwYFUJcW8pM6ABa6xXAikvWvVzo9+eB56s2tAoo3NKlYRNW7D6F1hqllGPjEkKIK6Dm9BQFCCic0P04dz5PpqYTQlwzalZC9/ID34ZFhwCQahchxDWiZiV0gIZdIHYDber7AtLSRQhx7ah5Cb39SDgXj09SFE0DaktLFyHENaPmJfQ2w02P0ehvadfAT0roQohrRs1L6LXqQoshEL2Edg18iEvNIjMn39FRCSFEtat5CR2g4xhIP0Fvj4NoDftPpTs6IiGEqHY1M6G3GQZuXrRLXQvIi1EhxLWhZiZ0T19odSM+h5fxpNdShq4dDl/e5eiohBCiWtXMhA7QYQwq4zTTWYBbXgYcWgN50slICFFz1dyE3n4kjPmQN9ovZIblfrDmQWK0o6MSQohqU3MTuosrdB5Lg7B2ROY2NetO/OHYmIQQohrV3IRu066hH8cJJNfDXxK6EKJGq/EJvU0DX1yU4njttnBip6PDEUKIalPjE7qXuyvNAr3ZQwtI3At55x0dkhBCVIsan9ABwpv481NqA9AWOLXH0eEIIUS1uCYS+iODWhJlaWYWpB5dCFFDXRMJPSzQm9v6dSdJ1yH5wO+ODkcIIarFNZHQAf48uBUxLi3JiN2GxaodHY4QQlS5ayahe3u6EdTmOkLz4/l47W5HhyOEEFXumknoAK3D++OqND+v/Zkf95xydDhCCFGl7EroSqmhSqkYpdQhpdRzpex3u1JKK6Uiqi7EqqMadwdguP9xnvg6ij3Hzzk4IiGEqDplJnSllCvwLjAMaA+MV0q1L2Y/X+AxYEtVB1llfIKhXgsmNEjAv7Y7Uz6LJCUjx9FRCSFElbCnhN4TOKS1PqK1zgUWACOL2e8V4F/A1T2kYdM+eB7fwtx7u5GalcsTC3dilZekQogawJ6E3hiIL7ScYFtXQCnVDQjVWi8v7URKqalKqUilVGRSUlK5g60SYf0g+ywd3Y4z47b2/HYgiffWHXJMLEIIUYUq/VJUKeUC/Bd4qqx9tdZztdYRWuuIoKCgyl66Ypr2Mf+N28SEnk0Y0aUR//35ANtiUx0TjxBCVBF7EvpxILTQcoht3QW+QEdgnVIqFrgOWHq1vhilbhOoEwpxG1FK8Y8xnahb24MvtxxzdGRCCFEp9iT0bUArpVQzpZQHMA5YemGj1vqc1jpQax2mtQ4DfgdGaK0jqyXiqtC0D8RtAq3x8XSjX8tA1h9MRmupSxdCOK8yE7rWOh94BFgF7AMWaq2jlVKzlFIjqjvAatG0L2QmQsphAPq3CiQ5I4f9p9IdHJgQQlScmz07aa1XACsuWfdyCfsOrHxY1axpX/PfuA0Q2JL+rUx9/oaDybRr6OfAwIQQouKuqZ6iBQJagHcwHPwZdnxBg5UPMLbeYX476KCWN0IIUQXsKqHXOEqZevS9S2D/MlAuPOF1gIFHW5KdZ8HL3dXREQohRLldmwkdoP9TENQGWg+FkztpuOxxwq3RbI/rSd+WgY6OTgghyu3arHIBaNgZBr0AjbtBl3HoWgFMcVvJbweTOJ2WzUtLdvN91PGyzyOEEFeJazehF+ZeC9XjAQa77GBb5FbGzl5Ct+3PsXPxv4iKP+vo6IQQwi6S0C/oMQWt3Hg6dw7L3J5hjOsGXnb9jFWfvcbZrFxHRyeEEGWShH6Bb31U5zvp7boX3+Cm8NBGzoUM4um8Ocyf97YM4CWEuOpJQi/E5aZXYOR7MGUNNOhInYlfkuTfhSmJ/2D5ssWODk8IIUolCb0w70AIvxvcPMyyR22Cpy7hjEcD+mx/gpj90SUfqzWc+AMs+VcmViGEuIQk9DKo2v7UmvgNHiof14V3s/vIcZ5auJOOM1bx+5EUs1P6aVgwAeYOhO3zHBqvEOLaJQndDnVC2xM/5D2aWWKJ/uRhVuw+iauL4t8/7kcf/gXe6wWH14KHDxzb7OhwhRDXKEnodmrffzRHw+7iDvdNbH68G0/f3IYdx86Q9f1TUKse/Gk9tBgMx7c7OlQhxDVKEno5tLzpT7jpXOoeXcbYiFBu9juGd9phdL/HIag1hETAmVjITHZ0qEKIa5Ak9PJoFA5BbWHnAjzcXHg2eAsZ2ov17v3N9sa2OT2klC6EcABJ6OWhFHQZD/Fb4EQUzU7/xFq3fry2NoF8ixUadgHlIgldCOEQktDLq/NdJmkvuh+Vl0W9/lPYezKNTzYeBU8fCG4PCVfvZE1CiJpLEnp5+TWE5oMg9TAEtaPvgJu5oV19/vvzAWKTM9GNu5NzbBt/W7qHPIu19HOlHIav74VzCVcmdiFEjSYJvSK6TjD/7TYR5eLC30d1xN3FhacX7eTjo/XwzEvjl82/89J3e9Bxm+HkTrAWk9x//RfsWwpLHzUdk4QQohKu3fHQK6P9KMjPho63A9CgjhfPD2/HC9/tJse9EVNc4ekOGSzdsRi15w1zjHcQtLsNhr4Gbp6mVL5nMQS0NG3Yd3wO3Sc57JaEEM5PEnpFuLpB+D1FVo3rEYqHmws9mwyAD//G8PzV3Oi5lT8sLdkRfDs98rfTOfITtKsnathrsGWOKZXfsxi+fwRWvWjasdcNLf3aJ3fC0d+gz6PVeINCCGckVS5VxMVFcUf3EJoE+UKjcFTsetx9g/ih/Wy+zuvLPWenMi//ZtSW92HXNxD5KXQYBf5hMPId0FZYPAVysy6e1GqFvOyLy/k5sOgB+OklSD54he9QCHG1syuhK6WGKqVilFKHlFLPFbP9IaXUbqVUlFJqg1KqfdWH6kTC+oOnH+ruhbw8bhA/PXE9O2fcRHz3Z4m2NoVvp0BuOvR+xOzvHwYj3zbNIRdMMEn81G6Y0w/e7mY6KwFsfhdSbIk8esnF6+39Hl5vKS9XhbjGKV3GyzillCtwALgRSAC2AeO11nsL7eOntU6z/T4CeFhrPbS080ZEROjIyBravM9qgdwM8KpTZLXFqpn5yRKei/8T6QFdCH7kJ1xc1MUd/pgP3z8MDTpB4n6oXc+Uymv5w+0fwWe3mWqZzGRz/mkbTbXN3IFwMgrC7zWlfSFEjaWU2q61jihumz0l9J7AIa31Ea11LrAAGFl4hwvJ3MYbuLabbLi4XpbMAVxdFC9OGsEL9T/gphNTGPP+JrbHpVLwoRp+N9z6himdtx0O0zabOvaMRPj4RpO8h/4TOoyG03sg6YBp834yCuo2haj5Zp0Q4ppkT0JvDMQXWk6wrStCKfVnpdRh4N/A9OJOpJSaqpSKVEpFJiUlVSRep+fl7sr/HhrFy3f25cTZ89z+/mb6vraWpxbu5KfoU1i7TYZnY+HOz8A7wIwPM/5LcPOCwS9C3SbQfgSgYO8S2DoXPP1g0lJwrw1rX3HsDQohHMaeKpc7gKFa6ym25XuBXlrrR0rYfwJws9a61DZ4NbrKxU6ZOfl898dxNh1OZvPhFM5k5dEq2IdpA1swsmtjXAtXx+Rlg7vXxeVPhkLaCfPT4wEY9i9Y9xqs+yc8uBYad7c/kIxEM2KkqzR6EuJqV9kql+NA4bZ0IbZ1JVkAjLI7umuYt6cb91zXlPfu7s62F2/gzXFdcVGKJxfuZPK8rSRn5ACQZ7GyNSGL7DzLxYM7jIazcWDNgx5TzLref4bagbD4QTgTZ18Q547DG53g4xvsr645fwZ+ngE5GeW4WyFEdbMnoW8DWimlmimlPIBxwNLCOyilWhVavAWQNnXl5ObqwsiujVn5WH/+MboTW4+mMvzN9cxcGk2f19Yy9oPN/O2HQlPgtbNVu7QYDIG2P7+nL4z7ErKS4eOb4HQpU+ZdsONz00nqTCx80B/WvAKb3oGNb5V8/Ma3YOMbpmOUEOKqUWZC11rnA48Aq4B9wEKtdbRSapatRQvAI0qpaKVUFPAkIF0eK8jFRTGhVxOW/LkvPp5ufPF7HF1D63JLp4Z8tTWeyNhUs6NfQ7jjExg+u+gJmvSC+340I0POG1Z6e3VLPvzxhflQePh3aDYA1s+Gn16En/8Kc/qbDk+FS+LZabDtY/P7vh+q9uaFEJVSZh16dZE69LLlW6ycz7Pg6+VOZk4+N/3vN7w9XVn2aH883Mr4LD4TC3MHmZeoU1aDq/vl+8SshK/GwdgvbC9agfNnzWiSeedh3T9g+6dQJxQmfg8BLWDjm/Dzy9DsejPd3tOHim3RI4SoHpWtQxcO4ubqgq+XScTenm7MGtmBA6czmPlDNK+t3M+4uZtZvL2EzkT+YXDbm6ZJ47rXit9n+6fgUx/aDLu4rlZd8PID3/rm+PtXQV4WfD4SUo/A5vdMMh/0Alhy4eDPVXjHQojKkITuRIa0q8+wjg34cssxPt5whKPJmTy7eBdbj5pqGK01q6JPEX3inDmg/Qgz5syG/5rxXwo7lwAHfzLbiyu9X9DkOrjnW8g+B3MGQMYp6PcEhPQ0HwZS7SLEVUOqXJxMZk4++0+l076hH3lWK6Pe2ci583l88UAv3v3lEMt3nyTI15PVT1xPndrukJNuhhA4EwuBbaBZfzNezMkoSNwHj+0E/6ZlXzhuE3wx2kzBN3WdqaP/4XHYtRCeOQzutUo/Pv20uWbrmyv9NxDiWiZVLjWIt6cb3Zv6U8vDFT8vd+ZOjCA338rwt9azKvoUk/uEkZqZy6srbCMzePrC/T/Bzf+EOo0h6ks4ss7Uew/9p33JHKBpH5i2CSYsNMkczHDAeZnmfKU5E2uaRX451tTBCyGqhfQkcXItg3149+5uzPn1MM8MbUvX0LrU8nDl/XWHua1LI/q3CjL14b0fNj+VEdCi6HJYf/CsY+riw/qZD49LJR8yY9DkZUGLIeaFau2Ay4YfFkJUnlS51EDZeRaGv7menHwrc+7pTqeQamyFsuYV09TRqy70nGpa1VhyIf2kGWcmYZsZtmDi96a9/Jdj4eh6GDe/6MtYIYRdSqtykYReQ22PS2Xix1vJzLXQJbQuT97YmutbB1X5dbTW6ITtuGz4D8SsuLhBuUL99mYIgt6PQmBLsz4nAz69xbSPv/9HaNjZDDq2e5EZVbLVDVUTWOpR2PA/0xrHt0HVnFOIq4Ak9GtUWnYe325P4NNNsZw8l82yR/vRqn7RapFDiRmsi0mkQ6M6XNe8HkqpEs52uYOn05m+IAr/2u7Mn9ILlZFoSueu7qaOvqQXpemnTBt55WIGFVv3T9j9jdk2ZIZpRVNWHPk58OVdUCfEjCsf3Lbo9oWTzOBlgW1g8jLwCbb7voS4mklCv8Ylpmcz7I31BPt5seTPffBwdWHpzhPM/e0I0ScujnzcrqEfD13fnJFdLxtMswirVfN1ZDx/+yEaqxVyLVa+nnodvZoH2B/UyZ1mgLH8HEDDwOchKQb2LDJztrrXNtU1tfzhpldM88nCtn8GP0wHF3cznk2722DMh+ZDJHE/vHedqdI5ss4MLXzPIvBrfPkHxdl403pnwF+gyzj74xfCQSShC9bsO80Dn0UyvmcoKRm5/LT3NG0b+HJnRCg3tqvP5iPJzNsYy/5T6Tw7tC3TBhZ9Aaq15mhyJj/sPMk32+NJOHOefi0D+eeYTox6dyNdQuvyyeQe5QsqZiX88g+TsJsPNFUvv9pGjawdACE94NQuSDsOnceZESVr1TUTiLzb0yT9e7+DLR/Ab/+GLuNh1Pvw7YOwfwU8sceMLf/lWDNejbs3BDQ3+zToZGJY9ID5EHH1hAd+gkZdq+LPLUS1kYQuAPjrkj188XscHm4uPH1TG+7v16zIEL0Wq+aJr6NYuvMEf721PaO6NuLH6FOs2ZfIH8fOcCYrD4C+LQO4q0cTbu3UEBcXxVtrDvLfnw+w6vEBtGlQTEuX8srNNMlaKfP7+v+Y5o5N+5hOTjErYeG9cMc86DjGHLPuX2aogl7TYOsHZhLtG2eZbaf2wNFfzciSexaZc//pVzO65Mc3QMQDcGAVuLjA1F/NTFFCXKUkoQsAzuda+Gj9EYZ1akDL4OITb77FyqNf/cHKPadwdVFYrJqwgNr0bFaP8Cb+9GsZSGi92kWOOZOZS5/X1jK8U0P+M7ZL9QQf9SUsmQbd7zOl9qxUeHS7mR0KzITaX98DMcvBrRY8vht8inkJHLfZvJRtO9zU5Z+NN+dJ2m+qgML6wV3/B54+1XMfQlRSaQld2qFfQ2p5uPLokFal7uPm6sKb48JpuHI/Xu4u3Nq5Ee0a+pb6stTf24O7eoQyf0scjw1pRZOA2iXuW2FdJ5g69o1vmOVb/nsxmYMpXY+eA/PvNL1Ri0vmAE17ww0zzWiSACPfNck7JAJuewOWPgofDTFJPfCSv9XvcyBhK7i4gV8jGPAMeFTDvQpRQVJCF1UiPjWLoW/8Ri0PN96dEE6v5gFYrZq41Cwa1vHCy9212OPyLFbcXe3ssGy1wqL74MQf8OctZQ83UBKtTWk/7Tjc+735MLjgyDpYdD/k58Kdn15sRrnnW3Ntv8YmoZ+Ng8F/NS9Ta7L00+bv3abUOd/FFSRVLuKKOHA6nYe+2E5cahaD2gQTFX+W5Iwc6tlK8Hf3akKIvynRZudZmLVsL9/uSOC9u7sxuG19+y6itWka6eZZ+YC1Lr555LkEM6xw8kG4exHUaw7v94GAlqbtvKs7fDUBYtfD9Cgz96uzslrgj/+DVjeZMfYL0xo+H2EGdrv/JzPWvnA4SejiiknPzuOF7/aw9WgKvZoF0CPMnw2Hkvl572kABrcNZmTXxny4/gi7Es7RsI4XKZm5fDKpB/1aBTo4+kIyU+DT4aaOPai1qe55aMPF4Q8S98P7vaHXQ2ZMnKpitUDsBlOX71L8t5oqdeAn+PJO8A6C2z+G5tdf3LZ/BSwYbzqJhfaC+1aU3D/AaoFDa8zxlfmwzc+BY7bJVsrRJ+JaIoNziSvG18udt8eHs+WFG3hrfDj39g7jg3sjWP/sYB4e2JKo+HM8+tUfHE3KZO693VkxvT/NA72Z8vk2thxJcXT4F3kHmOEKfIJNlcPQfxYdyya4rRmPZuuHZvCxwnKzzLyrpbnQO3bls2DJu7h+x+emVLzkYZMkq1vMCvDwMZOEfzHKtBay5Jkqp59egsDWcPM/4NgmOLS65PNEfmI+GD4cbD7sKmrjm+b+D/5U9r4ph82sW6KAlNDFFZVnsbL5cAotgn1oXNfUgSdn5HDXB5tJSs/h24f7lNgCxyHSTkLcRuh4++UlxrQT8Fa4mbB79JyL67++F05EwSPbwN3r4vrsNJPoz8Wb9vdxG836u/7PdIwC+GSYGWY4Lwu6TICR7xRfUs87D1+NN6NldpsIjboVje9MrEl4LQaXUqq2wv/aQ2hPGPkeLHsCdi+E4A6miei2D83omi0Gwzs9TOL/029F3zmASapvh5u2/OfPQG4GDH0NIu4r/ronokz11aUtifKy4Y2OkJkEDbuYJqTFxa61GeN/zSxoHAFj5l4+cFx1On/GvGtpPazo871CpMpFXPXiU7MY/d4manm48N3DfQn0Kftr+/lcC17uLsW2wNFak5adT51apUzeURVWPmcS3+N7TB302Xh4szNoKwx7HXpNNfv98k/4tdDMUbXqwZC/wq+vQ4OOcPc3cPYYvNEJBr9kku26f5heri6ugIKxn13sEBX9HXwz+WJP2XrNIbi9makqfqtpjQOm92znscXHfnwHfDgIRn9wsZfsvmWw4mlIP2E6e927xCTV3Ytg8QMQcT90GGM+BC5Urez6Br6dAuO+MmP3LHkIDq+F6x6Gm/5etGnpL383/QoC25gJzS+M8QOw4wtY+ojpILbzq6IfdBdobUbs3PSWGb3z+HbzTqXng+YDRVuh052XDwVRGks+uJajwd93D5n4fBpA38fMB1dFX9BXgCR04RR2xp/lrrmbaRXsy8A2QWTmWGgR7M24Hk0KOkAdSkznm8gENhxKZu/JNJ65+fJerXkWKy9+t5vFO47z8aQIBra5OI5LvsWKm72tauyRegTe6mZauwx+yTb65H9Mcs1KgceiTMemT24yJbq2w83IlGF9zbAGa20J7vHdZrKQNX+zTToSZqpzjqwzo1XGrIT2I2H0++a6C+42o1k+vNkk98NrzUvcM0choBV0vtMck7gPpm00o2Beau2rZqTMpw8X7UyVnWaGRO4w6uJxF1oY7fsBtMXcw+g50HoovN/XrJu22ZTerRYzufiW9832bpNM1dWmt2Dv9+Y+YjeYRHrHx9DqRpOo3+ttkv/UX83QDS5uJnZrvvk2E7fZdBCL3wI9HoRh/zYzaH3/CBxeYwtegYe3mUC9tMlUEvfBrq/N3+3kLjOAW1Bb8yF23bSS3wMkHYD3epkPmqxU82I8qC3c+Zn5EMlIhLWvmGdRO8A0b414oHwfMGWQhC6cxqroUzy+IIrsfAu13F3JyrUQ0dSfv43swNKdJ/h4/VGUgm5N/EnLzufkufNsfHYw3p6mhJWZk8/D83fw64Ek6nl7oIAVj/Wnvp8XX209xoyl0fh5udO2gS+3dm7IuJ7FJLry+mq8KRVP/wPe7maqAfo8YjowDZlhSnO5WSb5evkVPfZMLLzZxYxls/d7M6b8A8XUH//wuDnPU/sBBbNbm9LysEvmiy3ccudMLLzfz4xoOekH09Ty3HFo2tck3vf7mXjuW4Hdss9B7EYzRMPJKGg3AvYthVFzoOv4ovtu/dC8I9AX3gUoM8xD70fMt5EFd8Pp3abKqNn15hvAhfNc+EbQfCAc/wNyzpnB3Bp0gk5jofefi1bH5NsGhUs/aQZtO70H+j9lvgm4ukOT3mZeAID4bWaOXEuOmUoxtCdknIbT0abTWkArGP5vU+KP32KSe88/mVL8N/eZXsWP7wLvQPNe4buHTI/m7pPhj/mmuiykh6maORtnXvT2nArXP1MlvZArndCVUkOBNwFX4COt9WuXbH8SmALkA0nA/VrruNLOKQldlCTfYi0okS+JOs6M76NJyzYvv8ZGhPDs0LYE+HiyPe4Mt7+/iZduaceU/s3JzrMwbu7v7D5+jr+P6khEU39ue2cD4aH+9GsVyOurYriueT1C/GsTFX+WQ4kZfDgxghvb29lksiRHfjUv8lrdZF7m3b3IlDo/u+3iXK73fAsthxR//OejTPVHzjkYPttUH1zq5E74YAAM/Zepe/7+zzBljekQVZoLPWy96phkDND2VhjyshkP56a/m2ESyivvvKlz3/kV+IWYbyLFzU2bmWySd2aSmYO28Fg5uVmmGmrTOybp+zQw31TcPMw3gg8HmWGQ291qSvVN+xQ/icqlcjPh26mwf9nFdZ5+5l4bhcNnt5oqr/tWmBJ0YQdXw/InTSIurPkgGPC0afnU/y+muuyC9FNmTKC4DaZ1zi3/vdgpLTPZfAvb/imgzd8qqLWpjmp1Y9n3UoxKJXSllCtwALgRSAC2AeO11nsL7TMI2KK1zlJKTQMGaq3vKu28ktCFvU6nZfPR+iPc3KEBEWFFSzjj5m7maHImvz0ziOcW72ZJ1HHev7sbQzuaNtULt8XzzOJdANzWpRH/ubMLHm4uZOdZuP39TcSnZrF8ev/LhjMoF61NO/XEvabOe3qUKQEf+x0+uRnC7zUvN0tSUB/uBk8dKLld+9xBJpH6NjBVK9Ojym7apzWsnmle4DbtDefPmpeJnn7mA+TRHRV/oag17FkMdUIr10b91G4TU4fRpkfwBfm55v5Km8S8tNjSjpsXredTYfXfTMJ1cTMfLPetLHn6xdwsc18+waakHbPCfEOy5pu/2+M7TXVZYZZ8SIyGBp2Lfyan9sCBHyH5gGkCO+Avl78fsFNpCd1MUFDKD9AbWFVo+Xng+VL2Dwc2lnXe7t27ayEq69eYRN302WX6rg826abPLtNvrj5QZLvVatWvrdyn//3jPp1vsRbZFpucoTu+/KMe8fZ6nZ2XX7lAIj/Veoaf1hveKLr+1B6t83NLPzYvW+t/t9D6y3H2XWOGn9arZ1U81n3Ltf57A63f6Vnxczgbi0XrrR9p/fFQrZMOlL3/pY6u13p2G603v1f1sZUTEKlLyKv2lNDvAIZqrafYlu8FemmtHylh/3eAU1rrvxezbSowFaBJkybd4+JKrZURokxaa0a8s5Hdx89xU/v6zLmnOy4uZZRaC/lxz0ke+r8ddA2ty5vjutI0wLtigVjyYMdnpqlhRcZ3ORNrSn+l1bHmZMB/2kJuOjz8OwS3q1isYKoylDIvX4V9SupZfIVdsY5FSql7gAjg9eK2a63naq0jtNYRQUFVPx2auPYopZg5oj1jwhvz37u6liuZAwzt2JB3JoRzOCmD4W+uZ8kfxysWiKs79JhS8cG6/MPKfmHm6WMm+m55Q+WSOUC9ZpLMy+sqSOZlsafx5XEgtNByiG1dEUqpG4AXgeu11jlVE54QZevetB7dm1a89cCtnRsR3sSfxxf8wVPf7KRjY7+rq3NTYYNecHQE4ipmTwl9G9BKKdVMKeUBjAOWFt5BKRUOfACM0FonVn2YQlSvxnVrMeee7tRyd+W1lTGODkeICikzoWut84FHgFXAPmCh1jpaKTVLKTXCttvrgA/wjVIqSim1tITTCXHVCvDxZNrAFqzed/rqGldGCDtJxyIhCjmfa2HQ7HXUr+PFkof7lDqxhxCOIKMtCmGnWh6uPHlTa3bGn+Wj9UdxVIFHiIqQhC7EJW7vFsKgNkG8umIfD36+neSMi+/4kzNy+PVAEit2nyTfYnVglEJcTqpchCiG1aqZtymWf/24H6tVF0yTdz7v4hjlU/o146Vb2zsqRHGNkkmihSgnFxfFA/2a0b9VIN/9cZx8ixWtoUEdLzo0qsOK3Sf5aMNR2jb0447uIUWOTc3M5WhyBp5urvh4utE0oLbUxYsrQhK6EKVoXd+XZ4dePvRpRJg/h5MyeOHb3SigsX8tcvKtfLsjgZW7T5FbqDpmQq8mvDqqoyR1Ue0koQtRAe6uLrw7oRuj3tvIU9/sLFjv6+XGhF5NuL51EHkWKxsOJfP55jh8PN14fljbIkn9WEoW22JTualDfXy9qnkiDnFNkIQuRAX5e3uwYnp/DiZmkJWTT55V0yPMn9oeF/+3ujA079zfjqC15u5eTWlSrzafb47lXz/GcD7PQt3l7vxpQAtubF8fL3cXPN1ccXNRuLgofDzdCoYSFqIs8lJUiGpmtWqeWbyLRdsTAFOKT8/O5/rWQdzXN4zPNsXyS0xSscd6e7jSOaQu3Zv680C/Zvh7e5R6rbiUTDYeSmFbbCqJ6dnc1aMJt3RqeNmHQmJaNhataVjnyk2dJqqGzFgkhINprTmclMnmIyn8cewMfVoEcnu3xgVVMLsTznEkOYOcPCs5+RbyrRqLVROfmsUf8WeJPpFGqH8t5t3Xk7CA2izecZzXV+2nvp8XA1sHUc/bgyVRJ4iKPwtAoI8H3p5uxKVk0TSgNi8Ob8dNHRoAF+dvtVitLJ7Wh+ZBPgUxZudZqeVRzKTUdsrNt3LgdDr1vD1oVFc+LKqDJHQhnNz2uFQe/Hw7Vq3p1LgO6w8m0zW0Lm4uih3HzmDV0Ka+L7d3b8wN7erTLNAbreGnvad5c81B9p1M47lhbRnXI5Tb399EUnoO7q4ueHu68e3DfUhMy+Ev3+xk78k0mtSrTfuGfkzp3+yyCUVK8sexM/zth73sPZFGrsVKoI8nqx7vT4Adk32L8pGELkQNEJeSyX2fbuPE2fM8O7Qtk3qH4eKiOJuVS0pmLs0DvYttSZOdZ+Ev3+xk2a6TBPl6ci4rj88f6ImXuyvj5m4m2NeLE2fPU7e2B3f1CCE2OYutsamkZuby5I2teej6FkTFn2FV9Gka1fFidHgIdWpffIm7cvdJHv86ikAfT27t0pBQ/9rM+mEvA1oH8eHE7iilyMrNJyM7n2A/ryv5J6uRJKELUUNk51nIzMkvd8nXatXM/imGub8d4T9juzCya2MAft57mofnb+eWTg2ZOaIDdWubOvr07Dxe+G4PP+w8gZ+XG2nZ+bi5KPKtGk83Fwa3DaZOLXfO51n4PuoE3ZrU5cOJEQVxfbzhKK8s28vfR3XE18uNV5fvIy07j7fGhRdU/ZTl623H8K/tYff+1wpJ6EIIwHwgeLm7lrkOTJ36N5EJ/HogiUFtg7m5Q33iUrL4ausx1sUkkWexYtWagW2C+fuojkXOYbVqJs3byvqDyQB0DqmDAnYdP8dLt7RncNtgDidmkJadR8fGdWgR5FPkxe176w7x7x9jcHVRfDQpgkFtggFITM8mNjmLhnW8qO/nhYdb2aOXbDmSQszpdM5m5QEwrmcowb7Ff1PIt1hxc626EVHOZOaW+SK7vCShCyGuuMS0bJ5etIubOtRnXI8m5OZbeeLrKH6MPnXZvj6ebvRtGcBtXRpx/Mx5/rlyP7d2bsjR5ExikzP5aup1bDmSyv9WHyAr1wy/4KKga2hdBrcNpkOjOmTm5pOZk0/jurXpFFKHlIwcXlm297IWRD6ebjwyuCX39Q3D0818CGXm5PPg55FExp2hW5O69GkRyA3t6tOuoW+RaiyLVfPR+iN8vS2e9Jx8svMsjOsRygvD2xXZL89i5bWV+/l4w1HG9wxl5ogOBdeqLEnoQoirgtWq+e6P41i0pmWwDz6ebuxOOMeOY2f4ee9pEtPNQGjDOjbg7fHhpGTmMvrdjZxMy0ZrGNI2mLuva0Jyei5xqZlsOJjMzoRzxV7L1UVRy92V6UNaMjo8hLq13Uk4c55Xl+9l9b5Emgd68/fRHQkP9WfyvK1Exp3h9m6NiT6Rxt6TaWgNzQK9uaFdMO0b+VHfz4vZq2LYcews1zWvR7NAb5LSc1i9L5FZIzswsXcYAKfTsvnz/B1Exp3huub1+P1IKt2b+vPkja3ZmXCW34+k8kC/ZlzfumLTcEpCF0Jc9SxWzdajqcScSmNCr6YF1Skxp9J5dcU+JvQM5eYODS578ZuUnsOx1Ex8vdyp5e5KbEomuxLOkZGTz319w4qtXvn1QBJ/XbKHY6lZhPjX4sTZ8/zvrq4F7xaSM3JYFX2KFbtPsvVoKnkWkyfr1HJn1sgOjOjSCKUUVqtm6heR/BKTxJx7urPn+Dk+Wn8EDbx2e2dGdGnEsl0nePqbXQUDu7UK9uGpm1oztGPDCv2dJKELIcQlzudaeHvtQT7fHMfMER0uG2TtgjyLlaPJmRxJyqBbU//LPiDSs/MY894mDiZmADC8UwOevrktzQK9C/aJTc4k5nQ63Zv6E1jJppyS0IUQogRa60oPnHYsJYsP1x9hbEQonULqVFFkxZPhc4UQogRVMQpmk4DavDKqYxVEUzkyY5EQQtQQktCFEKKGsCuhK6WGKqVilFKHlFLPFbN9gFJqh1IqXyl1R9WHKYQQoixlJnSllCvwLjAMaA+MV0pdOpHiMWAy8GVVByiEEMI+9rwU7Qkc0lofAVBKLQBGAnsv7KC1jrVtk2nQhRDCQeypcmkMxBdaTrCtKzel1FSlVKRSKjIpqfgB/YUQQlTMFX0pqrWeq7WO0FpHBAVVrNurEEKI4tmT0I8DoYWWQ2zrhBBCXEXsqUPfBrRSSjXDJPJxwITKXnj79u3JSqm4ch4WCCRX9tpXCbmXq5Pcy9WrJt1PZe6laUkb7Or6r5QaDrwBuAKfaK1fVUrNAiK11kuVUj2A7wB/IBs4pbXuUMFgS4sjsqQur85G7uXqJPdy9apJ91Nd92JX13+t9QpgxSXrXi70+zZMVYwQQggHkZ6iQghRQzhbQp/r6ACqkNzL1Unu5epVk+6nWu7FYcPnCiGEqFrOVkIXQghRAknoQghRQzhFQi9rtMermVIqVCn1i1Jqr1IqWin1mG19PaXUz0qpg7b/+js6VnsppVyVUn8opZbZlpsppbbYns/XSikPR8doL6VUXaXUIqXUfqXUPqVUb2d9NkqpJ2z/xvYopb5SSnk5y7NRSn2ilEpUSu0ptK7Y56CMt2z3tEsp1c1xkV+uhHt53fZvbJdS6julVN1C25633UuMUurmylz7qk/odo72eDXLB57SWrcHrgP+bIv/OWCN1roVsMa27CweA/YVWv4X8D+tdUvgDPCAQ6KqmDeBH7XWbYEumPtyumejlGoMTAcitNYdMX1GxuE8z+ZTYOgl60p6DsOAVrafqcD7VyhGe33K5ffyM9BRa90ZOAA8D2DLBeOADrZj3rPlvAq56hM6hUZ71FrnAhdGe3QKWuuTWusdtt/TMQmjMeYePrPt9hkwyiEBlpNSKgS4BfjItqyAwcAi2y7OdC91gAHAxwBa61yt9Vmc9Nlg+pXUUkq5AbWBkzjJs9Fa/wakXrK6pOcwEvhcG78DdZVSDa9IoHYo7l601j9prfNti79zsd/OSGCB1jpHa30UOITJeRXiDAm9ykZ7dDSlVBgQDmwB6mutT9o2nQLqOyqucnoDeAa4MFRyAHC20D9WZ3o+zYAkYJ6tCukjpZQ3TvhstNbHgdmYuQlOAueA7Tjvs4GSn4Oz54T7gZW236v0XpwhodcISikfYDHwuNY6rfA2bdqOXvXtR5VStwKJWuvtjo6lirgB3YD3tdbhQCaXVK840bPxx5T2mgGNAG8u/9rvtJzlOZRFKfUiphp2fnWc3xkSutOP9qiUcsck8/la629tq09f+Jpo+2+io+Irh77ACKVULKbqazCmDrqu7Ws+ONfzSQAStNZbbMuLMAneGZ/NDcBRrXWS1joP+BbzvJz12UDJz8Epc4JSajJwK3C3vtgBqErvxRkSesFoj7Y39OOApQ6OyW62OuaPgX1a6/8W2rQUmGT7fRLw/ZWOrby01s9rrUO01mGY57BWa3038AtwYS5Zp7gXAK31KSBeKdXGtmoIZiYup3s2mKqW65RStW3/5i7ci1M+G5uSnsNSYKKttct1wLlCVTNXJaXUUExV5QitdVahTUuBcUopT2VGtG0FbK3whbTWV/0PMBzzZvgw8KKj4yln7P0wXxV3AVG2n+GYuuc1wEFgNVDP0bGW874GAstsvze3/SM8BHwDeDo6vnLcR1cg0vZ8lmBGDHXKZwP8DdgP7AG+ADyd5dkAX2Hq/vMw35weKOk5AArT8u0wsBvTssfh91DGvRzC1JVfyAFzCu3/ou1eYoBhlbm2dP0XQogawhmqXIQQQthBEroQQtQQktCFEKKGkIQuhBA1hCR0IYSoISShCyFEDSEJXQghaoj/B/Ho0tmUDq5EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(1,len(total_train_loss)+1), total_train_loss,label = 'Train Loss')\n",
    "plt.plot(np.arange(1,len(total_val_loss)+1),total_val_loss, label= 'Validation loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6a9b0b57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8lklEQVR4nO3deVzVVf7H8dfhsoOgCIiKC5q477iNWS5tLmlmlmalLVpNU1MzNVNTU01TU8040/KrbMrKstLJSjPTTEuz0lTcBUFRUUFFBNm5cJfz++NckFVRQbj4eT4ePOR+7/d+v+d7v/i+557vOeertNYIIYRwfx71XQAhhBC1QwJdCCEaCQl0IYRoJCTQhRCikZBAF0KIRkICXQghGomzBrpS6n2l1Aml1O5qnldKqdeVUklKqZ1KqX61X0whhBBnU5Ma+jzgujM8Pxro5PqZBcy58GIJIYQ4V55nW0FrvU4p1f4Mq0wAPtJmhNKvSqmmSqmWWutjZ9puaGiobt/+TJsVQghR0ZYtW05qrcOqeu6sgV4DrYEjZR6nuJadMdDbt29PbGxsLexeCCEuHUqpQ9U9d1EviiqlZimlYpVSsenp6Rdz10II0ejVRqCnAm3KPI50LatEa/2O1jpGax0TFlblNwYhhBDnqTYCfSlwh6u3y2Ag+2zt50IIIWrfWdvQlVILgOFAqFIqBXgG8ALQWr8NLAfGAElAAXDn+RbGZrORkpKC1Wo9302Ii8DX15fIyEi8vLzquyhCiDJq0stl6lme18ADtVGYlJQUmjRpQvv27VFK1cYmRS3TWpORkUFKSgpRUVH1XRwhRBkNaqSo1WqlefPmEuYNmFKK5s2by7coIRqgBhXogIS5G5BzJETD1OACXQghGiurzcGLy/eQmlVYJ9uXQC8jIyODPn360KdPHyIiImjdunXp4+Li4jO+NjY2loceeuic97l9+3aUUnz77bfnW2whGo3dqdk8tWQXOVZbueV5RfZqX/O/zYd5cME2lmxLPeN6JbTW/Hogg1XxaTVaH+BUfjEzP4plztr9OJyVb9v5WewRrnt1HWsST5Qu+2JLCjHPr+LJxbs4ll1I4vFcJrzxC/9dd4A1CScqbaM2qPq6p2hMTIyuOFJ0z549dO3atV7KU9Gzzz5LYGAgjz76aOkyu92Op2dtDK497c9//jPr16+nQ4cOfPjhh7W67brUkM6VqH3FdicA3p5V1/msNge+XpazbsfmcOJlKb8Nh1PjoSo33e1OzWba3I1kF9qY0KcVr97SB6UUn2w8xFNLdvPn67pw35UdS9d3OjUvf5vAf9cdINDHk7wiOz6eHsS0b0b/ts2IaR/CwKiQcuXcnZrNiyv28EtSBgBeFkXfts24LDyQyGZ+2B2a3anZJGfkM6FPa2Zd0YE8q51pczeScDwHp4aYds2YPbk37UMDsDmcPL8sng83HCLA20KBzcFDIzuRY7XxwS/JdAoPJDkjH6UUCmji68m/JvdmROfwGp2HqiiltmitY6p6rnbTqRGaMWMGvr6+bNu2jaFDhzJlyhR+//vfY7Va8fPz44MPPqBz586sXbuW2bNns2zZMp599lkOHz7MgQMHOHz4MA8//HCVtXetNYsWLWLVqlUMGzYMq9WKr68vAC+//DIff/wxHh4ejB49mpdeeomkpCTuu+8+0tPTsVgsLFq0iI4dO1barhDnItdqo4nv6S6oySfzuent9ZzMK8bPy0KnFoG8eWs/2oT4o7XmxRUJvP/zQX4/qhP3D++Ip8WD5JP5rEk8wfDO4USFBlBsd/LmmiTmrN3P6J4RPDm2KyH+3sxbn8xrq/dR7HDSuqkf7Zr7MyAqhA6hATz+5S4CfTy5oU8rPtxwiCs6hdE80Junv4qjqZ8XL61IwNNDcc+wDhxIz+Pf3+3lm13HuGNIO54e140dKVks23mMTQczeWNNEk4Nvl4eDOlgOlokHs8lNauQZv5ePHN9NzpHNGHd3pNsOJDBt7uPk5lvvoV3CA0gJMCbf61M5OsdR/FQiqT0PD64cyCn8ov561e7GT57Lb5eHvh6WcgqsDFzWBS/vyqaZ76K47Xv9wFw59D2/GVMV45nW3lzTRK5RXaevb47YU186uxcNthA/9vXccQfzanVbXZrFcQz13c/59elpKSwfv16LBYLOTk5/PTTT3h6erJ69Wr+8pe/8MUXX1R6TUJCAmvWrCE3N5fOnTtz//33V+q3vX79eqKioujYsSPDhw/nm2++YdKkSaxYsYKvvvqKjRs34u/vT2ZmJgDTpk3j8ccfZ+LEiVitVpxO5/m9EaLRcDo1uUV2lIIg36rHBVhtDmKTT/HTvnQS03K5tnsEN/RpTXahjX9+m8CX21L549XRPDiqEw6n5tFFOyiyO/nD1dHkWm18FpvC5Lc38PE9A1m0JYV31h0gukUg/161l7V704kI9mXFrmM4NTy3LJ6RncNJzSok4XguQzo0Z8Wu4/yQcIKIIF/2ncjjyugwolsEcjTLyt60XNYkJgLQMtiXBTMH07qZHwnHc/nrV7vxUIroFk1YOHMwTyzeyfPf7OHzLSkkHM/F4qF4YnQXZl3RAaUU/duF0L9dCAD5RXY2JWfyY2I66/am42XxYED7ZtzZuj03D2hT+l79pmNo6fuU72p+CfAxsfhd3HH++tVuThXYeOf2/lwZbUa3D4wKYemOo2TmF3Mqv5gRXcIZ07MlALMn9+KK6FC8LR6Mdi1rE+LPS5N61fapr1KDDfSGZPLkyVgs5mtbdnY206dPZ9++fSilsNlsVb5m7Nix+Pj44OPjQ3h4OGlpaURGRpZbZ8GCBUyZMgWAKVOm8NFHHzFp0iRWr17NnXfeib+/PwAhISHk5uaSmprKxIkTAUpr8uLSo7Vm+a7jvPxtAimnCnBqUAq6tQzi8stC6RgWSLMAb6w2B9/Fp/HDnjTyix14WRQtgnxZm7iLl1YkUGx34nBqerdpyr9X7aV5oA95RTZiD53iPzf35sZ+5u/1pv5tuO29jYx9/WeK7E5uG9yWv0/owdIdR3lqyW72Hs/l3is7MqFPK1bsOs7Hvx7C06KYe0cMV3VrwYH0PJ5ZGsfBk/m8fVt/ru3eolxzy8m8InYcyaJnZDDhTczf9Su39GH0az/h6+XB+zNiCPb34rUpffGy7CDxeC6Pj+7CxL6taRFU9f+DAB9PRnQOP6emjZIgL3FN9wiGXhbKqYJiIpv5ly5v1dSvXNNPWUopJvRpXeN91rYGG+jnU5OuKwEBAaW///Wvf2XEiBEsXryY5ORkhg8fXuVrfHxOf62yWCzY7eUvvjgcDr744gu++uorXnjhhdIBO7m5uXVyDKLhsdoc7EzJpl/bpnhaKrdV51htxCZnsiYhnS2HTtEmxI9ekU3NssR0urcK4oERlxHs50VBsYNfkk7y/i8HsTlOXxcLCfBmfJ9WXNW1BYM7NMff28Lm5FPM//UQXh6Kh6+KpmVTX2Z+FMtTS3bh6eHBNd1aMLHv6VDqHNGERfcOYdb8WIZ0aM4z13cvDa5rukWg0fh7myjpEhHEQ6M6lWsj7xAWyPy7B6G1rrLLa2igD6O6tii3rFVTP5Y9eDk+Xh6lIe9l8eC1KX0v/I0/BwE+npWCviFzn5I2ENnZ2bRubf7Y582bd97b+f777+nVqxcrV64sXTZ9+nQWL17M1VdfzXPPPce0adNKm1xCQkKIjIxkyZIl3HDDDRQVFeFwOEpr8aJ+WG0ObA5nuTboquxNy2VnSjYT+7bG4qHIK7Jz5web2Jx8ilbBvtw6qC0hAT7sSs0i/mgOhzILyCow3/78vCz0bduUvWl5rIxLI8DbwtPjunHHkHblPggeGtUJq81Bem4RpwqKcTg1PVsHV/qwGBhlLhaW9da0fkybu5EjmYX848aelYK3fWgA3z1yZaXj8vOufGHU4lH1OIVzHb/QJkT+ts+VBPo5+tOf/sT06dN5/vnnGTt27HlvZ8GCBaXNJyUmTZrEnDlzWLFiBdu3bycmJgZvb2/GjBnDP/7xD+bPn8+9997L008/jZeXF4sWLaJDhw4XekjiPBXZHdz41nrij+XQITSAHq2DiWzmR4sgXzqFBzIwKgRPiwdfbU/lz1/sxGpzMv/XQzw9rivPf7OHXSnZPHJVNJuTM5n93V4Agv286N4qiLE9W9I2xJ9urYIY0P50T43sQhseimo/QHy9LLQJ8T/nMPT39mTRvUMosDmqbYsXDZ90WxTnRc4V/GtlAm+u2c/0Ie04lm0l7mgOaTlW7K5+yqGB3vRoHczaxHQGtg9hUv/WvPxtIpn5xXhZFG/c2o9ru0cAcCSzAKfWtA3xl5G44oyk26IQtWzHkSzmrN3P5P6R/G1Cj9LlTqcmI7+Y2ORMvt55lA37M7hraBRPjOmCl8WDq7q24M01+xneOYwrok/fE0CaF0RtkEAXogpaa3akZPPV9lSOZBbw0qRehAaaC91Wm4M/LtpBiyBf/np9t3Kv8/BQhDXxYXTPlqXd1spqHujD0xVeI0RtkUAXooLCYge3vLOBnSnZeFs8QMFtczeyYOZgNHDv/FiSTuTx0V0Dpb1ZNCgS6EJU8M66A+xMyeaZ67txY79IdqVkc9eHm5k2dyN5RXaO51h549a+5ZpMhGgIZHIuIco4mlXInB+TGNuzJXcOjSLYz4vLO4Xy39v6s+9ELgXFdhbOGsy4Xq3qu6hCVCI1dCHKeHFFAlrDE2O6lFs+oks4yx4cRrMAr9KBLkI0NFJDL2PEiBHlBvoAvPrqq9x///3Vvmb48OGUdL8cM2YMWVlZldZ59tlnmT179hn3vWTJEuLj40sfP/3006xevfocSn9mDz/8MK1bt5b5X1y2HMpkUewRiuyO0mWr4tP4esdR7r2yY7mh3iU6RzSRMBcNmtTQy5g6dSoLFy7k2muvLV22cOFC/vnPf9bo9cuXLz/vfS9ZsoRx48bRrZvpAfHcc8+d97YqcjqdLF68mDZt2vDjjz8yYsSIWtu2O9qblssd720iv9jBK6v2ctuQdvyYmM7Gg5lEhQZw35UyWEu4J6mhl3HTTTfxzTfflN7MIjk5maNHjzJs2DDuv/9+YmJi6N69O88880yVr2/fvj0nT54E4IUXXiA6OprLL7+cRNdscgDvvvsuAwYMoHfv3kyaNImCggLWr1/P0qVLeeyxx+jTpw/79+9nxowZfP7554CZJqBv37707NmTu+66i6KiotL9PfPMM/Tr14+ePXuSkJBQZbnWrl1L9+7duf/++1mwYEHp8rS0NCZOnEjv3r3p3bs369evB+Cjjz6iV69e9O7dm9tvv/0C39W6dzSrkPm/HuJETtX3Oc0qKCYjz7xn2QU2Zn0Ui5+3J69P7UtEsC///DaRQxkFPHN9N5Y/NKx0XhIh3E3D/ctd8Tgc31W724zoCaNfqvbpkJAQBg4cyIoVK5gwYQILFy7k5ptvRinFCy+8QEhICA6Hg1GjRrFz50569ap6SswtW7awcOFCtm/fjt1up1+/fvTv3x+AG2+8kZkzZwLw1FNP8d577/Hggw8yfvx4xo0bx0033VRuW1arlRkzZvD9998THR3NHXfcwZw5c3j44YcBCA0NZevWrbz11lvMnj2buXPnVirPggULmDp1KhMmTOAvf/kLNpsNLy8vHnroIa688koWL16Mw+EgLy+PuLg4nn/+edavX09oaGjp1L0N1cm8Im5991eSMwp4dmkcw6PDeGhUJ3q3aQrA8WwrN7z5CydyrQyMCqHY7iQ1q5CFswbTv10I1/dqyYGT+UQ288PH8+w3bBCiIZMaegUlzS5gmlumTp0KwGeffUa/fv3o27cvcXFx5dq7K/rpp5+YOHEi/v7+BAUFMX78+NLndu/ezbBhw+jZsyeffPIJcXFxZyxPYmIiUVFRREdHA2YCr3Xr1pU+f+ONNwLQv39/kpOTK72+uLiY5cuXc8MNNxAUFMSgQYNKrxP88MMPpdcHLBYLwcHB/PDDD0yePJnQUDNPdEhISKVtXmyn8ov5dONhkk/ml1tuJrjazPEcK2/e2o97r+jAjpRsbv7vBr7dfdw8P28zuVYbM4d14GReMVsPZ/H8DT1K581WStExLFDCXDQKDbeGfoaadF2aMGECjzzyCFu3bqWgoID+/ftz8OBBZs+ezebNm2nWrBkzZszAaq366/3ZzJgxgyVLltC7d2/mzZvH2rVrL6i8JdP0VjVFL8DKlSvJysqiZ8+eABQUFODn58e4ceMuaL91ZU3iCT7ecIiO4YEM6dicxOO55m4vVjveFg9mXhHFTf3bEJucyaebDhN/LId37+jPyC4tGNurJXdfHsU9H8Vy/ydbiA5vQlJ6Hu9Nj2F453CeGNOV7AIbwf4yGEg0TlJDryAwMJARI0Zw1113ldbOc3JyCAgIIDg4mLS0NFasWHHGbVxxxRUsWbKEwsJCcnNz+frrr0ufy83NpWXLlthsNj755JPS5U2aNKlyLvTOnTuTnJxMUlISAPPnz+fKKytPY1qdBQsWMHfuXJKTk0lOTubgwYOsWrWKgoICRo0axZw5cwAzP3t2djYjR45k0aJFZGSYey5erCaXE7lWfvfpVu78YDM7U7OZ90syd36wmZdWJDCgfQgLZg5mXK+WvLlmPyNmr+Wxz3dyJLOA/9zcm5FdTs+l3TzQh0/vGcxVXVuQmJbL3yf0YHiZmxzUaZgXnqq7bQtRAw23hl6Ppk6dysSJE0ubXnr37k3fvn3p0qULbdq0YejQoWd8fb9+/bjlllvo3bs34eHhDBgwoPS5v//97wwaNIiwsDAGDRpUGuJTpkxh5syZvP7666UXQ8HcmeiDDz5g8uTJ2O12BgwYwH333Vej4ygoKODbb7/l7bffLl0WEBDA5Zdfztdff81rr73GrFmzeO+997BYLMyZM4chQ4bw5JNPcuWVV2KxWOjbt+8FzfteUV6RnaeX7CY6ogmzhnXAw0MRfzSH6R9sIrvQxh+ujubeKzvgdMKWQ6fw97HQr20zAIZ0bM60wW3ZnZrDwKgQukQ0qXJmQj9vC2/f1p+UUwW0ax5Q6fk6kbAc/jcNrnkBhvz24uyzvqRugQ1vwTXPQ1Dl+Wrc3p6v4bunYMxs6HR1fZfmnMj0ueK8nM+5yiooZsYHm9l+JAuAEZ3DuHVQO/7w2XYCfTyZd+dAOkc0qYPS1jFrNrw5CPLSAAXTl0L7y89tG/kZ4O0PXn51UsRaU5AJbw+DnBRofhnM+AaaRNR3qSorLjDnIySq5q+xF8Oqp2Gj+dZKRC+4d525v19tKcqDrR9B13HQtO15beKCp89VSl0HvAZYgLla65cqPN8OeB8IAzKB27TWKedVWuGWnE7Nd/HH+c1loaUTVhUWO/j7N/EUFNlpG+LPyrg0Dp7M553b+5OWY+W5ZfGsSUynY1gA8+8eRKumDTzMqrPqaRMed3wFy/4Ai2aYIAiqwfQANiusfx1++jc07wQzvga/ZtWvn7oFDv9qgtVuhYEzoVn7mpXTYYOk76HDcPA6jwFSWsOS35pjHTMbVj8L88bBjGV1E+oJ30DcYsg5CsV5EHMX9L0dPKq5gG3NhrR42PUZ7PrcvOb616Gfq+vtpndhwxsQ0gFax0C3CRDhmvq4KBc+nQKHfoZB90FoJ/jmj7D/B7hsVNX7czph9TPgKIau10PbIdWXLf+k2f+m/5qmOaVgcPUDFs/XWWvoSikLsBe4GkgBNgNTtdbxZdZZBCzTWn+olBoJ3Km1PmMHZqmhu7eK5+qr7an8fuF2BrRvxvy7B+Hj6cFDC7ezbOdRWgX7cSy7kABvT96+vT9DLzM9aHYcyWLJ9lQeGtmJZgHe9XUoZ6Y1WLOqD9nkn2HeWBjyO7j2BTiRAHNHmdCY9jk0aVH16wBO7IGFt0LmAbjsKji4znStvX0JOO2w8zPISTUfDBZv2P4ppLr+zygPQEF4V7jn+9MBfSIBju+E3GPg5W9CsCRkfnge1v0LmkWZQG7RHbbNh32rTCANuhc8feDIJtj4NnQdD91vOF3eX16HVX+F616GwffBoQ3w8SQIbm1q6oE1vyHzWeUchdf6gG+Q+SZQnG+OK6InjP0PtBl4et3N78GPL7u+IQGefias847DgbVw9d/h1EGIfd8Eub0ITsSZ93DkX6H/DPj0ZkiJhRvmQO9bzDqv9YHmHc0HVlV+eAHW/RM8vMBpg4BwGPoQDLjHfNNy2ODQetj6oWnGcRRDl3Ew9GFoM6DqbdbAmWroNQn0IcCzWutrXY+fANBav1hmnTjgOq31EWUaNbO11kFn2m51gd6lSxe5Y0sDp7UmISGhNNDtDifXvLKO3CI7J/OKuKZbC3pFNuVfKxP503Wd+e3wyyi2O3FqXXorNbegNSx7BLZ9DFM+hehryj+fEgsLpoB3ANy/wTSZAOxbDZ/dDn4hcOtCE0IVOewwdyRkp8KN75haYMJy87rgNiaQ7VYT5A4z0I3ml8HAe6HHjWbbSavh08kw+Ldw3Yuw5UNTXn16OgOGPgxX/80E/duXQ/uhZp8Z+0ygaaf5ZpCxD5q2M/vY/z14eJoPlX7TYcDdsOYfsPdbE0i3fHy6GeLQehPqTdvC9GXmw2P3F3BsBxTlmOMc8tvyTVD2YrB4nd5GYRbsXQmRMSZAwdSOt8yDB7eYbyBaQ9yXsOoZyE83H5ZRw8x7/clNpnYcfY05lvaXg19TE8pf3AN7lpptXv6ICXAPi/mGs+xhiP8KvJuAvRAmvVf+A2z9G/Ddk+YDM7JCfu7+Ej6/E/rcBqNfNudiyzw4sAaatISwzuaD0VYAvsHQ+1bz4RoWffa/u7O40EC/CRPW97ge3w4M0lr/rsw6nwIbtdavKaVuBL4AQrXWGdVtt6pAP3jwIE2aNKF58+YS6g2U1pqMjAxyc3OJijLtk19uTeEPn+3g7dv6cTTLNKUAjO/ditem9HHfc7npXVj+KPg2NeF625cmEAHilsDie01Tw62LKv9HPbbDfIW3Zru+2vc0r23Z2zy/4U1Y+Re46QMT0CXiFsPKJyH6OhMALbqb8Ck8ZWr9HhU6pi3/k/ka32OSCdLLrjIXZoNamaagLR+YfWx6B9IT4Hex4NMENs812+w91YRo0vfw3V8h9yj85iGz719ehZ9fBbQJvSseNc0Enj7ly3DwJ/hksgnRggzzARQQbh5bs01zw1XPQp9p8Msr5n319IVWfU1NNmm1eU1gBNz9nQn61/tB39vg+lfL7ys/A+aNgewUGPcqLP8jBLeFu1eaD9aKHHbTnBXWuXxYg/mQ2PYx/Pwfc4G3S4V7BBflwSvdzXsZ3Mbs09MbAsLMMbfsBdO/Lv9+JP8Ma18y7227oebDpdPVtXpt5GIEeivgDSAKWAdMAnporbMqbGsWMAugbdu2/Q8dOlRuXzabjZSUlPPu4y1qn83hxGpz4nBqHE6Nr5cHzYMDiYyMxMvLC7vDyVX/+RE/b0++efByPDwU//f9PrYfyeKNW/tVeVd4t3BwHcyfaAJy/P+ZZpWcYyYUUmIhfQ9EDoSpCyAgtOpt5ByDFX+CIxtPNwfE3A0DZ8G7I03A3/rZhV10s1nh3RFwIt7UFq9/1dR+wdRQ540zzTTaCRPeNCF5Lg78aJotBt9/5iaVA2vN6O4OV5rgbukaRW3Nga9+a5ocPDxNOXrcZALu6FZTO+8yDtoNgaUPmg+CFt0hcQU8tM0051SUcww+GG2aUfxCYNZaaNbu3I6rpta/AT/NhuBIE+oOm/mG4BtkavS12cxUQ3Xe5FJh/UAgQWsdeabtVlVDFxeP1ebA00Phaak8FOFYdiHzfklm2c5jpGYVAhDgbcHXy0JukZ1vfz+MDmGBACyKPcJjn+/kndv7c43rhscc3WZqZh2G1/2B2ApNG3Bemmm7vJBwtBeZIIlbbJoXmraDe1ab/7zZKfDRDZB/wgR5+6Ew6P6aX1zMTTMXPze8acro6Qu//bV2gijrCKRsgu43Vj7+nGMm8EM7wR1La7fHRk1pbdrkj243zR7hXapeL/kX8yHqKDIXJke/XP02Tx0y356GPnz6W9Ml4kID3RNzUXQUkIq5KHqr1jquzDqhQKbW2qmUegFwaK2fPtN2JdAvvv3peTy2aAcHTuaTVWCjdVM/5t89sDScj2UX8trqfXyxNQWnhuHRYVzbPYLhXcIIC/QhPbeIUf/5ke6tglgwczBxR3OY+s6vdAgLYMkDQ1FFOfD9c+YilcUbHksyYVjbbIXma3rcYkj8FmyuKQGmfFr5a3NNHdlkenBk7DNfqbuON+HTtM3pdbQ2PxWbPc7FoQ2w8glzIa7/jPPfzrkoyjPnw7OBXngua88y0xNl8odnvqB8CbugQHdtYAzwKqbb4vta6xeUUs8BsVrrpa5mmRcBjWlyeUBrXXSmbUqgX1zJJ/O55Z0N2B2a0T0jCA30Yf6GQ1g8FAtmDSbpRB5/+nwnhTYHt8S0YdYVHaq8E/2CTYd54std/HZ4RxZuPoKfl4VF9w2hlXehueiWe8yE6p6vYeJ/ofeUyoXRGlI2w44FcHw3TJ5X9Vfrshx2OPgj7Pyf6c5WnAf+zU3wdptgamvKAvevB4un2UfOUdOOW1Xbaul2bab73YY3zdfqMf+CTtdU3/1MiHp2wYFeFyTQ65bWmoTjuRQUOyiyOXh00Q6sdicLZg4uHbyzNy2XW9/9FavNSV6Rnb6t/HltfFvatu9Y7XadTs2Ud35lU3ImoYE+LLpvCFGhAebi0lcPwG1fQMdR8GpPCO8G0z4rv4G8dNMz4+g2070MDW0Gma56FWu+J/eZHgopW0w7dMFJ02Og2w3QfSK0H2bCGyB+qekhcv3rpg33m0fMAA4w7axDHjAX9coqLjA9FfZ+ay4CXv2cuWAoRAN2wQOLhHvRWvPCN3uY+/PB0mXBfl7lwhwgukUTFswczG8/2cqILuH8yX8Znh9PNV3Fqqkxe3goXprUk38sT+DRa6NNmINpe27SyoS5UqbWvNE1iKKkD3duGnx4PWQfgetfMz0zdn1uuo9tfPv0kHmb1fRM+PkV0783pKPp1tdlHERfW7mXBZh+1JEDYe2Lpgtc4jcwYKYZmp70g+l21+NG01MEzMW4BVPMIJ2x/zFd84Rwc1JDd3MOp+bV1Xv5fEsK9wzrwO2D2/HmmiRe+34ftw5qy7XdI3BqTdeIICKCfCD5JxNiR7eZASbX/eP0xuaNM8+XvSCVn2FGy/WYVH3bsc0K/+xgBmSMe8UsS91ienKMf8OM1Ms5ZsI856iptZf0S9YaFkw1+7juH5B5EBKXm8E2vaaYPtQ1HYV4aAN8cB2gTPkH3WuW5x6H13qbWv3Et00/6A/HQepWmPSuWS6Em5AaeiMSfzSHZTuP0iuyKZeFB/DM0jh+ScqgQ1gAf18Wz39/3M+J3CKm9g3j+Y4JeLTveHrAy4//gjXPA8o0XSSuMM0Q/iGmLTl1i+latmUeXP4H0x1v0XQT8r7BlQfWlEj+2VyYjB59elmrfqaXSNxi6DjChHneCdMk027I6fWUMt0C5wwxg0ksPqav9pjZ1Q+5rk67IXDti2YgSpcxp5c3iTA9YH59yxxX7HumCeem9yXMRaMige5GiuwOHlywlf3pp2/04O3pwcuTenJzTBvWJqbzz5WJXH5ZKC80/QKPL1+HsK4w+QMzEGLN86bWO3Y2pMXB+9eaMO423twdylYAI58yzRMb/s+EePJPJmQ3z60+0BOXm2HmUVecXqaUCcv1/wfvjzbdGG9fUvWQ58Aw05e48BSEdTndj/p8VDfT4eWPQOwHZkbEk3th8APmW4cQjYgEuht558cD7E83k1s1D/RhZ0oWQzo2p0uE6Ro4oks4I7qEmzlC3n4Loq40A07eGW5GOnYeawaXWDyhdX/wDjQDQrqNN932wAxRTt8Lm+aa0Xs9Jpk27HX/glPJlSeC0tq0WXccWblPdo8bzWjDohyY/pUZGVid4EjzU1cCQk0TzM//McPEr/5b3e1LiHoigd6AHc0qJDkjn4HtQ0g5Vcj/rUlibK+WpQN4+rerYsIorU3ThU8TM+TbaTMj8Dw8TRNDSa8Qi5cZmnxgrXl85FczEi64tWmG2bXIzM8x7hUzMdJP/zaTG139XPn9Hd9lplId/njlskT0Mhc/2wyufjDJxXT5w+Z9GHDPhX0LEKKBkkBvoEr6jaflFNE8wJtgPy98LB48M67bmV+4839w6BcTpAHNzbJpi6pet8OVsG+lGWl4eOPptu2wzmZIemgn0+ziG2z6lm+dD8P/YmriTqdrBrv3AGV6n1Sk1MUbPFMTvsEw8sn6LoUQdUYCvYHYnZrN0axCBndsTk6hjWlzN1Jsd/LSjT35KekkaxNO8PT13QgPKtOsUXHWOq1N+3fr/tD3jrPvtGRo/vZPzKRMbQaffq5ie/kA16x1i2eZyaJSt54eodl+WL3MaSGEKE8C/Xw4nRC/2LTFVnUTg/S9ZgrSmLurHm59ZJPp8RHeFZpFsSctj1v+u4H8YgcWD4W/twUFfDpzMD1aBzNlYBV3Njm6zcx7MfgBuPIxs+zkXsg65GpaqMHw9PBuZpj7r2+Zx20HVb9u1BXQoqcZmh3RE/pOM00qLbqbHyFEvZNAPx97V8Dnd5mJ7XvfAsMeLX+rq59mm6aPXZ/DzR+Wv9iXFm/6ezvMzAiOgHCesv2NQN9wXpvSl21HTrEvLY8HRlxGj9bBVe//+G4T5oWnzA0KrnjU1NL3rTLPX1bD+yAqZS6c7v7cXCANP0MwKwX3rDKz5Z1pKL0Qot5IoJ+P+K/MHNk9J5tAPbQeHtxaftL/sC6Qnmjuv3jju9DpKrTNSvFnd+PwCGBp9H9o6TzGbxJf4nrnMmLunUOP1sFcFd3M3NMw9j1Ye9zMT9KqD7TsY244kHvc9Djx9INhd5mLlcd3mr7bSavNfstOKHU2HVyBHhlz+oJpdRr6/S6FuMRJoJ8re7GZ4a/rONOfO7wrfPMHyEgyFxGzjpih7aP/aYbBf3YHfDKJLZF3sD8tm5tt8dxV/CjrdoRgdzbjDe+BTPP5Ca8wV6+LTe+YGxM0aWUGxNiLTPONdp4uQ3Ab06fbr6kZHh+/1Nxp5tAvZq7tc1HSjt72N7Xw5ggh6pME+rk6+CMUZZtZ/sD0vwYzdD20ExzeYB63HYKtWQf+3uJ1uh57kakpH9Ef2Bd5Ey/e/BjhTXyw2pxwOBivj8eaboLdxpvad8dRcPuXp/dZnG9uIeblC4EtzGRTJW3k7Yaai5WRA0y/8cuuOrfjadrWzJN9pj7iQgi3IIF+rkruQdhxhHkcEmXmRNn/gxm4cugX8AlCh3fjz1/s5MutaUwb9CzHIibT8tgaOo39N/iYnip+3hboOBRa9DC35crYZ0ZUVhz04h0Akf2rLk/J1LEb3gCvAGh3HjXtDlee+2uEEA2OBPq5cNjNXNwVZ/zrOBJ2LDTNMYc2QNvBvPTdPr7cmsofro7moVGdgJ5AFXODK2W6BC572NyJvPfUqm8qXJ0u40ygJ/9k5lKpaiZCIcQl4QJuvXIJOvQzFGaaWnFZHUeaPtl7V8DJRH4q7sR/fzzAtEFteXDkZWffbq+bwSfY9Jo514EvQS3NfOJw7pNZCSEaFamhg5l1sGUfE45nEr/UTEJVpp06PbeII5Ye9FUWnGtexAK8ujeUG/u15rkJPWp2x3vvALj+FdO//XzmM+lxk7lxcadqJs8SQlwSJNAz9psbHXS6pvoh8mC6C+5YYJo4XNPR2h1Opr+/ifhjOSzy7siA9D0UaS/GjR7LjGHRNQvzEhcy89+Ae6DTVXV353MhhFuQJpdtH5t/931n5jOpzpoXzJzhI54oXTRvfTLxx3J49JpoHO3NRVJri77ceUXncwvzC+XhcfpOPEKIS9alHegOO2z/1AxrDwhz3fyhCmlxJvgHzioNzpRTBfz7u72M6hLOAyMuY/DVkwEI7iI9RoQQ9ePSbnJJWg15x2Hsv81goG8fhwM/mm58xQWmX7dPEzPQx6cJP7S4g83fJhAVGsCynccA+NuE7qY23ro/jHoGet1SzwclhLhUXdqBvm0+BISbbohOB/zyOnz9EPgEQdrucqMzd/f4E3f9b3+5lz85piuRzVy3d/PwgGF/uJilF0KIci69QI9faubFbtoG9n4Lg39rpqC1eJEx5An8f3gS7+C2WIb90YzILMol+ZSVm2N7MjAqhA9mDOBkXhEncovo37aKG0wIIUQ9ubQC/WQSfHZ7+WV9Tz+ekxnD3Ly36ZPflHmDBxDs58XXO4/xxA87aRvmz7t3xBDg40mAjyftmsuMg0KIhuXSCvTjO82/o/8F+emm/3dYdOnT245k0SLIh/hjOdz83w0E+XoRe+gU3VsF8d50E/BCCNFQXVqBnhZnpqPtP73SEPliu5NdqdlMH9KOEV3CmflhLBlexbx4Y09ujmmDxeMidkMUQojzcOkFemh0lfOdxB/LodjupG/bZvymYyhrHhuOv7cngT6X1lskhHBfNeqHrpS6TimVqJRKUkpVur27UqqtUmqNUmqbUmqnUmpM7Re1FqTFVXu7tG2HTwHQt21TAMKb+EqYCyHcylkDXSllAd4ERgPdgKlKqYq3nn8K+Exr3RczpeBbtV3QC2bNhuzD1Qb61sNZtAz2pWWw3JVHCOGealJDHwgkaa0PaK2LgYVAhekG0UCQ6/dg4GjtFbGWpMWbf1v0qPLpbYdPldbOhRDCHdUk0FsDR8o8TnEtK+tZ4DalVAqwHHiwVkpXm07EmX+rqKGfyLWScqqQftKvXAjhxmprLpepwDytdSQwBpivlKq0baXULKVUrFIqNj09vZZ2XUNpcWZAUVCrSk9tO5wFIDV0IYRbq0mgpwJlbyMf6VpW1t3AZwBa6w2ALxBacUNa63e01jFa65iwsLDzK/H5SoszzS1VzIK47XAWXhZF91bBF7dMQghRi2oS6JuBTkqpKKWUN+ai59IK6xwGRgEopbpiAv0iV8HPwOk0behn6OHSrVUwvl6Wi1wwIYSoPWcNdK21HfgdsBLYg+nNEqeUek4pNd612h+BmUqpHcACYIbWWtdVoc9Z9mEozq0y0AuLHexMyaZvm6YXv1xCCFGLatTRWmu9HHOxs+yyp8v8Hg8Mrd2i1aK0kguilXu4fLPrGIU2B6N7RFzkQgkhRO26NG5wkRYHKAjrUumpBZsO0yEsgIFRIRe/XEIIUYsujUBP3QIhUeATWG5x4vFcthw6xdQBbS/uLeOEEKIONP5AL8yC/T9A9GgAnM7TTfsLNh3G2+LBpP6R9VQ4IYSoPY1/spKEZeAoxtl9Eh/9cpDZ3+3liuhQfj8qmsXbUrm2RwQhAd71XUohhLhgjT/Qd32OPbg9074pYmNyPP3aNmVtYjrLdx0HYOrANmfZgBBCuIfGHeh5J+DgjyxvMoXdR3P45029mNw/kvTcIl5ZvY9T+cUM6dC8vksphBC1onEHetwS0E7ezujLrb9py80xpjYeHuTLizf2rN+yCSFELWt8F0ULMiFjPzgdsPsLcoOiiXe0ZkSX8PoumRBC1KnGV0NfMAWObASLDziK+KXFTJr4eDKgvfQzF0I0bo0r0IvzISXWdFFs3hGde5xXEwZzRXQYXpbG92VECCHKalyBfnQbaAfE3AnR1xKXmk1C7M/cI80tQohLQOOqth7ZaP6NHADADwknUAqGd77IU/UKIUQ9aGSBvhmadwJ/017+fcIJekc2JTTQp54LJoQQda/xBLrWkLIJ2gwC4GReETtTshgpzS1CiEtE4wn0zANQkAFtBgKw5dAptIahl1W6cZIQQjRKjSfQS9rPXYG+51gOSkHXlk3qsVBCCHHxNKJA3wQ+wRDaGYD4ozlENQ/A37txdeQRQojqNK5Aj4wBD3NIe47n0LVVUD0XSgghLp7GEejWHDgRX3pBNNdq40hmId1aSqALIS4djSPQUzYDGtqY/ucJx3MBaT8XQlxaGkegJ31v5m6JNBdE44/mANBVauhCiEuI+we61pC4HKKuKL1n6J5jOTTz9yIiyLeeCyeEEBeP+wf6yb1w6iB0vq500Z5jOXRtGSQ3fhZCXFLcP9ATV5h/XTeBdjg1iWm50twihLjkNI5Aj+gFwa0BOHgyH6vNKYEuhLjkuHeg558087d0Hl26KP5YyQVR6eEihLi0uHeg7/sOtLNcoO85loOXRdEpXAJdCHFpqVGgK6WuU0olKqWSlFKPV/H8K0qp7a6fvUqprFovaVUSV0CTltCyT+mi+KM5dAwLxNvTvT+rhBDiXJ11ohOllAV4E7gaSAE2K6WWaq3jS9bRWj9SZv0Hgb51UNbKkn+GzmPA1Zslv8jOxoMZTO7f5qLsXgghGpKaVGMHAkla6wNa62JgITDhDOtPBRbURuHOqCATCjMhvEvpotV70rDanFzfu1Wd714IIRqamgR6a+BImccprmWVKKXaAVHADxdetLM4ddD8G9KhdNGynceICPIlpl2zOt+9EEI0NLXd0DwF+Fxr7ajqSaXULKVUrFIqNj09/cL2lFk+0LMLbfyYmM6Yni3x8JABRUKIS09NAj0VKNsoHelaVpUpnKG5RWv9jtY6RmsdExZ2gTduzjxg/m3WHoBV8WkUO5xc37vlhW1XCCHcVE0CfTPQSSkVpZTyxoT20oorKaW6AM2ADbVbxGpkHoCgSPDyA+DrHUeJbOZHnzZNL8ruhRCioTlroGut7cDvgJXAHuAzrXWcUuo5pdT4MqtOARZqrXXdFLWCzAMQEgXAqfxifkk6ybherWT+FiHEJatG92fTWi8HlldY9nSFx8/WXrFqIPOA6bII/JR0ErtTM6ZnxEUtghBCNCTuOfrGmgP56aUXRA+m5wMQ3UJGhwohLl3uGeglF0RdgX4oM5+IIF98vSz1WCghhKhfjSLQj2QW0La5fz0WSAgh6p+bB7q5KHooo4B2IRLoQohLm5sG+kEIjADvAAqLHZzILaKd1NCFEJc4Nw30A6XNLYczCwBo2zygPkskhBD1rvEEujS5CCEuce4X6MX5kHe8TPu56bIobehCiEud+wV6hUm5DmcW0MTXk6b+XvVYKCGEqH9uGOgV+qBnFNCuub8M+RdCXPLcPtCPZBZI+7kQQlDDuVwalB6TILQT+AbhcGqOnCrgmu4yh4sQQrhfoDdtY36AY9mF2Bxa+qALIQTu2ORSRkmXRenhIoQQ7h7oGSbQ20igCyGEewf6ocwCvCyKVk396rsoQghR79w60A9nFBDZzB+L3BRaCCHcPNCly6IQQpRy60A/lm2lVVPf+i6GEEI0CG4d6FabAz8v9+t5KYQQdcGtA73I7sDXy60PQQghao3bpqHDqbE5tNxHVAghXNw20K02BwA+nm57CEIIUavcNg1LAl1q6EIIYbhtoBfZnQDShi6EEC5um4ZSQxdCiPLcONBNDV3a0IUQwqhRGiqlrlNKJSqlkpRSj1ezzs1KqXilVJxS6tPaLWZlVrvroqjU0IUQAqjBfOhKKQvwJnA1kAJsVkot1VrHl1mnE/AEMFRrfUopFV5XBS5R5Kqh+3pKoAshBNSshj4QSNJaH9BaFwMLgQkV1pkJvKm1PgWgtT5Ru8WsrKSGLhdFhRDCqEkatgaOlHmc4lpWVjQQrZT6RSn1q1LqutoqYHWK5KKoEEKUU1sToXgCnYDhQCSwTinVU2udVXYlpdQsYBZA27ZtL2iHclFUCCHKq0kapgJtyjyOdC0rKwVYqrW2aa0PAnsxAV+O1vodrXWM1jomLCzsfMsMSLdFIYSoqCaBvhnopJSKUkp5A1OApRXWWYKpnaOUCsU0wRyovWJWdnpgkQS6EEJADQJda20HfgesBPYAn2mt45RSzymlxrtWWwlkKKXigTXAY1rrjLoqNJStoUuTixBCQA3b0LXWy4HlFZY9XeZ3DfzB9XNRnG5Dlxq6EEKAO48UtTvwsii5n6gQQri4baAX2ZwyqEgIIcpw20C32h0y7F8IIcpw30C3ye3nhBCiLLdNxCKbUwYVCSFEGW6biKaGLk0uQghRwm0DvcjulEAXQogy3DbQpQ1dCCHKc9tEtNodMqhICCHKcN9Atzmlhi6EEGW4bSIW2R0ysEgIIcpw20C32pwysEgIIcpw40B3SD90IYQow20Tscgm3RaFEKIstwx0p1NT7JCLokIIUZZbJqLcrUgIISpzy0AvvVuRtKELIUQpt0xEq90EuvRyEUKI09wz0G0lTS5uWXwhhKgTbpmIRfaSJhepoQshRAm3DPTTNXQJdCGEKOGmge5qQ5eLokIIUcotE7E00KWGLoQQpdwy0E/3Q3fL4gshRJ1wy0Qs7YcuNXQhhCjlloFeJBdFhRCiErcM9NKBRXJRVAghStUoEZVS1ymlEpVSSUqpx6t4foZSKl0ptd31c0/tF/U0qaELIURlnmdbQSllAd4ErgZSgM1KqaVa6/gKq/5Pa/27OihjJTKXixBCVFaTRBwIJGmtD2iti4GFwIS6LdaZWe0OPD0UnhYJdCGEKFGTRGwNHCnzOMW1rKJJSqmdSqnPlVJtaqV01bDanNJ+LoQQFdRWKn4NtNda9wJWAR9WtZJSapZSKlYpFZuenn7eO7PaHNJ+LoQQFdQk0FOBsjXuSNeyUlrrDK11kevhXKB/VRvSWr+jtY7RWseEhYWdT3kBM7BIAl0IIcqrSaBvBjoppaKUUt7AFGBp2RWUUi3LPBwP7Km9IlZmtTnwkVGiQghRzll7uWit7Uqp3wErAQvwvtY6Tin1HBCrtV4KPKSUGg/YgUxgRh2WGavNKVPnCiFEBWcNdACt9XJgeYVlT5f5/QngidotWvWK7FJDF0KIitwyFYukhi6EEJW4ZaBb7Q6ZaVEIISpwy1SUbotCCFGZmwa6DCwSQoiK3DIVpYYuhBCVuWWgy8AiIYSozC0DXQYWCSFEZW6XilprU0OXbotCCFGO2wV6yQ2ipYYuhBDluV0qlt6tSGroQghRjtsFesn9ROWiqBBClOd+gV5y+zlpchFCiHLcLhWtriYXH2lyEUKIctwu0IvsUkMXQoiquF0qltTQpQ1dCCHKc8NAlxq6EEJUxe1SsSTQpQ1dCCHKc79At5c0ubhd0YUQok65XSoWSQ1dCCGq5HaBfrqGLoEuhBBluV2gF8lFUSGEqJLbpWLbEH9G94iQGroQQlTgWd8FOFfXdI/gmu4R9V0MIYRocNyuhi6EEKJqEuhCCNFISKALIUQjIYEuhBCNhAS6EEI0EhLoQgjRSEigCyFEIyGBLoQQjYTSWtfPjpVKBw6d48tCgZN1UJz6IMfSMMmxNFyN6Xgu5Fjaaa3Dqnqi3gL9fCilYrXWMfVdjtogx9IwybE0XI3peOrqWKTJRQghGgkJdCGEaCTcLdDfqe8C1CI5loZJjqXhakzHUyfH4lZt6EIIIarnbjV0IYQQ1XCLQFdKXaeUSlRKJSmlHq/v8pwLpVQbpdQapVS8UipOKfV71/IQpdQqpdQ+17/N6rusNaWUsiiltimllrkeRymlNrrOz/+UUt71XcaaUko1VUp9rpRKUErtUUoNcddzo5R6xPU3tlsptUAp5esu50Yp9b5S6oRSaneZZVWeB2W87jqmnUqpfvVX8sqqOZZ/uf7GdiqlFiulmpZ57gnXsSQqpa69kH03+EBXSlmAN4HRQDdgqlKqW/2W6pzYgT9qrbsBg4EHXOV/HPhea90J+N712F38HthT5vHLwCta68uAU8Dd9VKq8/Ma8K3WugvQG3NcbndulFKtgYeAGK11D8ACTMF9zs084LoKy6o7D6OBTq6fWcCci1TGmppH5WNZBfTQWvcC9gJPALiyYArQ3fWat1yZd14afKADA4EkrfUBrXUxsBCYUM9lqjGt9TGt9VbX77mYwGiNOYYPXat9CNxQLwU8R0qpSGAsMNf1WAEjgc9dq7jTsQQDVwDvAWiti7XWWbjpucHcgcxPKeUJ+APHcJNzo7VeB2RWWFzdeZgAfKSNX4GmSqmWF6WgNVDVsWitv9Na210PfwUiXb9PABZqrYu01geBJEzmnRd3CPTWwJEyj1Ncy9yOUqo90BfYCLTQWh9zPXUcaFFf5TpHrwJ/Apyux82BrDJ/rO50fqKAdOADVxPSXKVUAG54brTWqcBs4DAmyLOBLbjvuYHqz4O7Z8JdwArX77V6LO4Q6I2CUioQ+AJ4WGudU/Y5bboaNfjuRkqpccAJrfWW+i5LLfEE+gFztNZ9gXwqNK+40blphqntRQGtgAAqf+13W+5yHs5GKfUkphn2k7rYvjsEeirQpszjSNcyt6GU8sKE+Sda6y9di9NKvia6/j1RX+U7B0OB8UqpZEzT10hMG3RT19d8cK/zkwKkaK03uh5/jgl4dzw3VwEHtdbpWmsb8CXmfLnruYHqz4NbZoJSagYwDpimT/cXr9VjcYdA3wx0cl2t98ZcQFhaz2WqMVcb83vAHq31f8o8tRSY7vp9OvDVxS7budJaP6G1jtRat8echx+01tOANcBNrtXc4lgAtNbHgSNKqc6uRaOAeNzw3GCaWgYrpfxdf3Mlx+KW58aluvOwFLjD1dtlMJBdpmmmQVJKXYdpqhyvtS4o89RSYIpSykcpFYW50LvpvHektW7wP8AYzJXh/cCT9V2ecyz75ZivijuB7a6fMZi25++BfcBqIKS+y3qOxzUcWOb6vYPrjzAJWAT41Hf5zuE4+gCxrvOzBGjmrucG+BuQAOwG5gM+7nJugAWYtn8b5pvT3dWdB0Bher7tB3ZhevbU+zGc5ViSMG3lJRnwdpn1n3QdSyIw+kL2LSNFhRCikXCHJhchhBA1IIEuhBCNhAS6EEI0EhLoQgjRSEigCyFEIyGBLoQQjYQEuhBCNBIS6EII0Uj8Pw3i81vcQ1EYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(1,len(total_train_acc)+1), total_train_acc, label = 'Train Acc')\n",
    "plt.plot(np.arange(1,len(total_val_acc)+1), total_val_acc, label = 'Validation Acc')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cec6b7e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAAsTAAALEwEAmpwYAABcrUlEQVR4nO3dd1zV1f/A8de5l72noqCCezDde1ZamiNHmjkqK/1Vtm2XX8u+DftWtsvShrmyHKmZI9Pce4CogCAgArLhAned3x8fRE1UVBSunufj4UPuZ55zP/C+574/55yPkFKiKIqi2D5ddRdAURRFqRoqoCuKotwkVEBXFEW5SaiAriiKcpNQAV1RFOUmYVddJ/bz85PBwcHVdXpFURSbtHv37tNSSv+K1lVbQA8ODmbXrl3VdXpFURSbJIRIuti6y6ZchBDfCSEyhBCHLrJeCCFmCiHihBAHhBCtr6WwiqIoytWpTA59DtDvEuvvBJqU/XsE+OLai6UoiqJcqcsGdCnlRiD7EpsMAn6Qmm2AlxCiTlUVUFEURamcqujlEggkn/M6pWzZBYQQjwghdgkhdmVmZlbBqRVFUZQzbmi3RSnl11LKtlLKtv7+Fd6kVRRFUa5SVQT0VKDeOa+DypYpiqIoN1BVBPRlwNiy3i4dgTwpZVoVHFdRFEW5ApXptjgP2Ao0E0KkCCEeEkJMFEJMLNtkJZAAxAHfAP933UqrKIpSgRs1DfilzlNislywLCO/BKv17D6nC0v54M8jJGQWXpfyXXZgkZRy1GXWS+CxKiuRoii3JIPRzPurj7DiQBrvD4+gR1PtPtuJLAPfb01kVPt6NK7lft4+B1Jyeev3wxxMzSMs0JOo+l40C3An2M+VRn5ueLrYX3COtYcz+H3/SQxGC80D3GkW4E4tDyd8XR2wWCUpOcVkFJTQqZEvzQM8sFolP+84wfurjxDo5cyEbiEMCK+Lg53WHt6dlMMHfx5hS3wW/VoFMLlPE0rNFmb8eYTNcVkEeDjRLzQAk8XKL7tTMFqs1PJwoqG/W5W/h6K6HnDRtm1bqUaKKsqNI6WkoNSMh9PZIGcwmjmYkkdEPS+c7PUAHDlVwOM/7yHHYMTJXk89bxee79eM1vW9KTZa+HDtUVZHn+KusDqM7xyMm6Mdm46dZn9KLuGBnnRu7Ed+sYmftiXxR/Qpoup5Map9fep6ObNoVzLL9p/EbJX4uDrg7+ZIQ39X6no5893m4yRnF1PH04nMglLeGRqOq4OeKb8coKDUjINex5O3NeGe1oHsScrlz5hTLN13El9XB+5oFcDhtHyiT+ZhspyNaXU9nWhRxwOA1NxiErOKKDFZCfBwwt/dkSPpBRjN1ou+Z+2CvREIdiRm0z7Eh5wiI8cyCnFztMPLxR4HOx0JmUX4uTlwe8sAft9/koJSMwC+rg7c16E+R04VsOFoJkgY2iaQCd0a0ugagrkQYreUsm2F61RAVxTbE5dRyLJ9qWQUlGIwWtDrBI1rudG0tjv+7o64OeoxWyUHU/I4kJJHTFo+R08VUFBqpqGfK92b+lNYambVwTSKjBaa1Xbn41GRlJqsjJu9Awe9jtta1qbEaGFz/GnS80u5JyqQPSdySMwyEFXfi/3Jueh1AoHAaDkbFPU6gVVKdELQqaEv+1NyKSjRgpwQ0LWxH35ujuQYjJzKK+H46SJKzVaCfV14d2g4Let6MPGn3WyOywIgop4X/xnYim82JrDi4Nnbcy4OesZ0asBjvRqXf0gZzVZOZBtIPF1EXGYhh9PyOZyWj16nI9DLiXo+LvRtFUD7YB90OoHZYiU5p5iswlKyiowIIMjbBU8Xe1YeSOOn7UnkFBl5tX9LhrcNAuDvo5msj82goMRMUamZqPrejO3UAFdHO/IMJubuSMJep+O+DvVxddSSIEWlZsxWiafz+d8YroYK6IpSAx1LL2DDkUwi6nkRVd8Le72O/BITqTnFnC4sJavQyPHTRcSk5XP8dBE+rg7U93EhOdvA9uPZ6HUCX1cHXBz0GM1WTuaVVHgeN0c7WtbxoHkdd2p7OLEzMZttCVnY6XT0D6tDRD0v/rfmKPklJux1Ah83B+Y+1JH6vi4AFJaa+XjtUb7bnEiglzPvDA2jcyM/krKK+GlbElJCnxa1iarvxcHUPP4+kom9XseIdkHU8XSm2GhhxcE0MgtKGRhZl0Av5/PKZ7FK0vKKqeXuVJ7GMJqtvL3yMK6Oep7s07R8+frYdBIyi2gb7EOruh7Y669vz+sz8VEIcV3PcyVUQFeU68BqlXy87hiHUvMI8nYmyNsFb1cHPJ3tqevlRLPa7tjpdew5kcOHa46SkFnEkKhAhrUJYvGeFL78O748PeDioMderyOv2HTeOXQCQvxcaeTvRo7BSHJ2Mc4Oeoa3DWJ4m3r4uzuWb1tQYiIuo5DsIiNFRgtSSlrV9aChnxs63fkBqdRsQSDKA+XpwlJe+e0gJ3NL+GZsWwI8nS6o76m8Erxc7MtTM0r1UAFdUaqY2WJlyi8H+HVvKg39XcnIL6WwLHd6hrO9nga+LsSeKsDX1YFWgZ5sOpbJmT+5IVGBTO7ThCOnCtgSfxqLVVLfx4Ugbxf83R3xdXOgrqczzg4qgCpnXSqgV9v0uYpSk5WYLESfzGN3Ug4FJWYe7BKCt6tD+bpnFu5j5cFTPHt7Ux7v3RiA/GIzucVG8opNHD9dxN4TucSeyueFfs3Lc6zJ2QaWHzhJRJAXXRr7AVoLvF9oQLXVVbl5qBa6ovxLQmYh9369jcyCUkC7kefr6shbg1tRWGrhgz+PkJZXwqv9WzChW8NqLq1yq1EtdEWppIz8EsZ+twOrVfL56Na0C/Yhs6CU5xbtZ+JPewCICPLkw3sj6djQt5pLqyjnUwFduSUVGy3kl5io7XH25l+ewcS42TvJLjIy/5GOhAd5AeDv7sjSx7vw49Ykans4cVdYQI3q9aAoZ6iArtxy9iXnMumn3aTlldCqrgc9mvoTe6qAf+JOY7VKvh3frjyYn2Gv1/Fg15DqKbCiVJIK6IpNM1mszFx3jD8OnSI00JMOIT7c0SoAn7IbmKVmC5+ujyM1t5iIIC/MVsm7q2Lxd3fk2dub8teRDD7fEE+QtzNjOjZgcGQgYUGe1VwrRbk66qaoUqNIKUnLK8HXzQFHOz0mi5V/jp3m76OZRNbzol9oQHk/6LiMQp5ZuI8DKXm0C/bm+OkiThcacXe044k+jenVrBZPL9zHodR8/NwcOF1oBLSRip+MiirvtVJUasbFQa/SKIpNUP3QlRqvoMTEb3tT+Xn7CWJPFaDXCYJ9XcguMpJjMGGnE5itEncnO1rV9SAhs4iMglK8XOx5554w+oXWQUpJ9Ml8/rfmKOtjMwDwdLZnxvAIbmtRi1P5JaTkFBNVzwu76zHCsLQQHKt+wiVFOZcK6EqNsTtJGzXp5+bAY70a09DfjXk7TvDBn0fIMZgIDfRgUEQgecUmjqYX4Oyg5+7wunRt4seeEzks2pVCQmYhjWu50yzAjUGRgefd2Dxj07FM/jh0ikk9GxHk7XL9K7ZzFqx4Dto/An2ng/7a5+yosfbPhx3fQI8XoOkd1V2aqlWQDls/gX0/wx3TIfKSk81eHaNB6wtr73z5bSugArpy3Ukp2XjsNG0aeONWNiFRqdnC91sSKSq14OvmwMGUPBbtTsHf3ZGiUjPFJgt1PZ1JzS2mQ4gPL93Vgsh6XtVbkauxcxaseBZ8G0NWHIR0h+Hfg4vP5fctLYSYJXDwF23/Pq+Dk0fF21otcHIfnD4CxiKwGKHVPeBRyWeyW0yQuAmC2l/dNwkp4e93YcN/wd4VTEUQNgL6vQOu16ELZ9JWSNgAxTlgLoHGt0HTfmDncOG2phIoTIf8k5CyAxL/geJc6PmCtp+UEPs7xCwFt9rgHQzBXaFWC21/Y5FWt21fgtUE7nXBkAWP/g3+zSoun9UK+3/WzlM3CupEXPx9zUuBuLVwZJVWpwEfXfWHhQroynX35d/xvLMqlvo+LswcFUUtd0cmzd3D/uTc8m3sdIKHuoaUzRdt5dt/EtiekM24zsEMCK9TM3PYGbFwbDW0fejCP1aLCbZ8Auv+A03vhBHfw6HFsPxJcPGFXi9DxH2gv0jfg4O/wPKnwFgAXvW1P3r3unDbVCg4Ccc3akHF2Rv0DpC8XQtu5/IIgvsXQ63mYMiGrZ/C6WNQkgv2LtD3bfBtBBYz/DoBon8DBzcIHQoegVqAT9sP9dpD+L3gURcOL4f49RDUDro8CX5NIXmHduzDyyByNNz5HmyZCZs+ACcvuOt9aDVEa3kasiEvGUzFYDVrx7FzvLD+5zKXanU88zuQsgtm36m9x2c+4ErytPeizQPatwN7Jy0QL38SDi46/3h+TbUPvJxEaNYfCk9B6m5w8dP2MRdr2zW+HZrfBZs+hLwT2vXq/pz23n3ZBdzrwIR12rnOVZwLv02Eo6vOLtPZQcOe2vvgEQinj0J6tPbhkh2vbePVAJrdCVH3Q0DYpd+Ti1ABXbmuNsedZsy32+nS2I+EzCJO5Zfg5miHxSrL89c5BhN6nSjvfWITDi2GpY+DyQDeIXDP11rgKy2E+HWw9j/aH2qLu2Hot2eDVuoeWDUFUnaCf3No1Bs862mtwZAeoNPBoV9h8UNQrwP0eQPqd9SC2JJJkHVMO45fM/CqpwUPY5HWCmzcR/vfyRNyk2DeKC0YtpugfVMozdeCmZMXZMaiTcL9LRxYCAcXQpenoChTO7+5RAsqdcIh4W8tCAPoHbXyJO/QAp97HShI04Jct2e1f2cCb3o0LH0MTu6F4G7aB056tHbeMzwCodsz0KALHP5dC4J2TlorWe+g1TsjWgv893wNds7wdU8tbfXIBu2bjsWstWz3/qC1sv2aQZ/X4K//QuZhaP8o1G4JbgFaS9m9tva+bPkENs7QjtHzRS1gC532gbl/ntYiN5zWjnf3R9Cg89lyH/0Tfh6uXT+PumDIAZ1e+4BJ3Ky9X33f1gL4yb1a4I5ZArknzh7DyRPqd9Kue6Ne2u/DNTZcVEBXrojVKjmYmseeEzkcTM3jWLo2g1+uwUifFrWZMTyifJa+1Nxi7v7kH3xdHVjyWBfMVslbv+0mOTOXt0Z1pXGtG3CT0JCtpSPc/K/9WEVZkLoLYlfAnu+hXkfo9BisfgXyU8C1ltbaA+2P8/Zp0OSOC/9IpdRas/98CJlHtfQEgE9DaHYXbPtCC+b3/wIOrmf3MxVD0mao1apyqZScRPjxHu2DpWEvLcDUbnl23fzRkH5Ie937Na31CdqHktWktXhBSx+c2KoFt0a9wdEdik7D9q+0AN1igPbB5ej+7xJowXbrp7Dja/BrAg3KUhn2zmAshK2fad8uABBa4NbZQc5x7YMqsLVW370/apt41oPsBJiwpuJWbNxaWPqEFpSdvGDYd9oH3cUYDdr5KkzVFGsfwEFtK/4W8dd/YfNH2nlcfEFaoCRf+7Y28BPtg+9cUkLaPm0b/2ZaeqeKv3mqgK4A2oCaD9cc5VReSfkoyakDW5XnrfeeyGHhrhTWHU4no2weE393R5oHuOPv5ggCft2TSt9Wtfn0vtbsTsrhuUX7yTOYWPp4FxrKZNj1HexfoH1FfWJ3xQHgDKtFa/1eapt/Mxvh1EEt6Kbs0v7PTtD+YEcv0oLRlbKYtPzq9q+0oAZaK67dw3DHW1ogKMmHje9pHx4+DaFWSy2QXyydci4ptZZr/HrtHCk7tDz2mF+vrO4XU5yjpVmC2l0YPEoL4c9Xtfx858ev/VxXQ0qtdZ1zXMuBe9SteLucJPj1YS34D/0WwoZd/JjFubB7DrQcBD631oAvFdBvUfklJoxmK1Yp+XbTcb7ZlICfmyOR9bzwcLZn07FMMgtKGdm+PgmZhWxLyMbdQdC9WQC3taxFp4Z+1PZwPC+3PXvzcf6zPIbmAe7Eniqgga8LH90bSZRDivY1Wei1m1BHVkDPl7WbUueymGDXbDj6h/aV3mSAAR9Cm3EXVqC0ANIOaIEgKw6Sd2oB3Fz2IAe3AK1lFdhGy0fnpcCEteDfFI6t1W5Y2btoLasWd2vbnktKbb+1U7XWt1cDiBoDDTpBncjr1wUx84jWCnW4Ab1vbI3FrKWSfBtVd0lqLBXQb3LFRgvxmYW0qOOBXicoKjXz9srDzN1+4rztRrWvx0t3tcDD0Q4KM8i36HlvQxo/bTtBHU8nZgTvpFPqd+ge/ENrhV7ErE0JTF95mPs7NOClu5rj4mAHvzyo5Ryf2AXuAdpX/YS/4cn9Z3tApOyG5ZO1FIB/cy1fmZ2gtd76vAFdn9ZuoqUfgt3faze6jGVPR9fZQUC49hW3XnutNeoReLZFmnsCvumtpS9qtYQjK8HVX/uAMWRpX6cnrNNuHgJkHIbfn4ETW7Tg3fNFrcWtU3OPKzWbCug3kewiI1vjswj0dqaulxPL9p3ky7/jOV1opJa7I3eF1WFdbDonc4p4IawI5wZtsAp7Wtb1oJ31AKx5Xcvpmou1VuLkvZwutuLhaIfDlx20G3IB4fDQGi0Ibv4Y9vyg9aQ456utwWjWAjloQfmTNtD5CS2nDFrvkC86afnn7lNg/VtajtW9jtYjosUAbTuLSbsReHCRtq4wHaRVu2kWOgxaDdZaa571Lt+3O3knzOmvbdf9eeg4SatD/kn4qoeW3njkLzi+CX59REsL9Xlda5WrQK7YCBXQbxIZ+SXc+/U2jp8uOm95l8a+DAivy/rYDDYcySDQ05FFgfPxP7ZQ6w7X5Uktx7r9Sy2X2rSfFjS3fQ4jftDykCf3aimTloO0XgStx2mBcecs7SQhPWDs0opv8Cx/CvbNhacOaq3zM36bBNG/grOP1kui3YSK+1lbrbD5Q+1DwLtB2Y3DO8/esLsSmUfB2Qvcap2/PGkrfD8AfBpp/bgD28LIueeXV1FsgJoP3UadLiwlKauIlnU8MRjNjJ61nfT8Er4a0wYBnMg2EFHPi3bB2gCWUe3rU1Riwnn9q+h2LNRanpmx2qAXgA4TtT7O9s7aDcnDy2Hnt1oQP7BQ60J298dad7LNH2v7dHlSax2vfA72/gStx5xfyIJTWjCPvO/C4NjzRa0bl4sv3PsTBLWpuKI6ndYVrir4N614eYNO2gCYlc9p/a3vnnlh32JFsXEqoNcQUkqsEvQ6gZSSRbtSePP3GApKzeh1AncnO4qNFuY80J5OjS4+Ks91+4ew40vo+H9aFzbQeg3oHbTuYWfo9NqNyPVvaTfpDi3WcsjO3lr3tpI8rb9zm/FaC/rQr/DnK1qfZUO2lmbJS4GkLVreu/PkCwvj3QCejgZHj8r1Brne2j+s1dGrfpV3JVOUmkClXK6G1aKNuPNvDrVbXRgcsuIh4S9tWHRFw7iTd2p38j0CwbcRcQZnHv95Lwmni2js74aDnY59ybl0CPFhbKdgDqflcyyjgLGdgsufQ0naAVjzGkTeD+HDtWUF6fBRKDQfoPXNvVzQKkiHD1tC7VCt7+yZ9EtFTsfBF53BUnp2mc5Oq0PUGOjxfKXeOkVRro1KuVS1vT9pvTVAS0d0ehw6Tjy7/s9XtV4W697U1nV4RBsxBlh3fodY8QyibCSdVdgx3fISmfYR3N+hAfGZhZAdz9ddddwWYkKnO03/iGDwaqJ9kGQf1yZH2jSjrEdIDDTvr3WB2zlLu8nY+9XKtUDda2vd+aJ/A0dPaNL34tv6NYbRC7WWuW8TLc/tHqBuJipKDaJa6FfKbNR6dLj6QtsHYc+P2hwRTx0AzyBtdN0HzbRWsrkUjq7CbO/O7lpDOZFrYnjRXNZZonjPfC+1RQ5T7b7H1R74vy3U9vXVPiyWPnb5coTfCy0Hw/xR2uCXdhPgw1ba6MNR8ypfn+ObtJuFUWNg0KdX+64oinKDqBZ6Vdo3V5vEZ8D/oMnt2sx6H0dqg2X6vKZ1v7OaoccLbMj1Y1bCEkaW/MJdKd/TQUj2+/Qlt9O7POfqQrHJQkZWKB03joGdH2g9O5Y/qU3w0/15rVVvMWnpmdxkrdeJo4fWU6V+B608DXvCPx9p6Q9DlpY7vxLBXaHvf7VWvqIoNk210K+EuRRmttZSDRPWnk1rzBuljXp8Ohq+vR0rgjcDv2D25kSa1Xbn6dub0MU7F/esQ9osd7p/PVzh92e0IfOOHtr8HQ+u1rreVcaJ7fDdHdoAmtot4dFN6oafotzELtVCvw6PbbFBWfHafMqXs/dHbYh4r5fKg6bRbCWt+VgwnMay/k04dYCPMtsye3Mi4zsHs/TxLvQLrYN7YAvt5uW/gzloXQk9ArUW+H0LKh/MQWupN+qtTRrU8TEVzBXlFqZSLhmH4cuu2jD0+3+7ePe6jMPadKn1OkIjbWa3nCIjo77ZRuwpC2sd6tJ466eYpJ59Xrfx833t6dzIr3JlcPKAh9dpc4tU9mEF5+r3jvYEmdChV76voig3jVu7hS6l9tgwnZ32MIE1r1e8XUE6zB2uDcgZOguEIM9g4v5vt3P8dBGv9m9JcuP7AcgN6sn3j99Z+WB+hnvA1QVz0Kbp7D+j4ulBFUW5ZdzaLfSDv0DSP9rjoDIOw7bPtFGS3g20NIylVJtsf/88pCGLL0Jm8tf8ZGq5ZxKfWUhCZhFfj21Dz2a1oP1kWLAD/55TVNpDUZRqcesF9MO/a32nPQK1kY91W0PrsSCtGE8exGHVhQNkrDoHXrZ7loUHXWjTAA6fyqfUZOWz0a21YA7axE9jl97gyiiKopx1awX0UwdhwehzFgitz7ZOD+j5r9dU0hJWYnT2Y1Dv7ji7erBybwJ/HcvB3dOHRRMjadOgEg/+VRRFqQa3VkCPW6v9P/oXbQCQk6f2cATAbLGy9HA+jRv0w2yx8uTyFEB7Ys/Ibq14rFdjPJwuM32roihKNapUQBdC9AM+BvTALCnlO/9aXx/4HvAq2+ZFKeXKqi1qFYhbB7XDtAFB/7IlPovsIiMPdgnhjpa1WXs4HXs7Hd0a+2Gnv7XvHSuKYhsuG9CFEHrgM+B2IAXYKYRYJqWMOWezV4GFUsovhBAtgZVA8HUo79UrLYQT26BTxSMpl+8/iZujHT2b+aPTCe5opebJVhTFtlSm6dkeiJNSJkgpjcB84N9T8kngzLSCnsDJqitiFUncpD3lvNGFTwcvNVtYHX2KO1rVxsleTTalKIptqkxADwSSz3mdUrbsXFOB+4UQKWit8ycqOpAQ4hEhxC4hxK7MzMyrKO41iFunPTC4fscLVm06epr8EjN3h1/kaeSKoig2oKqSw6OAOVLKIOAu4EchxAXHllJ+LaVsK6Vs6+/vX0WnrqS4tdpEWnaOF6z6/cBJvFzsz841riiKYoMqE9BTgXrnvA4qW3auh4CFAFLKrYATUHOiY3YC5ByvMN2SUVDCnzHp9GsVgIOduvmpKIrtqkwE2wk0EUKECCEcgJHAsn9tcwLoAyCEaIEW0K9LTsUqreQb869sp7h12v+NLwzo01ccxmyRPNy9YRWUTlEUpfpcNqBLKc3A48Bq4DBab5ZoIcQ0IcTAss2eBR4WQuwH5gHj5XWal/fbg9/SZV4XSs99FNqlSKnNUe4doj1l5xz/HDvN0n0nmdSzEY383a5DaRVFUW6cSvVDL+tTvvJfy14/5+cYoEvVFq1iHg5aZ5r80nz8XSqRhz/6ByRvp7jvDH7enMjy/SdpUcedu8Pr8trSQwT7ujCpZ6PrXGpFUZTrz+ZGino4lgV0YyUCutUCa6eS7xpMt1V1yDPG0KKOB0v3nWTeDq3jzk8PdVBdFRVFuSnYXkB3OBvQL2v/PMiMZSrP0jDAm2kDQwkL8sRgNLMmJp1Ss5WuTWrOvVtFUZRrYbsBvfQiAX3zTG1uc4+6cHQ1mR6h/JrRml8HtCQsyBMAFwc7BkX+uyu9oiiKbbO9gO54iRZ6aQH8NV17NmfaPmRpAa/rnqBdsA+t63vf4JIqiqLcWLYX0C+VcoldCeYSbV7y+h1ZtjeFVQv2881gddNTUZSbn82NpHF3cAcuEtAP/QKe9SCoPVJKvt50nEb+rvRpXusGl1JRFOXGs7mAbqezw9Xe9cIcelEWxK+H0HtAp2PPiVyiT+YzoVtDdDr1SDhFUW5+NhfQQUu7XNBCj1kCVjOEDgNgfWw6ep3grrCrfPCyoiiKjbHJgO7u4H5hQD+0GPyaQUAYABuOZNKmvjeezuopQ4qi3BpsMqB7OHicn3LJS4WkLRA2DIQgI7+E6JP59Gx+g2d0VBRFqUa2G9DPbaHvnqP9H6alWzYc1eYF69lU3QxVFOXWYZsB3fGcgF5aADu+hub9yyff2nAkg9oejrSo416NpVQURbmxbDOgO3hQYCzQXuz+HkpyoevTAJgsVjYdO03PprUQQvVuURTl1mGzAb3YXIyptBC2fgrB3SCoLQB7knIoKDHTs5nKnyuKcmuxzYB+Zvj//h+hIK28dQ5a/txOJ+iiJt1SFOUWY5sB/czw/12zISAcGvUGwGi28vuBk7QN9sbDSXVXVBTl1mLbAT3vOIQOhbJc+YJdySRnF/NodzV3i6Iotx6bDOjl87nodFCvPQAGo5mZ647RPthH5c8VRbkl2WRAL8+h6+2gTiQAszcnkllQypR+zVTvFkVRbkm2GdDPpFy8AsHBhTyDia/+jqdP81q0Dfap5tIpiqJUD5sM6J52bgDke2pPHVp5KI38EjNP3960OoulKIpSrWzuARcA9tnxOFut5Lv6ArA/ORcvF3ta1fWo5pIpSs1mMplISUmhpKSkuouiXIaTkxNBQUHY21e+x55NBnRSduJutVLgpLXU9yXnEhHkpXLninIZKSkpuLu7ExwcrP5eajApJVlZWaSkpBASElLp/Wwy5ULKTjykIF9ovVuOphcQUc+rukulKDVeSUkJvr6+KpjXcEIIfH19r/iblI0G9F142LuRb8znUGo+VgmR9Tyru1SKYhNUMLcNV3OdbC+gF+dCZiwezr7kl+azPzkXgPAgr+oslaIolZCVlUVkZCSRkZEEBAQQGBhY/tpoNF5y3127djF58uQrPue+ffsQQvDHH39cbbFthu3l0E/uAcDDvS5HipLZl5JLkLczfm6O1VwwRVEux9fXl3379gEwdepU3NzceO6558rXm81m7OwqDktt27albdu2V3zOefPm0bVrV+bNm0e/fv2uqty2wvZa6Cm7AIGHZwPyjVoLXeXPFcV2jR8/nokTJ9KhQwemTJnCjh076NSpE1FRUXTu3JkjR44AsGHDBgYMGABoHwYPPvggPXv2pGHDhsycObPCY0spWbRoEXPmzGHNmjXn5aTfffddwsLCiIiI4MUXXwQgLi6O2267jYiICFq3bk18fPx1rn3Vsr0WeodHIbgbHjn7KDIVUZBTyLhOwdVdKkWxOf9ZHk3MyfzLb3gFWtb14I27W13xfikpKWzZsgW9Xk9+fj6bNm3Czs6OtWvX8vLLL7N48eIL9omNjeWvv/6ioKCAZs2aMWnSpAu6+G3ZsoWQkBAaNWpEz549WbFiBUOHDmXVqlUsXbqU7du34+LiQnZ2NgCjR4/mxRdfZMiQIZSUlGC1Wq/ujagmthfQnTyhQSc8DAkACH2JaqErio0bPnw4er0egLy8PMaNG8exY8cQQmAymSrcp3///jg6OuLo6EitWrVIT08nKCjovG3mzZvHyJEjARg5ciQ//PADQ4cOZe3atTzwwAO4uLgA4OPjQ0FBAampqQwZMgTQ+oHbGtsL6GXODP/X6YsJDVQDihTlSl1NS/p6cXV1Lf/5tddeo1evXvz2228kJibSs2fPCvdxdDx730yv12M2m89bb7FYWLx4MUuXLmX69OnlfbsLCgquSx1qAtvLoZc5E9Ab+OtwcbDZzyVFUf4lLy+PwEBtWo85c+Zc9XHWrVtHeHg4ycnJJCYmkpSUxNChQ/ntt9+4/fbbmT17NgaDAYDs7Gzc3d0JCgpiyZIlAJSWlpavtxW2G9DLZlys56f61CrKzWTKlCm89NJLREVFXdDqvhLz5s0rT5+cMXTo0PLeLgMHDqRt27ZERkYyY8YMAH788UdmzpxJeHg4nTt35tSpU9dUlxtNSCmr5cRt27aVu3btuur943PjGbx0MJ3dn+SreyZUYckU5eZ1+PBhWrRoUd3FUCqpouslhNgtpayw/6bNttBd9No8LkKvJhlSFEUBGw7oerSbKFLYVo5LURTlerHZgG6y6JFWO6ROBXRFURSoZEAXQvQTQhwRQsQJIV68yDYjhBAxQohoIcTPVVvMCxWWmJEWF4zy5u2CpCiKciUu299PCKEHPgNuB1KAnUKIZVLKmHO2aQK8BHSRUuYIIWpdrwKfUVhqwmryJs+Ufr1PpSiKYhMq00JvD8RJKROklEZgPjDoX9s8DHwmpcwBkFJmVG0xL1RYakEafck2nrzep1IURbEJlQnogUDyOa9TypadqynQVAixWQixTQhx3ac0KywxYzX5kl2aSYlZ9XRRFFvQq1cvVq9efd6yjz76iEmTJl10n549e3Kmi/Ndd91Fbm7uBdtMnTq1vC/5xSxZsoSYmPLEAq+//jpr1669gtJf2lNPPUVgYGC1zv9SVTdF7YAmQE9gFPCNEMLr3xsJIR4RQuwSQuzKzMy8phMWlpqwGrVniqYWpl7TsRRFuTFGjRrF/Pnzz1s2f/58Ro0aVan9V65ciZeX11Wd+98Bfdq0adx2221Xdax/s1qt/Pbbb9SrV4+///67So55NSoT0FOBeue8Dipbdq4UYJmU0iSlPA4cRQvw55FSfi2lbCulbOvv73+1ZQa0lIvV6ANAckHyZbZWFKUmGDZsGCtWrCh/mEViYiInT56kW7duTJo0ibZt29KqVSveeOONCvcPDg7m9OnTAEyfPp2mTZvStWvX8il2Ab755hvatWtHREQEQ4cOxWAwsGXLFpYtW8bzzz9PZGQk8fHxjB8/nl9++QXQpgmIiooiLCyMBx98kNLS0vLzvfHGG7Ru3ZqwsDBiY2MrLNeGDRto1aoVkyZNYt68eeXL09PTGTJkCBEREURERLBlyxYAfvjhB8LDw4mIiGDMmDHX+K6eVZlJUHYCTYQQIWiBfCRw37+2WYLWMp8thPBDS8EkVFkpK3Am5QJwIv/E9TyVotycVr0Ipw5W7TEDwuDOdy662sfHh/bt27Nq1SoGDRrE/PnzGTFiBEIIpk+fjo+PDxaLhT59+nDgwAHCw8MrPM7u3buZP38++/btw2w207p1a9q0aQPAPffcw8MPPwzAq6++yrfffssTTzzBwIEDGTBgAMOGDTvvWCUlJYwfP55169bRtGlTxo4dyxdffMFTTz0FgJ+fH3v27OHzzz9nxowZzJo164LyzJs3j1GjRjFo0CBefvllTCYT9vb2TJ48mR49evDbb79hsVgoLCwkOjqat956iy1btuDn51c+dW9VuGwLXUppBh4HVgOHgYVSymghxDQhxMCyzVYDWUKIGOAv4HkpZVaVlbICRUYzTjo33O3dVQtdUWzIuWmXc9MtCxcupHXr1kRFRREdHX1eeuTfNm3axJAhQ3BxccHDw4OBAweWrzt06BDdunUjLCyMuXPnEh0dfcnyHDlyhJCQEJo2bQrAuHHj2LhxY/n6e+65B4A2bdqQmJh4wf5Go5GVK1cyePBgPDw86NChQ/l9gvXr15ffH9Dr9Xh6erJ+/XqGDx+On58foH3IVZVKTVMopVwJrPzXstfP+VkCz5T9u+6klBSUmHFzdCDIPYjkQhXQFeWKXaIlfT0NGjSIp59+mj179mAwGGjTpg3Hjx9nxowZ7Ny5E29vb8aPH3/FT7w/Y/z48SxZsoSIiAjmzJnDhg0brqm8Z6bprWiKXoDVq1eTm5tLWFgYAAaDAWdn5/KnK91INjdSNPe3JRwfPASDoQQ3Rz31PeqTnK8CuqLYCjc3N3r16sWDDz5Y3jrPz8/H1dUVT09P0tPTWbVq1SWP0b17d5YsWUJxcTEFBQUsX768fF1BQQF16tTBZDIxd+7c8uXu7u4VzoXerFkzEhMTiYuLA7QZF3v06FHp+sybN49Zs2aRmJhIYmIix48fZ82aNRgMBvr06cMXX3wBaPOz5+Xl0bt3bxYtWkRWlpbEuKEpl5pG7+lB6ZEj1I7ZjZuTHfXc63Gy8CRm69VPs6koyo01atQo9u/fXx7QIyIiiIqKonnz5tx333106dLlkvu3bt2ae++9l4iICO68807atWtXvu7NN9+kQ4cOdOnShebNm5cvHzlyJO+//z5RUVHnPSvUycmJ2bNnM3z4cMLCwtDpdEycOLFS9TAYDPzxxx/079+/fJmrqytdu3Zl+fLlfPzxx/z111+EhYXRpk0bYmJiaNWqFa+88go9evQgIiKCZ56pusSGzU2fK81m4nr15pB7IAsGT2ZErzTe2PIGq+5ZRZB70OUPoCi3MDV9rm256afPFXZ2eA4ZQtPj+wkw5lPPXetReaJA9XRRFOXWZnMBHcBr6D3opCQqZkt5QE8pSKnmUimKolQvmwzoDg0aEFO7MS32/42/kx8OOgfVdVFRlFueTQZ0gD/qd8A9K52Snbup515PDS5SFOWWZ5MBvdRs4e+AUEwubmR98w313FRfdEVRFJsM6EWlFox6e1IHjaZo82Y6RJtIKUihunrsKIqi1AQ2GtC1PueFdw7GKSyM8J93oyswqBujilLDZWVlERkZSWRkJAEBAQQGBpa/PjNh18Xs2rWLyZMnX9H5zp3M61ZQqaH/NU1BiRbQ3V0cqTPtP5QMG8a4v/W8VP8lvuv7HQ56h2ouoaIoFfH19WXfvn2ANoe5m5sbzz33XPl6s9mMnV3FYalt27a0bVth92uljE220AvLWuiujnY4tWiBz9hx9Nxr4tTRfUzfPl2lXhTFhowfP56JEyfSoUMHpkyZwo4dO+jUqRNRUVF07ty5fGrcDRs2lM+PMnXqVB588EF69uxJw4YNmTlzZqXPl5iYSO/evQkPD6dPnz6cOKF1qFi0aBGhoaFERETQvXt3AKKjo2nfvj2RkZGEh4dz7NixKq591bLJFvqZlIubo1Z8z8GDyZ49m0f1vZh27Fda+bZiRLMR1VlERanx3t3xLrHZFc/vfbWa+zTnhfYvXPF+KSkpbNmyBb1eT35+Pps2bcLOzo61a9fy8ssvs3jx4gv2iY2N5a+//qKgoIBmzZoxadIk7O3tL3uuJ554gnHjxjFu3Di+++47Jk+ezJIlS5g2bRqrV68mMDCw/KlIX375JU8++SSjR4/GaDRisViuuG43kk220AvKArq7kxbQHRs1RDg50a2gDpH+kcyLnXep3RVFqWGGDx+OXq8HIC8vj+HDhxMaGsrTTz990elv+/fvj6OjI35+ftSqVYv09Mo9MH7r1q3cd5/2SIcxY8bwzz//ANClSxfGjx/PN998Ux64O3XqxNtvv827775LUlISzs7O11rV68omW+iFJWdTLqBNB+DUvDkl0TH0uLsnH+/5mJySHLydvKuzmIpSo11NS/p6cXV1Lf/5tddeo1evXvz2228kJibSs2fPCvc5M60tXHxq2yvx5Zdfsn37dlasWEGbNm3YvXs39913Hx06dGDFihXcddddfPXVV/Tu3fuaznM92WQL/d8pFwCn0FBKDh+mjV8UAHsy9lRL2RRFuTZ5eXkEBmrPoZ8zZ06VH79z587lD9iYO3cu3bp1AyA+Pp4OHTowbdo0/P39SU5OJiEhgYYNGzJ58mQGDRrEgQMHqrw8VckmA/qZlIurw7kBvRXSYKBJvguOekd2p++uruIpinINpkyZwksvvURUVNQ1t7oBwsPDCQoKIigoiGeeeYZPPvmE2bNnEx4ezo8//sjHH38MwPPPP09YWBihoaF07tyZiIgIFi5cSGhoKJGRkRw6dIixY8dec3muJ5ubPhdg2vIYFuw8QfS0fuXLSuPiSBhwN3Xe+S/POC+jyFTEggELqqq4inJTUNPn2pabfvpc0FIubk7np/8dQkIQLi6UHIqmTe02xGbHUmgsrKYSKoqi3Hg2GdALS83lN0TPEHo9Ti1aUBKtBXSrtLIvc1/1FFBRFKUa2GxAd3e8sIOOc2grSg4fJsyrJXbCTuXRFUW5pdhsQP93ygW0ni6ypAT9iVO09GupArqiKLcU2wzoJebzeric4dQqFICSQ4doU7sNB08fpMRccqOLpyiKUi1sM6BfpIXuENwAnasrJdHRtK3dFrPVzIHMmt1vVFEUparYbECvKIcudDqcW7emYP16IrxDEQh2Z6i0i6LUFL169WL16tXnLfvoo4+YNGnSRffp2bMnZ7o433XXXeXzrJxr6tSpzJgx45LnXrJkCTExMeWvX3/9ddauXXsFpa/YuZOGVTebC+hSygp7uZzhPWoU5lOnEH9vo4l3E/akqxGjilJTjBo1qnyU5hnz589n1KhRldp/5cqVeHl5XdW5/x3Qp02bxm233XZVx6qpbC6gl5qtWKyywpQLgFvPHtg3qE/W99/TulZr9mfux2y99tFmiqJcu2HDhrFixYryh1kkJiZy8uRJunXrxqRJk2jbti2tWrXijTfeqHD/cx9YMX36dJo2bUrXrl3Lp9gF+Oabb2jXrh0REREMHToUg8HAli1bWLZsGc8//zyRkZHEx8czfvx4fvnlFwDWrVtHVFQUYWFhPPjgg5SWlpaf74033qB169aEhYURG3vp2Smzs7MZPHgw4eHhdOzYsXyqgL///rv8QR5RUVEUFBSQlpZG9+7diYyMJDQ0lE2bNl3bm4sNTs515uEWbhdpoQudDp8xY0l/6y06ZfdivrmY2OxYQv1Cb2QxFaXGO/X225Qertrpcx1bNCfg5Zcvut7Hx4f27duzatUqBg0axPz58xkxYgRCCKZPn46Pjw8Wi4U+ffpw4MABwsPDKzzO7t27mT9/Pvv27cNsNtO6dWvatGkDwD333MPDDz8MwKuvvsq3337LE088wcCBAxkwYADDhg0771glJSWMHz+edevW0bRpU8aOHcsXX3zBU089BYCfnx979uzh888/Z8aMGcyaNeui9XvjjTeIiopiyZIlrF+/nrFjx7Jv3z5mzJjBZ599RpcuXSgsLMTJyYmvv/6avn378sorr2CxWDAYDFfyVlfI5lroFU3M9W9eQwajc3enwaoDOBol8b/Pp7jsKSmKolSvc9Mu56ZbFi5cSOvWrYmKiiI6Ovq89Mi/bdq0iSFDhuDi4oKHhwcDBw4sX3fo0CG6detGWFgYc+fOvej0u2ccOXKEkJAQmjZtCsC4cePYuHFj+fp77rkHgDZt2pCYmHjJY/3zzz+MGTMGgN69e5OVlUV+fj5dunThmWeeYebMmeTm5mJnZ0e7du2YPXs2U6dO5eDBg7i7u1/y2JVhcy30wkoEdJ2rK17Dh5M9Zw6z11uxsywm2e9vmmzaiBDiRhVVUWq0S7Wkr6dBgwbx9NNPs2fPHgwGA23atOH48ePMmDGDnTt34u3tzfjx4ykpuboux+PHj2fJkiVEREQwZ84cNmzYcE3lPTNN77VM0fviiy/Sv39/Vq5cSZcuXVi9ejXdu3dn48aNrFixgvHjx/PMM89c8+RfNtdCv1zK5QyfceNw7dSJo70bs6aTM5bTpymt4Y+PUpRbgZubG7169eLBBx8sb53n5+fj6uqKp6cn6enprFq16pLH6N69O0uWLKG4uJiCggKWL19evq6goIA6depgMpmYO3du+XJ3d3cKCgouOFazZs1ITEwkLi4OgB9//JEePXpcVd26detWfs4NGzbg5+eHh4cH8fHxhIWF8cILL9CuXTtiY2NJSkqidu3aPPzww0yYMIE9e669A4fNBfTylMtFboqeYV+7FvW/nYWY/AC/Rmk3YAxbt1738imKcnmjRo1i//795QE9IiKCqKgomjdvzn333UeXLl0uuX/r1q259957iYiI4M4776Rdu3bl69588006dOhAly5daN68efnykSNH8v777xMVFUV8fHz5cicnJ2bPns3w4cMJCwtDp9MxceLEq6rX1KlT2b17N+Hh4bz44ot8//33gNY1MzQ0lPDwcOzt7bnzzjvZsGFDeb0XLFjAk08+eVXnPJfNTZ+7ZG8qTy3Yx/pne9DQ3+2y2yfmJXL3krv5cY4Hvs3CqfflF1dTXEW5Kajpc23LTT99bkElcujnauDRAB8nHxKbemDYuRNpMl3P4imKolQbmwvolU25nCGEoGOdjvzpdwprURHFBw9dz+IpiqJUG5sL6EOiAln4aCec7fWV3ufpNk9zvJErVgG5mzdefgdFURQbZHMBvbaHE+1DfK6o+2GAawCv9X2PxNpwdM0iquu+gaLUBOr33zZczXWyuYB+tToHdsauXWt847L4escn6pdauSU5OTmRlZWlfv9rOCklWVlZODk5XdF+lUpECyH6AR8DemCWlPKdi2w3FPgFaCelvLonQF9HnQY+SuryR9m04iuyKeKF9i+gE7fMZ5qiEBQUREpKCpmZmdVdFOUynJycCAoKuqJ9LhvQhRB64DPgdiAF2CmEWCaljPnXdu7Ak8D2KyrBDeTWth3CyYnR2SFMif2Z3NJcpnedjp3O5gbMKspVsbe3JyQkpLqLoVwnlWmetgfipJQJUkojMB8YVMF2bwLvAjX2EUE6Z2dcu3ah8cFsnoyczMrjK3nln1ewWC3VXTRFUZRrVpmAHggkn/M6pWxZOSFEa6CelHLFpQ4khHhECLFLCLGrur7yud92G+b0dEbrOvJk6ydZeXwlr295Hau0Vkt5FEVRqso1J5CFEDrgf8Czl9tWSvm1lLKtlLKtv7//tZ76qrj37Al6PQVr1jIhbAKPRT7Gsvhl/BjzY7WUR1EUpapUJqCnAvXOeR1UtuwMdyAU2CCESAQ6AsuEEBUOTa1uei8vXNq3o6Ds0VMTIybSulZrfj32q7rzryiKTatMQN8JNBFChAghHICRwLIzK6WUeVJKPyllsJQyGNgGDKyJvVzOcO9zG8aEBEoTEgDo37A/CXkJHM05Ws0lUxRFuXqXDehSSjPwOLAaOAwslFJGCyGmCSEGXnrvmsn9tj4AFKxdh7WkhB7ZtXA16Vlx/JK3ABRFUWo0m5ttsaocHz4C4/HjSKMRaTRytG1tPh1sz+qhq1XfdEVRaqybarbFquIzbhwODRvifd99eA4ZQtNd6XjGprEn/donmVcURakOt+yIGs8B/fEc0B8Aq8FA4ebNPLjuNL91/5VjucdYteV7mtaP4qXeb6HXVX4iMEVRlOpyy7bQz6VzcaH2888TkmZFN3cp2a+/yQvvJ9HtlSV8sOhJNfBIURSboAJ6GY8B/SGsOff9baXXYT0+I0fiKVzoM30dX335iBp4pChKjacCehkhBA3f+x++D0+gyR9/UOeNN2i1ZAWWuv70+GQLP7/3oOqnrihKjaYC+jkcQ0Ko9eyz2AdqMxvYBwTQZvEqssKCaDN7O8veegirtLLz1E4+2/cZuSW5gDbVZcGGDSRP+j9KYmIucQZFUZTr55a9KVpZeldXuv74O2sn3E3TuVv5b0Z75rctBSApL4lpAQ9xaup/KN67FwC72rWoM3VqNZZYUZRblWqhV4LO0ZHe3/1Oausg7t5QzAfhr/Nw2MP8kbCSY489ivHECQKmTsW1ezeK/tmsUjOKolQLFdAryc7egW5vfY29WRL5VwqTIicxPCkAu5R0PF5+Fu+R9+LWowemlBRMSUnVXVxFUW5BKqBfAceGIbj37UvO3LmIvEKGbraSVEsw020HUkrcunYFoPCfzdVcUkVRbkUqoF8hv4mPYi0qInnCw4jkNPLG3MnyxN+Ze3guDg0aYF+vHkX//FPp46n0jKIoVUUF9Cvk1Lw5bj17UhIdjWPLFgx76D161+vN+7veZ1PKJiztQsnZuomRvw0lKf/SqZfCzZuJ69GToq1bb1DpFUW5md2yk3Ndi+KDB0kafT9Bn36CW/fuGEwGxv0xjuN5xwk/XMLziy3MGOdBto89b20IwBUHnMPDce3cCbcePQCwGo0k3H03pqQT6NzdCf55Lo5NmlRzzRRFqenU5FxVzDksjKa7duLWvTsALvYufNL7E1r6tiSy3/1gp2fKiTCmfJtLcUw0JwpOkPnzTyQ/OpHML78EIHvO95iSThDw5jSEkyMnHn0UcyUey1eakEDe0qXXtX6Kotgm1Q/9KukcHM57HeAawA93/gBAUmQ0hr824+Pvxw+PNWCr80nyDXY8stxKt48+xlRcRP6PP+PWpw/ew4fj1LIlSWPGkjT+AYI+/QTHsqeylxw9irWgAJc2bQAwZ2dz4qEJmNPScGzaFKcWLQCQJhOlcXHlrxVFuTWpFvp14H3fKJzbtiFk3nymj/uJ9SPWs3PcXlymvsiOpoK8r2YhzWZqv/gCAM6tWlHvyy+wZGeTOHwEuYsXk/r8FI4PGkzS6PvJ/PxzpNlM6jPPYsnKQjg7k/3D2WegZsz4gOND7iH/zz+rq8qKotQAKod+g6079gdHX3uehLp63Iffw+gWo2no2RAhBKaTJ0l5YjIl0dEIJyd8xozBnJFO3tJlODRogDEpiTrTp1MSfYjcRb/QeMNfWA0G4u/qDxYLek9PGi5bil01PYBbUZTr71I5dJVyucH6NOlHyFdN+T76e5bGLWXR0UX4OPnQ0rcl7QPac+c3/8P7r124du2Kfe1aSClxbNqUjBkf4HXvvXgNvYfSqChyfp5HzoIFGOPiEXo99b76kuSJk0h7/Q2CPv8MIUR1V1VRlBtMtdCrUVZxFqsTVxOTFUN0VjRxuXEIBB3rdGRixERa125dvq0pIwM7f//yQH3ikUco3rsPa0EBvpMmUuvJJ8maM4eMd96l1gsv4DN+XKWCurRayfr6axybN8etR49KfxCYT5/Gzs/v6iquKMpVu1QLXQX0GuRE/gmWJyzn16O/klGcwZ3Bd9Krfi82p25mV/ouJkVMYlDjQQAUbvqH5IcfRu/jQ6M/V6N3c0NaraT832MUbtiAe9++BEx9Aztv70ueM+/3FZx87jkAnNu2wfeBB9A5OyOtEqdWLSvcP3/lSlKfeZb6c+bg2rFD1b8RiqJclAroNsZgMjA7ejazD82m1FKKu4M7Pk4+pBamMrvvbCJrRSKl5OSUF3Dv3QuPO+8s31daLGTPnk3GxzOx8/am/nff4ti4cYXnkWYzCf0HIBwd8R41kszPPsdy+nT5euHsjPeIEfg8+AD2tWsDYCkoIP6uu7Bknsb99tsI+uST6/tmKIpyHhXQbVR6UTppRWmE+oVSZCpi1IpRFJuLmd9/Pp6OniQXJNPAowEOeocL9i2JieHEo48i9HYE/zwX+7p1L9gm95dfSHv1NYI++xT3Pn2wGgzafO5CIE1m8n77jbzff0fY2VHnrbfwvHsAp95+m5wff9Jmltz0D43Xr8M+IOBGvB2KoqAC+k0jLieO0StHoxM6DGYDVmmlpW9LvrrtK7ycvADINGTiYu+Cq70rJbGxJN0/BrtatWgw96fz0idWo5H4vv2w8/cneMH8i+bOjSkppL34EoZdu/C85x7ylizBa8RwfCdMIP72O/CbNAn/yU/ciOorioIK6DeVbWnb+PXorzTwbIC7vTsf7/mY+h71eb/7+yw+tpj5R+YT4hnCj3f+iKu9K4adOznx0ASEkxPOkRE4tWyJLC6h9NhRirZspf533+LaufMlzymNRk69/Ta58xeg9/am0aqV6L28OPHoo5TExNBk/XqEvf0lj2HKyKA0NhbXbt1UDxxFuQYqoN/EdqTt4PH1j1NsLkYndPSp34f1J9bTJbALM3vNRK/TY9i9m7wlSyjet5/SuDiEkxN2/v649+pJrRdfrHSAzf9jNXa1auHSOgqAgg0bSJk4icCPPsSjX7+L7le4aRMnp7yAJScHr+HDCXjtVYTDhWkiRVEuTwX0m9zBzIMsPraY0S1G08S7CQuPLOTNbW9yf4v7eb7d8+jE2QHB0mQCO7sqaSVLi4X4O/oijUa8RozAc+Dd6H18kGYzlpwcSuPjMWzdSs7P83Bs2hSXjh3I+eFHXDp0IOjjj9B7eV1zGRTlVqMC+i3o3R3v8tPhn/Bz9qN3vd4Mbzac5j7Nq/w8hr17yZz5CYZt26Ci3yUh8BoxgtovvYjOyYm8ZctIe+VV7OvXp/6sb7CvUweAou070Lm54tyqVZWUy5yTQ96SpXgNH4beza1KjqkoNYEK6Lcgi9XCmqQ1/Jn0J/+kag/cmHXHLML9w8u3yS3JZdupbezL2EdDz4bc1uA2fJx8KnV8KSWbUjfxwa4P8Hby5qvw6RT9tQFpNCHs7NC5u+HYqDGODUPQubqet2/Rjh2k/N9j6NzdqTNtGjnz5lG4fj3odPiMHYv/k5MRTk6YMzLROTpU2JK3FhVx4pFHsfPxxqN/f9x69kTn5FS+PuWJyRSsWYNDcLA24dlFum5C2aAtPz+ETk1tpNR8KqDf4jINmYxdNZYCUwFz+s7BZDXx2b7P2JiyEYnEXmePyWpCJ3R0C+zGG53ewN+l4vlg8krz2Jy6mV/jfmV72nZ8nXzJKsliaqepDG06tNJlKomJ4cTDj2iTjbm44DdxIqaTqdqNVz8/pMmENS8P4eCA56CB+DzwAI4NG5bvnzFjBlmzvkXv54fl9Gn0Pj7U+/prnENbkb9mDalPTMZz8GAKN23CWlyMz+jR2AXUxr5uXdy6d0fo9UDZAK1HH8WjX1/qvvcewk7NhqHUbCqgKyQXJDNu1TgMZgNFpiI8HDwY2Xwk3QK7EeoXSnxuPKsTV/PT4Z9ws3fjfz3/R6hfKDFZMezN2MuxnGPE5cYRmx2LRVrwdfLl4fCHGdF0BBP+nEBCXgK/D/kdT0fPSpfJeOKElha5d0T5wKWi7TvI/uEH7Pz8cGzcmNL4OPJ+W4IsLcX/ycn4TpyIMT6ehMFD8Bw0kDrTpmHYsYO0V1/Dkp9P4P8+IO3lV9D7+hKyaCHm7GxOTnkBw44d5Skhj7vvpu47/8WclcXxwUMAsGRn496vH4Hvv3fZHjuKUp1UQFcAOJpzlNc2v0a3wG6MbTUWDwePCrd56q+nSCtKw0nvRKGpEAA/Zz8aeTUi3C+cnvV6EuoXWn6z9Uj2Ee79/V6GNR3Gqx1frfJym7OzSX/7v+T//jveY8dQeuQoJbGxNFq1EjsfLUVkSksjafx4TEknQKcjeMECnMNCy48hzWYsubnkLlpE5scz8Rw0EFN6BsX79hHyyyIK/95Ixvvv49a7N3XenIadr2+V10NRqoIK6MoVyTfm8/Huj7FipUOdDrSr3Q5f50sHuHd2vMPPh3/m7W5v0z+kf5X3NZdWK+nvvENO2TzwAVPfwHvkyPO2MWVkkPrkU7h174bfpEkXPdbpL78k86OPAajz1pt4DRsGQPYPP5L+/vvonJ3xf3Iy3iNHlqdmrMXFnJwyhZLoGNDrsfPzI3DG+9gHBlZpPRXlclRAV667fGM+j/75KIeyDtE+oD33NLmHYznHiM2JpaFnQ25vcDsR/hHndaHclraNdUnreDTiUfycLz9zo5SS7NlzKD1yhDpvTy8Ptlcj+8efsORk4/fEE+d9+JQmJJD+1lsUbdmKS8eOBH74P/RubqQ8/gSFGzdq8+bY6Slctx6nFi2o//2caypHTWBMScG+Tp0K65H9ww/k/raE+t99e9mJ3pQbQwV05YawWC0sPraYj/d8TL4xHzudHcEewSTlJ2GymvB39qdXvV50DuzMyoSV/JmkPWGplnMt/tfrf0T4R1RzDTRSSvJ+/ZVT/5mGnb8/Ti1bULBmLQFTp+I98l4AcpcsIe3Fl/B/9hn8Hn646s5tsWDYuQuXdm1vyAdF8f79JN47EseWLQh49VVcWp+dstmwZy9JY8aAxYJ7374EfvThRb95SYuFoq3bcGnb5rzeRldKms2UHI7FKbSVGlF8EZcK6Egpq+VfmzZtpHJzyi3JlYdOH5Il5hIppZQFpQXy9/jf5dN/PS3b/dROhs4JlW1+bCO/2PeFPJBxQPb9pa+M/CFS/nr012ou+fkM+/fLo926y5hmzWXGp5+et85qtcrkyU/KmNAwWRwdfd660uRkadi375LHNqalyVPvvSeTJjwszQWF5cuzvv9exjRrLpMeeECasrIueQxTRoa0lJZeYa3Ol/riS/JwVGt5tEdPGdOsuUx55llZEh8vzbm58mivXvJYn9tk+gf/kzHNmsvcZcsuepzMr76WMc2ay6M9e8ncJUuk1WK5qvKkv/9+2bmWX3I7S0mJzF6wQJaeOHFV57FlwC55kbiqWujKDVVqKWVvxl4auDegjps2qCivNI/n/36erWlb+b/I/2Ni+MQa0zozZ2ZSfOAAbr17X1Amc04OxwcNRu/jQ8jiXxB6vTYl8d0DMaWmErLkt/KuliVHj5K3eDGWgkIsWVkUbt4MVitYrfg//TR+jz6CNJmI69sXodNjzshA7+OD7yMPa+cVOjwHDypv/RqTk7Wpj+3scO3SBdfOnbAPqoddrVqUxh2jaMsWzOkZ1Hl7Ova1alVYN0tBAce6dcdz4EBqv/gCp7/6muwffkCWlGAfGIjp1CmCf56LU6tWJN0/htK4OBr88D2OzZqd12fflJpKfP8BOIeFaTN2RkfjFBZGnelv4dS06XnnLE04TuYnM3Ht0AGvESPOO44xMZH4uweClOhdXWn4+/IKH6dYeuwYqc8+R+nRo2Bvj/eIEXiPHo3Q65AWKw4N6l/XbzcFGzaQ9eVXuN9+O56DBt7wB72olItS45msJqZumcqy+GUMajSINrXbUGoppZ57PTrX7VweTDMMGWxM2ag94SknjgdCH6B3/d7nHcsqrSw4soDfE37nP53+Q2Pvs4OKpJRV+mGRt2IFJ599jjrT38Jr6FByFi7k1OtvIBwccGrRggY/z8WUlkbivSOxFhai9/FB7+aKS8dO+IwbS/qbb2HYt4/Ga9dQuGEDJ5+fQtAXn2NXqxapTz6FKSWl/Fx+jz2G/xOPA3DqrenkLFiA1+BBFP69EXNGxnnl0nl4IE0mHOrXp8GPP6D3uLBHU868eZz6zzSCFy0q7xFkzs4me/YcchYswP+x/8Nn3DhA62J6fPAQrAYDOnd3nFtH4T95Ms6tWpH8+OMUbd5CoxW/YxcQQP7vv5P+zrtYCgrwe/RR3Lp1RefhQcGfazj92WdIqxXMZpwjIgiY9h+cmjUDIHnS/2HYvp2gL78g+eFHcO3alaBPPwGrldKjRymJPUJJTAy5Cxeic3Oj9ksvYdi5k9xffgGLpbxeTi1bUuft6Tg1v/jIaFN6OkX//EPxgYMY4+PR+/hgHxSEc2QE7n36XPQDwZyVRcKAu5FGI9aiItDr8RwwAP9nnsG+di0shUXkLf4FY3IKeg8P9D4+uN9+O/a1K/5QvRrXHNCFEP2AjwE9MEtK+c6/1j8DTADMQCbwoJQy6VLHVAFd+TcpJTP3zmTWwVnnLQ/zC2Nsq7FsO7mNZfHLMFlNuDu4Y6/T+osvG7ysvP97WmEar215je1p27HT2eHl6MWcfnMIcgviu0Pf8dWBr3CzdyPIPYgeQT2YEDbhmgK8lJKkkaMwnkyl4dKlHB88BPu6dfEefR8nn5+C3//9H/mrV2POzCR4/rzzBkcBFB+KJnHYMPwef5yCNWvAaiFk6VKEToc0GjGfPo1wcODUf6ZRtHkzjdatRQjBsV698ejXj7r/fRspJeaTJzGdOoXp1CkcgoJwCg3FsH07Jx6diHN4OHXenIYxIQHz6Sw8BvRH7+ZGwj33gISQXxdf8B5U9MFnTEmlaOsWSg5FU7B2LZacHNx696Jw7TpqPfcsvhMmlG9rzs4mffrb5K9Ycd4x3O+4g9qvvoJh61Yt6Ofm4t6vLy6t25A+fXr5cbK+m03Ge+/h0q4dJbGxWAsKABCOjrh170bAG2+Ut4yNiYkYdu9B2NthKSjg9BdfYsnNxWv4MO3bid4O1/btcI6MBLQW9snnnsdaWIjO3R3Hxo2x5OZiSk1FGo04NGiAzwMPgBCUHj2KcHDA79FH0Ht5kfLkUxSuX0/Ir4tBryd3wQJyfp6HsLfHvW9fCtatw5qfj87NTQv4UoK9PZ533YXnkME41KuHXe3a1zSA7ZoCuhBCDxwFbgdSgJ3AKCllzDnb9AK2SykNQohJQE8p5b2XOq4K6MrFnCo6hUVacNQ7sjFlI1/u/5K0ojQc9Y4MbjyYUc1H0dCzIbHZsYxcMZJhTYbxWqfXiMmK4dE1j1JqKeX5ds/TulZrHvjjARztHAl0C2R3+m56BvXEx9mHuNw4DmQe4P4W9zOl3ZRrCurF+/aROHIUDsHBGBMTafDjDzi3bas9DvCvv8DenvqzZuHaoX2F+yc//jiFG/4Gs5k6//0vXkMGX7BNaXw8CXcPxGf8ePQeHmR+9BEhS5fi1KzphQc8R/6qVaQ+8+x58+zY1amDz5gxZLz3HgFvvI73qFFXXGdLfj6ZH31Ezrz5ODRqSMNff61wBs2SI0cxn0rDkpeHnb8/rp06nT1Gbi5Z335Hzs8/Yy0qwr5BfRouX47OwQFpsZD86ESMSUm4duyAS/sOOLVqVal0iiU3l/R33iVv2TItrVXGtWtXnFo0J2vWtzi1aEGd//4XxyaNy9M+0mKhYO06Tn/1JaUxhwHQubhgNRrRe3ri0f8ucn74Ef9nnsHvkbM3wo1JSaS/9z6F69bhfvtt+D7yCM5hYUiLBeOJE+TM/ZncX39FGgzaDno9Aa+/jve9I674fYdrvCkKdAJWn/P6JeClS2wfBWy+3HHVTVGlskrNpfLv5L9lpiHzgnXvbH9Hhs0Jkz/F/CQ7ze0kb190uzyee7x8/eGsw7LTz51k+5/ayyXHlkir1Sql1G5qvrP9HRk6J1S+v+P98uVXK+WZZ7WbmY88Ur7MeCpdJgwbfsmbiVJKWRwbq91Q7N5DWi9xkzN1ygvycHiEPNKps0x6aEKly1a4dZvMXrBAGvbvl0U7d8q4u/rLmGbN5eHIKGnOz6/0cSpSEhcnjenp13QMc26uzJoz54Kby9fKarVKq8kkzXl58vSsWfJIx07ajd+nn5EWg+GS+xkOHZKlycnSarHI4sOHZcLQYTKmWXOZMHyEtJpMFe9nNF70mOa8PFnwzz8ye+FCmf7RR9Jw4MBV14truSkqhBgG9JNSTih7PQboIKV8/CLbfwqcklK+VcG6R4BHAOrXr98mKemSWRlFuawiUxEDlwwkw5BBfff6zLpjVvnN1jPSCtPQ6/TUcjk/jyml5O3tbzP/yHw61+3MM22eoZlPs6sqhyktjVNvTafWs89ckFapjOwffsC+fn3ce/a86DbGlBTi77wLTKZKPZjkYqylpWTNmoWdr195N8xbgbWoiOLoaFzatbvib2TSbCZ/1R+4tG9XPk1FdbnWlEulA7oQ4n7gcaCHlLL0UsdVKRelqmw9uZX5sfN5teOrF51U7GKklPx0+Ce+3P8lBcYCRjQbwSsdXqkxvWz+LePDjyiJjqbeN1/X2DIq19elAnplMvOpQL1zXgeVLfv3SW4DXqESwVxRqlKnup3oVLfT5TesgBCCMS3HMLDRQD7Z+wkLjiygoWdD7mtxXxWXsmrUevqp6i6CUoNVZgLonUATIUSIEMIBGAksO3cDIUQU8BUwUEqZUcExFKVG83T05JUOr9AtsBsf7PqAuJy46i6SolyxywZ0KaUZLY2yGjgMLJRSRgshpgkhBpZt9j7gBiwSQuwTQiy7yOEUpcYSQjCtyzTcHNyYsmkKpRb1RVOxLWpgkaL8y8aUjTy27jFa+bbi2bbP0i6g3XnrpZSYrCYc9OpB18qNp0aKKsoVWpmwkv/t/h/phnTC/cPxdPBEIskqziIxPxGL1cLUzlO5u9Hd1V1U5RZzrTdFFeWWc1fDu+hdvzfzY+ez5sQaskuykUh8nX1pU7sNMVkxvPzPyxSZihjZ/Oy87Barhf2Z+zlRcAJ7nT3Ods50qtsJZzvnaqyNcqtQLXRFuQqlllKe2/AcG1I20COoB16OXlikhS0nt5Bdkn3ets28mzGz90zqutWtptIqNxOVclGU68BkNfHejvfYlraNEksJFquFNrXb0Kd+H1r5tcJitXA05yhTt0zFTmfH9K7T6VinI/Z6e6Kzovls72fsydhD73q9GdZ0GBH+Eeh1tv2wDOX6UwFdUarR8bzjTF4/mcT8ROx0dtRzr8fxvON4OnrSuU5nNqZupMhUBIBO6LDX2aMTOvRCj7eTN428GtHEqwkjm488b7RrkakIJ71T+YeA2Wrm0OlDRGdFcyznGNkl2fSq14s7gu/A1d71vDKdyD+BWZpp6Hnlo1qV6qUCuqJUM4PJwMaUjRzOPsyxnGOE+4dzf4v7cXNww2AysPbEWlILUzFZTJitZizSgkVayDRkEp8bT2J+Ii72LrzQ7gXa1G7DVwe+Ynn8cpzsnAj1C8XDwYPtadvJN+YD4OXohbOdc/nDvoc1HcYTUU/gYu/CxpSNPPf3c1illXe6vcNtDW4DICYrhiPZRwhyDyLYI/iKRt1KKTmWe4y4nDgS8xNp7tP8gmmNlaqhArqi2Lik/CRe3/w6ezL2IBDY6+y5p8k9AOzP3E9uaS4d6nSgS2AXovyjylvy+zP3s/jYYpbELSHILYg7Q+7ku0Pf0dS7KfZ6ew5mHmRixESOZB9hffL6887ZpW4Xnmv7HI29G2OymDiUdQg/Zz/qudc7b7vTxad5c+ubF+z/dte3VS+g60AFdEW5CVillV+O/sLJwpOMaj6K2q6VnyRq56mdvL75dVIKU+gS2IUPenyATuh4adNLrDuxDjd7N8a2Gkvf4L6cKjrFodOHmBM9B4PJQIR/BIezD1NsLgagsVdjOtXthIudC2armUVHF1FiLmFS5CR6BPUgwDWApzc8za5Tu/io10fohI450XMoMBYwpd2UC/r1X8y6pHW4O7jTvk7F0w7fqlRAVxQFg8nAtrRtdAvqVv5wEIvVwqbUTUTViip/SMgZuSW5fL7/c3an7yaqVhSd6nTiZNFJ1p9Yz77MfVisFiSS1rVa80bnN87LxxeZinho9UNEZ0UDEOAagF7oSS1MZVTzUbTwacHJopMYTAaCPYNp5NmI5j7NcbF3wWQx8e7Od1lwZAE6oeOVDq8wotkIrNLKP6n/kJCbgI+zD75OvjTzaYaf88UfAZdXmseSuCUk5SdRaCpEILi70d10qdvlgsnN0ovSScxPpJl3M7ycvC56TKu0kpSfRLG5GKPFSFPvprjYu1ywXYm5hH9S/6Fz3c4Vrr9aKqArinJdyEs80i+7JJsPd39I+4D29Avph8liYubemcw9PBcAgcBB71A+xYKdzo4I/whKzCVEZ0UzruU4EvIS2JS6iSGNh3Ao6xDHco5dcJ5At0AaeDSgxFxCsbkYfxd/QjxCKLGUsCx+GcXmYnycfHB3cKfAWEB2STaNPBsxrtU4BjQcgL3eng3JG3hp00sUmgoBqOtal9a1W9M+oD2RtSIJcgvCTmfH+uT1fLr3U+Jyz871E+wRzDd3fEOAa0D5e/JX8l+8t/M9UgtTqe1Sm5c6vESf+n04XXya2OxYGno2vOpurCqgK4pSY6QUpCCRBLgEoNfpOVV0irjcOHal72LbyW1kGDJ4scOL9Avuh9lq5q1tb7H42GIaejZkQtgEugd1J780n3RDOtFZ0ezP3E9aYRou9i446h1JN6STmJeIFSv9Q/ozttVYmnprT3YyWUz8kfgHP8T8QGx2LHVc69AuoB3L4pfR0rclkyImcTzvOIdOH2JX+q7yMQU6ocPL0YvskmyCPYIZ03IM/s7+FJoKeXv723g4ePD5bZ8TlxvH/Nj57ErfRSPPRoxtNZafDv/EsZxjeDl6kVuaC8CL7V9kdIvRV/X+qYCuKIrNklKSkJdAiGcIOlGZCWK1VJLRarzoCF0pJZtPbubrA1+zN2MvQxoP4ZWOr+Codzxvm/jceKKzokkuSCatKI22tdtyd6O7sdOdHWQfkxXDxDUTySnNAbTW/f0t72dk85HY6+wxWU3Mj53PkewjNPNpRnOf5rTwaYGbg9tVvR8qoCuKolRASkmGIYNaLrWu6YEhCXkJ/BTzE73q9aJz3c7XdYCYCuiKoig3iUsF9Mp9f1EURVFqPBXQFUVRbhIqoCuKotwkVEBXFEW5SaiAriiKcpNQAV1RFOUmoQK6oijKTUIFdEVRlJtEtQ0sEkJkAklXuJsfcPo6FKc6qLrUTKouNdfNVJ9rqUsDKWWFTx+ptoB+NYQQuy42QsrWqLrUTKouNdfNVJ/rVReVclEURblJqICuKIpyk7C1gP51dRegCqm61EyqLjXXzVSf61IXm8qhK4qiKBdnay10RVEU5SJUQFcURblJ2ERAF0L0E0IcEULECSFerO7yXAkhRD0hxF9CiBghRLQQ4smy5T5CiDVCiGNl/3tXd1krSwihF0LsFUL8XvY6RAixvez6LBBCOFR3GStLCOElhPhFCBErhDgshOhkq9dGCPF02e/YISHEPCGEk61cGyHEd0KIDCHEoXOWVXgdhGZmWZ0OCCFaV1/JL3SRurxf9jt2QAjxmxDC65x1L5XV5YgQou+1nLvGB3QhhB74DLgTaAmMEkK0rN5SXREz8KyUsiXQEXisrPwvAuuklE2AdWWvbcWTwOFzXr8LfCilbAzkAA9VS6muzsfAH1LK5kAEWr1s7toIIQKByUBbKWUooAdGYjvXZg7Q71/LLnYd7gSalP17BPjiBpWxsuZwYV3WAKFSynDgKPASQFksGAm0Ktvn87KYd1VqfEAH2gNxUsoEKaURmA8MquYyVZqUMk1Kuafs5wK0gBGIVofvyzb7HhhcLQW8QkKIIKA/MKvstQB6A7+UbWJLdfEEugPfAkgpjVLKXGz02gB2gLMQwg5wAdKwkWsjpdwIZP9r8cWuwyDgB6nZBngJIerckIJWQkV1kVL+KaU0l73cBgSV/TwImC+lLJVSHgfi0GLeVbGFgB4IJJ/zOqVsmc0RQgQDUcB2oLaUMq1s1SmgdnWV6wp9BEwBrGWvfYHcc35Zben6hACZwOyyFNIsIYQrNnhtpJSpwAzgBFogzwN2Y7vXBi5+HWw9JjwIrCr7uUrrYgsB/aYghHADFgNPSSnzz10ntb6jNb7/qBBiAJAhpdxd3WWpInZAa+ALKWUUUMS/0is2dG280Vp7IUBdwJULv/bbLFu5DpcjhHgFLQ0793oc3xYCeipQ75zXQWXLbIYQwh4tmM+VUv5atjj9zNfEsv8zqqt8V6ALMFAIkYiW+uqNloP2KvuaD7Z1fVKAFCnl9rLXv6AFeFu8NrcBx6WUmVJKE/Ar2vWy1WsDF78ONhkThBDjgQHAaHl2AFCV1sUWAvpOoEnZ3XoHtBsIy6q5TJVWlmP+FjgspfzfOauWAePKfh4HLL3RZbtSUsqXpJRBUspgtOuwXko5GvgLGFa2mU3UBUBKeQpIFkI0K1vUB4jBBq8NWqqloxDCpex37kxdbPLalLnYdVgGjC3r7dIRyDsnNVMjCSH6oaUqB0opDeesWgaMFEI4CiFC0G707rjqE0kpa/w/4C60O8PxwCvVXZ4rLHtXtK+KB4B9Zf/uQss9rwOOAWsBn+ou6xXWqyfwe9nPDct+CeOARYBjdZfvCuoRCewquz5LAG9bvTbAf4BY4BDwI+BoK9cGmIeW+zehfXN66GLXARBoPd/igYNoPXuqvQ6XqUscWq78TAz48pztXymryxHgzms5txr6ryiKcpOwhZSLoiiKUgkqoCuKotwkVEBXFEW5SaiAriiKcpNQAV1RFOUmoQK6oijKTUIFdEVRlJvE/wPpCI79hDWu4QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.arange(1,len(total_train_acc)+1), total_train_acc, label = 'Train Acc')\n",
    "plt.plot(np.arange(1,len(total_val_acc)+1), total_val_acc, label = 'Validation Acc')\n",
    "plt.plot(np.arange(1,len(total_train_loss)+1), total_train_loss,label = 'Train Loss')\n",
    "plt.plot(np.arange(1,len(total_val_loss)+1),total_val_loss, label= 'Validation loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a571112",
   "metadata": {},
   "source": [
    "### Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "29a4c778",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a97a821f8a75463691321428512f66f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# define batch size for testing\n",
    "test_batch_size = 64\n",
    "# create dataloader for validation set\n",
    "test_triplet_dataset = FaceDataset(df = val_df, num_sample = 3000)\n",
    "test_triplet_dataloader = DataLoader(test_triplet_dataset, batch_size=test_batch_size, shuffle=True, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1bd1254c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "776cb706467f4747b30f8798acbef54c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/47 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 0.931420972644377\n"
     ]
    }
   ],
   "source": [
    "# define path\n",
    "weights_path = './weights/best_triplet.pth'\n",
    "\n",
    "triplet_model.to(device)\n",
    "# load model weights\n",
    "triplet_model.load_state_dict(torch.load(weights_path))\n",
    "\n",
    "overall_acc = list()\n",
    "\n",
    "# set mode for testing\n",
    "triplet_model.eval()\n",
    "with torch.no_grad():\n",
    "    for anchor_img, pos_img, neg_img in tqdm(test_triplet_dataloader):\n",
    "\n",
    "        threshold_list = np.full(anchor_img.shape[0], model_config['margin'])\n",
    "        is_same = np.full(anchor_img.shape[0], 1)\n",
    "\n",
    "        pos_dist_list = list()\n",
    "        model_output = list()\n",
    "\n",
    "        anchor_img = anchor_img.to(device)\n",
    "        pos_img = pos_img.to(device)\n",
    "        neg_img = neg_img.to(device)\n",
    "        # compute the out in eval mode\n",
    "        anchor_img, pos_img, neg_img = triplet_model(anchor_img, pos_img, neg_img)\n",
    "        # compute loss in eval mode\n",
    "        loss = criterion(anchor_img, pos_img, neg_img)\n",
    "\n",
    "        distance_positive = (anchor_img - pos_img).pow(2).sum(1)\n",
    "        distance_negative = (anchor_img - neg_img).pow(2).sum(1)\n",
    "        # print((distance_negative-distance_positive>0).sum()*1.0 / distance_negative.size()[0])\n",
    "        # print(distance_positive.cpu().data.numpy())\n",
    "        acc = calculate_accuracy(threshold_list,distance_positive.cpu().data.numpy(),is_same)\n",
    "        overall_acc.append(acc)\n",
    "    \n",
    "print(f'Train Accuracy: {np.array(overall_acc).mean()}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "99f9ce39",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a3895a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}