PCa16TileSquare.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import cv2\n",
    "import time\n",
    "import copy\n",
    "import pickle  # Log dictionary data\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "import torchvision.transforms as T\n",
    "import torchvision.models as models\n",
    "\n",
    "import multiprocessing\n",
    "# We must import this explicitly, it is not imported by the top-level\n",
    "# multiprocessing module.\n",
    "import multiprocessing.pool\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import cohen_kappa_score,confusion_matrix\n",
    "from tqdm.auto import tqdm\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from datetime import datetime\n",
    "from multiprocessing import Manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CFG:\n",
    "    batch_size = 16\n",
    "    debug = False\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.float32\n",
    "    epochs = 10\n",
    "    lr = 1e-4\n",
    "    model_name = 'resnet'\n",
    "    num_classes = 6\n",
    "    nworkers = 3\n",
    "    nfolds = 4\n",
    "    seed = 524\n",
    "    TRAIN = '../yi_data/panda-16x128x128-tiles-data/train/'\n",
    "    LABELS = '../data/train.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>data_provider</th>\n",
       "      <th>isup_grade</th>\n",
       "      <th>gleason_score</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0005f7aaab2800f6170c399693a96917</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000920ad0b612851f8e01bcc880d9b3d</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0018ae58b01bdadc8e347995b69f99aa</td>\n",
       "      <td>radboud</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001c62abd11fa4b57bf7a6c603a11bb9</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001d865e65ef5d2579c190a0e0350d8f</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           image_id data_provider  isup_grade gleason_score  \\\n",
       "0  0005f7aaab2800f6170c399693a96917    karolinska           0           0+0   \n",
       "1  000920ad0b612851f8e01bcc880d9b3d    karolinska           0           0+0   \n",
       "2  0018ae58b01bdadc8e347995b69f99aa       radboud           4           4+4   \n",
       "3  001c62abd11fa4b57bf7a6c603a11bb9    karolinska           4           4+4   \n",
       "4  001d865e65ef5d2579c190a0e0350d8f    karolinska           0           0+0   \n",
       "\n",
       "   split  \n",
       "0      2  \n",
       "1      3  \n",
       "2      1  \n",
       "3      1  \n",
       "4      0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(CFG.LABELS).set_index('image_id')\n",
    "files = sorted(set([p[:32] for p in os.listdir(CFG.TRAIN)]))\n",
    "train = train.loc[files].reset_index()\n",
    "\n",
    "if CFG.debug:\n",
    "    df = train.sample(n=50, random_state=CFG.seed).copy()\n",
    "else:\n",
    "    df = train.copy()\n",
    "\n",
    "# Generate train/validation sets containing the same distribution of isup_grade\n",
    "splits = StratifiedKFold(n_splits=CFG.nfolds, random_state=CFG.seed, shuffle=True)\n",
    "splits = list(splits.split(df,df.isup_grade))\n",
    "# Assign split index to training samples\n",
    "folds_splits = np.zeros(len(df)).astype(np.int)\n",
    "for i in range(CFG.nfolds):\n",
    "    folds_splits[splits[i][1]] = i\n",
    "df['split'] = folds_splits\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/yasufuminakama/panda-se-resnext50-regression-baseline\n",
    "class TrainDataset(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, df, labels, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed; otherwise, validation set only get cancerous samples\n",
    "        self.df = df\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        #worker = torch.utils.data.get_worker_info()\n",
    "        #worker_id = worker.id if worker is not None else -1\n",
    "        #start = time.time()\n",
    "        # https://stackoverflow.com/questions/33369832/read-multiple-images-on-a-folder-in-opencv-python\n",
    "        img_fns = [fn for fn in glob.glob(f\"{CFG.TRAIN}/{self.df['image_id'][idx]}_*.png\")]\n",
    "        imgs = [cv2.imread(fn) for fn in img_fns]\n",
    "        # (D,W,H)\n",
    "        img = cv2.hconcat([cv2.vconcat([imgs[0], imgs[1], imgs[2], imgs[3]]),\n",
    "                           cv2.vconcat([imgs[4], imgs[5], imgs[6], imgs[7]]),\n",
    "                           cv2.vconcat([imgs[8], imgs[9], imgs[10], imgs[11]]),\n",
    "                           cv2.vconcat([imgs[12], imgs[13], imgs[14], imgs[15]])])\n",
    "        \n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "            \n",
    "        label = torch.tensor(self.labels[idx])\n",
    "        #end = time.time()\n",
    "        return img, label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transforms(phase):\n",
    "    assert phase in {'train', 'val'}\n",
    "    \n",
    "    if phase == 'train':\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.8776, 0.8186, 0.9090],\n",
    "                std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])\n",
    "    else:\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.8776, 0.8186, 0.9090],\n",
    "                std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use fold idx as validation set\n",
    "def data_loader(fold_idx):\n",
    "    train_idx = df[df['split'] != fold_idx].index\n",
    "    val_idx = df[df['split'] == fold_idx].index\n",
    "\n",
    "    train_dataset = TrainDataset(df.loc[train_idx].reset_index(drop=True),\n",
    "                                 df.loc[train_idx].reset_index(drop=True)['isup_grade'],\n",
    "                                 transform = get_transforms(phase='train'))\n",
    "    val_dataset = TrainDataset(df.loc[val_idx].reset_index(drop=True),\n",
    "                               df.loc[val_idx].reset_index(drop=True)['isup_grade'],\n",
    "                               transform = get_transforms(phase='train'))\n",
    "    \n",
    "    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=CFG.nworkers)\n",
    "    val_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.nworkers)\n",
    "    return train_loader, val_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, fold, dataloaders, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    # Send the model to GPU/CPU\n",
    "    model = model.to(device=CFG.device)\n",
    "    \n",
    "    train_acc_history = []\n",
    "    val_acc_history = []\n",
    "    loss_history = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()   # Set model to training phase\n",
    "            else:\n",
    "                model.eval()    # Set model to evaluate phase\n",
    "            \n",
    "            avg_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            print(' ', end='', flush=True)  # To workaround tqdm issue in multiprocess\n",
    "            for inputs, labels in tqdm(dataloaders[phase],\n",
    "                                       desc='[{}] {}/{}({:5s})'.format(fold, epoch+1,num_epochs,phase)):\n",
    "                inputs = inputs.to(device=CFG.device, dtype=CFG.dtype)\n",
    "                labels = labels.to(device=CFG.device, dtype=torch.long)\n",
    "                \n",
    "                # Zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # Forward, track history if only in training\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    loss = criterion(outputs, labels)\n",
    "                    \n",
    "                    pred = torch.argmax(outputs, 1)\n",
    "\n",
    "                    # Backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                    \n",
    "                # Statistics\n",
    "                avg_loss += loss.item()*(inputs.size(0)/len(dataloaders[phase].dataset))  # len(dataloaders[phase].dataset) not len(dataloaders[phase])\n",
    "                running_corrects += torch.sum(pred == labels)\n",
    "            \n",
    "            # End of epoch\n",
    "            with torch.no_grad():\n",
    "                epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "\n",
    "                if phase == 'val':\n",
    "                    val_acc_history.append(epoch_acc)\n",
    "                    # deep copy the model\n",
    "                    if epoch_acc > best_acc:\n",
    "                        best_acc = epoch_acc\n",
    "                        best_model_wts = copy.deepcopy(model.state_dict())\n",
    "                    # Apply lr_scheduler\n",
    "                    if scheduler is not None:\n",
    "                        scheduler.step(avg_loss)\n",
    "                else:\n",
    "                    train_acc_history.append(epoch_acc)\n",
    "                    loss_history.append(avg_loss)\n",
    "                print('[{}] {} Loss: {:4f} Acc: {:4f}'.format(fold, phase, avg_loss, epoch_acc))\n",
    "    \n",
    "    time_elapsed = time.time() - since\n",
    "    print('[{}] Training complete in {:.0f}m {:0f}s'.format(fold, time_elapsed//60, time_elapsed%60))\n",
    "    print('[{}] Best val Acc: {:4f}'.format(fold, best_acc))\n",
    "    print()\n",
    "    \n",
    "    model.load_state_dict(best_model_wts)\n",
    "                      \n",
    "    return model, loss_history, train_acc_history, val_acc_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Multiprocessing\n",
    "\"\"\"\n",
    "class NoDaemonProcess(multiprocessing.Process):\n",
    "    # make 'daemon' attribute always return False\n",
    "    def _get_daemon(self):\n",
    "        return False\n",
    "    def _set_daemon(self, value):\n",
    "        pass\n",
    "    daemon = property(_get_daemon, _set_daemon)\n",
    "\n",
    "# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool\n",
    "# because the latter is only a wrapper function, not a proper class.\n",
    "class MyPool(multiprocessing.pool.Pool):\n",
    "    Process = NoDaemonProcess\n",
    "\n",
    "def train_fn(fold):\n",
    "    model_ft = initialize_model(CFG.model_name, CFG.num_classes, use_pretrained=True)\n",
    "    \"\"\"\n",
    "    optimizer = optim.SGD(model_ft.parameters(),\n",
    "                          lr=CFG.lr,\n",
    "                          momentum=.9,\n",
    "                          nesterov=True)\n",
    "    \"\"\"\n",
    "    optimizer = optim.Adam(model_ft.parameters(),\n",
    "                           lr=CFG.lr)\n",
    "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=2, verbose=True, eps=1e-06)\n",
    "    #print(f'### FOLD: {fold} ###', flush=True)\n",
    "    loader_train, loader_val = data_loader(fold)\n",
    "    best_model, loss_history, train_acc_history, val_acc_history = train_model(model_ft, fold, {'train': loader_train, 'val': loader_val}, F.cross_entropy, optimizer, scheduler, CFG.epochs)\n",
    "\n",
    "    return best_model, loss_history, train_acc_history, val_acc_history\n",
    "\n",
    "def progressor(fold):\n",
    "    best_model, loss_history, train_acc_history, val_acc_history = train_fn(fold)\n",
    "    return {f'best_mode_{fold}': best_model,\n",
    "            f'loss_history_{fold}': loss_history,\n",
    "            f'train_acc_history_{fold}': train_acc_history,\n",
    "            f'val_acc_history_{fold}': val_acc_history}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet18\n",
    "Removed Kappa score\n",
    "log: resnet_05_28_2020_14_56.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "\n",
    "def initialize_model(model_name, num_classes, feature_extract=False, use_pretrained=False):\n",
    "    \"\"\"\n",
    "    Params:\n",
    "        feature_extract\n",
    "            True - fine tunning\n",
    "            False - fix the model\n",
    "    \"\"\"\n",
    "    model_ft = None\n",
    "    \n",
    "    if model_name == 'alexnet':\n",
    "        \"\"\"AlexNet\n",
    "        \"\"\"\n",
    "        model_ft = models.alexnet(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.classifier[6].in_features\n",
    "        model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
    "    elif model_name == 'resnet':\n",
    "        \"\"\"Resnet\n",
    "        \"\"\"\n",
    "        model_ft = models.resnet18(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.fc.in_features\n",
    "        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n",
    "    \n",
    "    return model_ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d2e580c308740f58bfc1c5aa938d187",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 1/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a747c305a0f423cbb7570c9fb067951",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 1/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e78bccfac0b647aeb7c6a1f803a96bab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 1/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "780da36cdedf42b38038532afe97d7e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 1/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 1.338491 Acc: 0.471535\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36acec91d83d40c48613e104fcecd188",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 1/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 1.341296 Acc: 0.466844\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1a6baaa265043a4bab3250473de8291",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 1/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 1.338499 Acc: 0.470014\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1dd34ce98f9f49e4b90772249653d151",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 1/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 1.323348 Acc: 0.473944\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b2187e4ab7e4957a291893e6aab4210",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 1/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.208988 Acc: 0.526056\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45cefa8cb4cb4e5d83da8cc86b6e065c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 2/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.357924 Acc: 0.477748\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "055403fd995a4a37babb5cac6b227c8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 2/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.198091 Acc: 0.535565\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd9ffe788172411aacd10c0f0cb0f543",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 2/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.363848 Acc: 0.462153\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0af8aafe423a4d3388f06511169ef49d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 2/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 1.122401 Acc: 0.561177\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "889bb632e2924a949c87b4b303e5fce3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 2/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 1.125063 Acc: 0.561811\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7253d59595d948ccaa37d9a74dca1a61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 2/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 1.105884 Acc: 0.569038\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3290b556100b499c8c362d32d4cbfd7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 2/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 1.123466 Acc: 0.564980\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dfdb471babf4030b4ba675880acc71c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 2/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.300040 Acc: 0.514644\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e7d6106f3a24c2881c193dae291b64b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 3/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.267590 Acc: 0.526056\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "797e5fa03a694f3c9fa4d5ea5c5c0d45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 3/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.279471 Acc: 0.529859\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14c28bc67ff2409896dd8e409a8269ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 3/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.268421 Acc: 0.522252\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e42b2fc282b4cd6a8b657f84697d23d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 3/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.902674 Acc: 0.647902\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c1de4cad0a044be890066623cf67ff7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 3/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.914870 Acc: 0.650311\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3874eaca89e4bc0b23123460ceef3e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 3/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.917547 Acc: 0.646507\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0304e4902caf4ca3a09d4974227dd8e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 3/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.895487 Acc: 0.651959\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20563509401449ce9af6d7e40c6c952e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 3/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.400616 Acc: 0.478129\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "578247c1e1af45cc8fbdaa2cf93c4743",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 4/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.269948 Acc: 0.527577\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc314e8d93b549958d629102336c14ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 4/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.206071 Acc: 0.540890\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5eee5c981db49b7a4990acab2f69c30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 4/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.237144 Acc: 0.547737\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d4414303aaf4c6fada9da7be9a0138a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 4/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.571096 Acc: 0.800558\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2e3974cd4a34766bfa0c3d92f5d08e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 4/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.572071 Acc: 0.793711\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bace1d49bd540118c5da17278039eeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 4/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.570317 Acc: 0.797388\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "278af4370f09419481f9f837f2fe6eda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 4/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.562791 Acc: 0.800178\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c904967ad7864cc8b9c3044f09cdb103",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 4/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     4: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[0] val Loss: 1.864150 Acc: 0.461773\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37f215f7dc23414badfc841826cc2531",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 5/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.425383 Acc: 0.481172\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b0f884b2c4e4a9599457de48183af36",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 5/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.422291 Acc: 0.523393\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b350fe160e4343bc8e0e0d96c00472d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 5/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     4: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[2] val Loss: 1.486392 Acc: 0.479270\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a69e3e78455418a9c251181f4f66691",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 5/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.214307 Acc: 0.958159\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4423fc06785b4459ae249aa0ea555d4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 5/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.200394 Acc: 0.963991\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "856e57b9ba9447439ba9e536de899c64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 5/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.233109 Acc: 0.935971\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bdb382fd90c475fbabfe2195c6537bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 5/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.225960 Acc: 0.938126\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09e03904e72b42f3b00dc3a59a4d850f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 5/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.176915 Acc: 0.572461\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73d3726497f348b3991331984aa422bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 6/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.235979 Acc: 0.572841\n",
      " \n",
      "Epoch     5: reducing learning rate of group 0 to 1.0000e-05.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81bd1340247c45de875d17a10a825ae6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 6/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] val Loss: 1.867550 Acc: 0.413465\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecf3d1fa9627481db741e1d770323232",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 6/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     5: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[3] val Loss: 1.451567 Acc: 0.541651\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3e6f7580a774f41be52b172ce17bdc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 6/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.133184 Acc: 0.985799\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "958de2569864462b97f322ef281d3489",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 6/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.081508 Acc: 0.989730\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "001e42a1174a4ea5a1b66ba63fb5b236",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 6/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.125396 Acc: 0.986941\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "258f36c0f4e942e58f0f0de5a44b954c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 6/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.082149 Acc: 0.989603\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb82185e1056415d9ca601f938aa67cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 6/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.204324 Acc: 0.580068\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4c91604e0b6499cb1ac922b553ac87f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 7/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.227698 Acc: 0.564473\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed45f096896040c9b5f8fe3d8110160d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 7/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.337478 Acc: 0.565614\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85e4a5ce9fae44f2ba46ec39bbe4fe55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 7/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.346345 Acc: 0.555344\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ef35eaeedc84604bdd1df0afe489074",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 7/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.097714 Acc: 0.992519\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "356a481503694f0c926a17ae3828acb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 7/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.050512 Acc: 0.996577\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b850d2c9c28e4effaec351130e46711c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 7/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.087825 Acc: 0.994675\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18f7a796e6a9450589f98076f3730ee4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 7/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.052260 Acc: 0.996069\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b17a4ed85284b918d352eb78c8c15b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 7/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.348474 Acc: 0.559909\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88dce5e695b140908b2b0350d5284540",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 8/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.374197 Acc: 0.561811\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "378440c64a114265812f6532047fb716",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 8/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.230687 Acc: 0.573602\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "959ebb73720c429cb1ba7fe261ad7562",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 8/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     7: reducing learning rate of group 0 to 1.0000e-06.\n",
      "[2] val Loss: 1.252564 Acc: 0.569038\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "974f62a949874dcdb85ace4ce2e39a5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 8/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.070488 Acc: 0.996957\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69abb440e9924d33a477f19f9e1d09be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 8/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.035679 Acc: 0.999112\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "752df7750d614a66a6cfad7fd77b7e1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 8/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.038387 Acc: 0.997464\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65776b8fabeb477ba616ed08002fcb52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 8/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.066915 Acc: 0.997971\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d305af62b25a4a62bc54e781edea4806",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 8/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     8: reducing learning rate of group 0 to 1.0000e-06.\n",
      "[3] val Loss: 1.351137 Acc: 0.561050\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87e2f1d4f7bf485a80729d46a1a871a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 9/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     8: reducing learning rate of group 0 to 1.0000e-06.\n",
      "[0] val Loss: 1.271008 Acc: 0.565234\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ed1bc815ead4257aaaaee9774dd2dca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 9/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     8: reducing learning rate of group 0 to 1.0000e-06.\n",
      "[1] val Loss: 1.346503 Acc: 0.561811\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da79a7397a424aa7ab7ae9e3c69e2983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 9/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.260358 Acc: 0.550019\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62f1f9aac6174583ae049e1019f12eaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 9/10(train)', max=493.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.057438 Acc: 0.998225\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5353ea587826490b91f1e18de49a200b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 9/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.031085 Acc: 0.998859\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fccf53988964307b11eb73de6b185af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 9/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.030233 Acc: 0.998859\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "414b996addfb425f9374910073c86110",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 9/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.064523 Acc: 0.997591\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5484a9b5317c4180bf631ca9976169fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 9/10(val  )', max=165.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.246147 Acc: 0.571320\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d36e67ebeb5c4a69957c856cb3916305",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 10/10(train)', max=493.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] val Loss: 1.387475 Acc: 0.567897\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69aae5cca5d64bfa96059ff813a23fa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 10/10(train)', max=493.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] val Loss: 1.353571 Acc: 0.556485\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ee0a92a1b0344c6a64ae9a4d4fc503d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 10/10(train)', max=493.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] val Loss: 1.262616 Acc: 0.565614\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e9649f24b774d80aee3ab91a27c6b80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 10/10(train)', max=493.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] train Loss: 0.051443 Acc: 0.999239\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15347728fb76415896994c82aeae9e27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[0] 10/10(val  )', max=165.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[3] train Loss: 0.031416 Acc: 0.998352\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a667542af2f844ca8bfbf41e426435b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[3] 10/10(val  )', max=165.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1] train Loss: 0.028014 Acc: 0.999493\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53d786eb30a3471f8e59123ac8d5f33b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1] 10/10(val  )', max=165.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2] train Loss: 0.061911 Acc: 0.998352\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb16270a09234b87aaf1b27d28717652",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[2] 10/10(val  )', max=165.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[0] val Loss: 1.246720 Acc: 0.574363\n",
      "[0] Training complete in 450m 26.997792s\n",
      "[0] Best val Acc: 0.580068\n",
      "\n",
      "\n",
      "[3] val Loss: 1.401558 Acc: 0.566755\n",
      "[3] Training complete in 450m 28.760445s\n",
      "[3] Best val Acc: 0.567897\n",
      "\n",
      "\n",
      "[1] val Loss: 1.374391 Acc: 0.548117\n",
      "[1] Training complete in 450m 29.200766s\n",
      "[1] Best val Acc: 0.561811\n",
      "\n",
      "\n",
      "[2] val Loss: 1.267853 Acc: 0.567516\n",
      "[2] Training complete in 450m 32.738552s\n",
      "[2] Best val Acc: 0.572841\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_dict = {'batch_size': CFG.batch_size,\n",
    "            'epochs': CFG.epochs,\n",
    "            'learning_rate': CFG.lr,\n",
    "            'model': CFG.model_name,\n",
    "            'nworkers': CFG.nworkers,\n",
    "            'nfolds': CFG.nfolds,\n",
    "            'random_seed': CFG.seed}\n",
    "\n",
    "nfold = range(CFG.nfolds)\n",
    "result_list = list(MyPool(CFG.nfolds).map(progressor, nfold))\n",
    "\n",
    "# Accumulate result from each process\n",
    "for result in result_list:\n",
    "    log_dict.update(result)\n",
    "\n",
    "# Log results\n",
    "log_file = f'{CFG.model_name}_{datetime.now().strftime(\"%m_%d_%Y_%H_%M\")}.pkl'\n",
    "with open(log_file, 'wb') as pkl_file:\n",
    "    pickle.dump(log_dict, pkl_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfor fold in range(CFG.nfolds):\\n    best_model, loss_history, train_acc_history, val_acc_history = train_fn(fold)\\n    log_dict[f'best_mode_{fold}'] = best_model\\n    log_dict[f'loss_history_{fold}'] = loss_history\\n    log_dict[f'train_acc_history_{fold}'] = train_acc_history\\n    log_dict[f'val_acc_history_{fold}'] = val_acc_history\\n\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "for fold in range(CFG.nfolds):\n",
    "    best_model, loss_history, train_acc_history, val_acc_history = train_fn(fold)\n",
    "    log_dict[f'best_mode_{fold}'] = best_model\n",
    "    log_dict[f'loss_history_{fold}'] = loss_history\n",
    "    log_dict[f'train_acc_history_{fold}'] = train_acc_history\n",
    "    log_dict[f'val_acc_history_{fold}'] = val_acc_history\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(log_dict['loss_history_0'], 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(log_dict['train_acc_history_0'], '-o')\n",
    "plt.plot(log_dict['val_acc_history_0'], '-o')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "\n",
    "plt.tight_layout(pad=3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAADyCAYAAABAr8L8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xU9Zn48c8zk8mFEAgQrgkQUOQiCmi0KNVFUUGrQl0v2Oqvtd2y3dZW3X1561q1/vpbbW21unVVam3tynpZFKWt5aqFWm+AIPebKBICJFwSbrnn+f1xzpBJMhMmkJkzyXneL/LKuc8zh8x55ny/5/v9iqpijDHGvwJeB2CMMcZblgiMMcbnLBEYY4zPWSIwxhifs0RgjDE+Z4nAGGN8Ls3rANoqLy9PCwsL27RP+dFadh+sora+gVAwQL9umeR2CSUmQGOMSUErVqzYq6q9o63rcImgsLCQ5cuXx739Gyt3cu/ra8irrT+2LBQKct81ZzBtXH4iQjTGmJQjIttjrev0RUOPzt9EZUQSAKisrefR+Zs8isgYY1JLp08EJeWVbVpujDF+0+kTwYDcrDYtN8YYv+n0ieDOycPJCgWbLMsKBblz8nCPIjLGmNTS4SqL2ypcIfzo/E2UlFcyIDeLOycPt4piY4xxdfpEAE4ysAu/McZE1+mLhowxxrTOEoExxvicL4qGUsUbK3daXYUxJuVYIkiScAvncOO2neWV3Pv6GgBLBsYYT1nRUJJYC2djTKqyRJAk1sLZGJOqLBEkibVwNsakKksESWItnI0xqSphiUBEnheRUhFZG2O9iMiTIrJVRFaLyFmJiiUVTBuXz8PXnEF+bhYC5Odm8bB1hW2MSQGJfGro98CvgT/EWH85MMz9+RLwtPu707IWzsaYVJSwOwJVXQrsb2WTqcAf1PEBkCsi/RMVjzHGmOi8rCPIB3ZEzBe7y1oQkRkislxElpeVlSUlOGOM8QsvG5RJlGUabUNVnQnMBCgqKoq6jYmftXA2xkTyMhEUAwMj5guAEo9i8Q1r4WyMac7LoqG5wP9xnx4aD1So6i4P4/EFa+FsjGkuYXcEIvISMBHIE5Fi4AEgBKCqzwBvAVcAW4GjwC2JisU0shbOxpjmEpYIVPXG46xX4PuJen0T3YDcLHZGuehbC2dj/MtaFvuMtXA2xjRn3VD7jI3hbIxpLq5EICK3Ab8DDgHPAeOAe1R1QQJjMwliLZyNMZHiLRr6lqoeBC4DeuNU7D6SsKiMMcYkTbyJINz46wrgd6r6CdEbhBljjOlg4k0EK0RkAU4imC8iOUBD4sIyxhiTLPFWFn8bGAtsU9WjItITe+7fGGM6hXjvCM4DNqlquYjcBNwHVCQuLGOMMckSbyJ4GjgqImOAu4DtxB5nwBhjTAcSbyKoc1sCTwWeUNUngJzEhWWMMSZZ4q0jOCQi9wI3AxeISBC33yBjjDEdW7x3BDcA1TjtCXbjDCDzaMKiMsYYkzRx3RGo6m4RmQWcIyJXAh+pqtURmJNiA+QYkxriuiMQkeuBj4DrgOuBD0Xk2kQGZjq38AA5O8srURoHyHlj5U6vQzPGd+KtI/h34BxVLQUQkd7AImB2ogIznVtrA+TYXYExyRVvHUEgnARc++LZV0SmiMgmEdkqIvdEWf9NESkTkVXuzz/FGY/p4GyAHGNSR7x3BPNEZD7wkjt/A84IYzG5TxY9BVyKMz7xMhGZq6rrm236iqre2oaYTSdgA+QYkzriuiNQ1TuBmcCZwBhgpqrefZzdzgW2quo2Va0BXsZph2CMDZBjTAqJe2AaVX0NeK0Nx84HdkTMFwNfirLdP4rIhcBm4A5V3dF8AxGZAcwAGDRoUBtCMKnKBsgxJnW0mghE5BCg0VbhDDvcrbXdoyxrfqw/Ai+parWIfBd4Abi4xU6qM3HuSCgqKooWj+mAbIAcY1JDq4lAVU+mG4liYGDEfAFQ0uz4+yJmfwP87CRezxhjzAlI5OD1y4BhIjJERNKB6cDcyA1EpH/E7NXAhgTGY4wxJoqEDV6vqnUiciswHwgCz6vqOhF5CFiuqnOBH4rI1UAdsB/4ZqLiMcYYE504nYp2HEVFRbp8+XKvwzDGmA5FRFaoalG0dYksGjLGGNMBWCIwxhifs0RgjDE+Z4nAGGN8zhKBMcb4XMIeHzWmo7ABcozfWSIwvhYeICc8NkJ4gBzAkoHxDSsaMr7W2gA5xviFJQLjazZAjjFWNGR8LpUGyLG6CuMVuyMwvpYqA+SE6yp2lleiNNZVvLFyZ9LjmPDI2wy5589MeOTtpL++8YYlAuNr08bl8/A1Z5Cfm4UA+blZPHzNGUn/Jp4KdRWpkoxM8lnRkPG9VBggJxXqKlpLRsk+P6lSTJYqcSSaJQJjUkAq1FWkQjKC1HmkN1XiCMeSyIRkRUPGpIBUqKuIlXSSXXGeCsVkqRRHMorsEpoIRGSKiGwSka0ick+U9Rki8oq7/kMRKUxkPMakqlSoq0iFZASpc2eSKnEkIyElrGhIRILAU8ClOOMXLxORuaq6PmKzbwMHVPVUEZmOM2bxDYmKyZhU5nVdRfi1vS4TT4VislSKIxkJKZF1BOcCW1V1G4CIvAxMBSITwVTgQXd6NvBrERHtaMOmGdNJeJ2MwLkziSybB2/uTFIljmQkpEQWDeUDOyLmi91lUbdR1TqgAujV/EAiMkNElovI8rKysgSFa4xJBalQTJZKcSSjyC5hYxaLyHXAZFX9J3f+ZuBcVf1BxDbr3G2K3flP3W32tXLcMmD7CYaVB+w9wX07IzsfTdn5aGTnoilPz0cgq1vPYNee+RJMS9f6upr6w/t3NlQe3N/GwwxW1d7RViSyaKgYGBgxXwCUxNimWETSgO5Aq28u1huJh4gsjzV4sx/Z+WjKzkcjOxdNdfbzkciioWXAMBEZIiLpwHRgbrNt5gLfcKevBd62+gFjjEmuhN0RqGqdiNwKzAeCwPOquk5EHgKWq+pc4LfAf4vIVpw7gemJiscYY0x0CW1ZrKpvAW81W3Z/xHQVcF0iY2hmZhJfqyOw89GUnY9Gdi6a6tTnI2GVxcYYYzoG62LCGGN8zjeJ4HjdXfiJiAwUkXdEZIOIrBOR27yOyWsiEhSRlSLyJ69j8ZqI5IrIbBHZ6P6NnOd1TF4SkTvcz8laEXlJRDK9jqm9+SIRRHR3cTkwCrhRREZ5G5Wn6oB/U9WRwHjg+z4/HwC3ARu8DiJFPAHMU9URwBh8fF5EJB/4IVCkqqNxHnzpdA+1+CIRENHdharWAOHuLnxJVXep6sfu9CGcD3rn62Q9TiJSAHwFeM7rWLwmIt2AC3Ge6ENVa1S13NuoPJcGZLltnbrQsj1Uh+eXRBBPdxe+5Pb4Og740NtIPPUr4C6gwetAUsBQoAz4nVtU9pyIZHsdlFdUdSfwC+ALYBdQoaoLvI2q/fklEUiUZb5/XEpEugKvAber6kGv4/GCiFwJlKrqCq9jSRFpwFnA06o6DjgC+LZOTUR64JQeDAEGANkicpO3UbU/vySCeLq78BURCeEkgVmq+rrX8XhoAnC1iHyOU2R4sYi86G1InioGilU1fIc4Gycx+NUlwGeqWqaqtcDrwPkex9Tu/JII4unuwjdERHDKgDeo6mNex+MlVb1XVQtUtRDn7+JtVe103/jipaq7gR0iEu7achJNu473my+A8SLSxf3cTKITVp77YsziWN1deByWlyYANwNrRGSVu+xHbktwY34AzHK/NG0DbvE4Hs+o6ociMhv4GOdpu5V0wlbG1rLYGGN8zi9FQ8YYY2KwRGCMMT5nicAYY3zOEoExxvicJQJjjPE5SwTGJJGITLQeTk2qsURgjDE+Z4nAmChE5CYR+UhEVonIs+54BYdF5Jci8rGILBaR3u62Y0XkAxFZLSJz3P5pEJFTRWSRiHzi7nOKe/iuEf39z3JbrBrjGUsExjQjIiOBG4AJqjoWqAe+DmQDH6vqWcAS4AF3lz8Ad6vqmcCaiOWzgKdUdQxO/zS73OXjgNtxxsYYitPS2xjP+KKLCWPaaBJwNrDM/bKeBZTidFP9irvNi8DrItIdyFXVJe7yF4D/FZEcIF9V5wCoahWAe7yPVLXYnV8FFALvJv5tGROdJQJjWhLgBVW9t8lCkR832661/llaK+6pjpiuxz6HxmNWNGRMS4uBa0WkD4CI9BSRwTifl2vdbb4GvKuqFcABEbnAXX4zsMQd36FYRKa5x8gQkS5JfRfGxMm+iRjTjKquF5H7gAUiEgBqge/jDNJyuoisACpw6hEAvgE8417oI3vrvBl4VkQeco9xXRLfhjFxs95HjYmTiBxW1a5ex2FMe7OiIWOM8Tm7IzDGGJ+zOwJjjPE5SwTGGONzlgiMMcbnLBEYY4zPWSIwxhifs0RgjDE+Z4nAGGN8zhKBMcb4nCUCY4zxOUsExhjjc5YIjDHG5ywRGGOMz1kiMMYYn7NEYIwxPmeJwBhjfM4SgTHG+JwlAmOM8TlLBMYY43OWCIwxxucsERhjjM9ZIjDGGJ+zRGCMMT5nicAYY3zOEoExxvicJQJjjPE5SwTGGONzlgiMMcbnLBEYY4zPJSwRiMjzIlIqImtjrBcReVJEtorIahE5K1GxGGOMiS0tgcf+PfBr4A8x1l8ODHN/vgQ87f5uVV5enhYWFrZPhMaYlFR+tJbdB6uorW8gFAzQr1smuV1CFsdJxLFixYq9qto72rqEJQJVXSoiha1sMhX4g6oq8IGI5IpIf1Xd1dpxCwsLWb58eTtGakxqeGPlTh6dv4mS8koG5GZx5+ThTBuX78sY7n19DXm19ceWhUJB7rvmjKTG0tniEJHtsdYl8o7gePKBHRHzxe6yVhOBMZ1R+MNe6X7Yd5ZXcu/rawCSctFRVV5bUcx9b66lqrbhWAx3v7aavYeruXhEH+oblLoGjfjdQF29M1+v7rL6yG0anHVN9onY192+Xpuun/Xh9mPnIayytp4fzVnDu1v3ogqK4v5DVd3fEfPuNqo0nXa3I2K+QbXJvrjbLPt8P9V1DS3iuGv2amZ9uL3lPhEx0CSmxjgat2t8naaxNR4r/P6+2H+U+gZtEcej8ze129+Gl4lAoizTKMsQkRnADIBBgwYlMiZjkq6qtp6H/7Ih6sXvgbnrOHC0htr6Bmrr1f3tTNfUOdN17vKaiHW19Q3U1DkX2/B002NEHsuZj6a6roGf/nkDP/3zhoSeg4BAWiBAMCAtzkPY0Zp63tu6FxHn0iHi/iDubxBpNk3TbWi2PBCxPU22p0USCKupbyAtEGjx+mHS7DiR8+HLXuO6iPjd6fCVMbzvZ3uPRI2jpLwynlMbFy8TQTEwMGK+ACiJtqGqzgRmAhQVFUX/izUmhdQ3KPuP1FB2qJqyw9WUHqyi7HC1Mx/+cecPVdXFPE5FZS0/+eP6JsuCASEUFELBAOnBAKFggFCaMx8KREwHA2SFgnTLTDs2H94vlBbe15lPCwZ4cvGWmHE8MX0swYCQFhCCgYD7u/En7dhv52KeFnTXiTSZP7Y+cn8RAoHGK+mER95mZ5SLXH5uFn+/5+IT+N84Ma3F8dKM8UmL4+PtB6LGMSA3q91ew8tEMBe4VURexqkkrjhe/UAstbW1FBcXU1VV1a4BpprMzEwKCgoIhZJfWdWZxVsurqocqq5rejGPuKBHzu87XE1DlK8sXTPS6J2TQe+uGYzs140Lh2XQOyeD3yzdRnllbYvt+3XL5C+3XUAozb1oBwJNLprt6bUVxTEvfFPHJq9M/M7Jw5sUkwFkhYLcOXl40mLwWxwJSwQi8hIwEcgTkWLgASAEoKrPAG8BVwBbgaPALSf6WsXFxeTk5FBYWHjstrGzUVX27dtHcXExQ4YM8TqcTiNa2fydsz9h3trd5OWkU3aomtKIi3y04oJQUMjrmkGfnAwG5GYyZmB3end1LvDHfrpmkpeTTpf06B+5/NysqB/2ey4fQY/s9MS8+WZS5cIXTsJeV1r7KQ4JV050FEVFRdr8qaENGzYwYsSITpsEwlSVjRs3MnLkSK9D6TTG/8didh+MfifZMzs9ygW95Xz3rFC7fEtPlSd2vI7BJIaIrFDVomjrvCwaaledPQmAP95jshysquXZJZ/GTAICfPzjS5Ma07Rx+Z5fdFMhBpN81sVEOygvL+e//uu/2rzfFVdcQXl5eQIiMrFU1dbzm6XbuPDn7/DUO5+SFQpG3a49K+KMSXWd5o6gLdr79jecCL73ve81WV5fX08wGP1CA/DWW2+d8GuatqlvUF77uJhfLdxMSUUVF57Wm7smD2dr6eGUKBc3xku+SwSJaLhzzz338OmnnzJ27FhCoRBdu3alf//+rFq1ivXr1zNt2jR27NhBVVUVt912GzNmzAAaW0kfPnyYyy+/nC9/+cu899575Ofn8+abb5KVZd9KT5aqsnD9Hh6dv4ktpYcZU9CdX1w/hvNPyQNgdH53wPsKQWO81Gkqi8MVqD/54zrWlxyMuf/KL8qpqW/55Ed6MMC4QblR9xk1oBsPXHV6zGN+/vnnXHnllaxdu5a//vWvfOUrX2Ht2rXHnu7Zv38/PXv2pLKyknPOOYclS5bQq1evJong1FNPZfny5YwdO5brr7+eq6++mptuuqnFa0W+V9O6ZZ/v55G/bGTF9gMMzcvmzsnDmTK6n9W1GF/yRWVxvKIlgdaWn4hzzz23ySOeTz75JHPmzAFgx44dbNmyhV69ejXZZ8iQIYwdOxaAs88+m88//7zd4vGbjbsP8ui8TSzeWEqfnAz+46tncH1RAWlBqxIzJpq4EoGIvAY8D/xFVdvvipkArX1zh9ZbC77yz+e1SwzZ2dnHpv/617+yaNEi3n//fbp06cLEiROjNnzLyMg4Nh0MBqmsbL/m435RfOAojy3czJyVO+makcZdU4Zzy/lDyEqPXU9jjIn/juBpnAZfT4rI/wK/V9WNiQsrcRLRaCYnJ4dDhw5FXVdRUUGPHj3o0qULGzdu5IMPPjjh1zHR7T9Sw6/f3sqLH2wHgRkXDOVfJp5CbpfkNMQypqOLKxGo6iJgkYh0B24EForIDuA3wIuq2rJtfIpKRCu9Xr16MWHCBEaPHk1WVhZ9+/Y9tm7KlCk888wznHnmmQwfPpzx45PXR0lnd6S6jt+++xkzl27jaE0d1509kNsvHUb/7lbJbkxbxF1ZLCK9gJuAm3E6h5sFfBk4Q1UnJirA5o5XWdzZ+em9xlJT18DLy77gycVb2Xu4mstG9eWuKcM5tU+O16EZk7JOurJYRF4HRgD/DVwV0TncKyJio8SYpGhoUP64uoRfLtjMF/uPcu6Qnjx789mcPbiH16EZ06HFW0fwa1V9O9qKWBnGmPaiqizdspefz9vIupKDjOiXw+++eQ4Th/e2R0GNaQfxJoKRIvKxqpYDiEgP4EZVbXu/Csa0wSc7yvnZvI289+k+Cnpk8fgNY5g6Jj9hXTEb40fxJoLvqOpT4RlVPSAi3wEsEZiE2FZ2mF8s2MRba3bTKzudB64axde+NIiMNHsU1Jj2Fm8iCIiIuAPNIyJB4LjP5onIFOAJIAg8p6qPNFs/CHgByHW3uUdVrQMeH9tzsIpfLdrCq8t3kJkW4LZJw/jOhUPpmuG7to/GJE28n675wKsi8gzOuMLfBea1toObLJ4CLsUZlnKZiMxV1chx9+4DXlXVp0VkFM5gNYVtewumo4rs/K9f90xGDejG37fupb5BuXn8YG69+FTyumYc/0DGmJMSbyK4G/hn4F9wumpfADx3nH3OBbaq6jYAd0jKqUBkIlCgmzvdnRhjFnc2Xbt25fDhw16H4anmnf/tqqhiV0UVZw/K5fEbxjGoVxePIzTGP+JtUNaA07r46TYcOx/YETFfjDM2caQHgQUi8gMgG7ikDcc/catfhcUPQUUxdC+ASffDmdcn5aWN4+fzNjZp3R22+2C1JQFjkizedgTDgIeBUUBmeLmqDm1ttyjLmrdeuxGnu4pfish5wH+LyOjm/RmJyAxgBsCgQYPiCTm21a/CH38ItW5fPhU7nHk44WRw9913M3jw4GPjETz44IOICEuXLuXAgQPU1tby05/+lKlTp55c7J1AbX0Dr60opqQi+shgJVH6gTLGJFa8RUO/wxl8/nHgIpx+h473/F4xMDBivoCWRT/fBqYAqOr7IpIJ5AGlkRup6kxgJjgti1t91b/cA7vXtBLVMqivbrqsthLevBVWvBB9n35nwOWPRF8HTJ8+ndtvv/1YInj11VeZN28ed9xxB926dWPv3r2MHz+eq6++2rfPvdfVN/DmqhKefHsL2/cdJRQUautb/lfayGDGJF+8iSBLVRe7Tw5tBx4Ukb/hJIdYlgHDRGQIsBOYDnyt2TZfAJOA34vISJy7jbI2vYO2ap4Ejrc8DuPGjaO0tJSSkhLKysro0aMH/fv354477mDp0qUEAgF27tzJnj176Nev3wm/TkcUbg38xOItbCs7wukDuvHbbxRxsLKWH81ZayODGZMC4k0EVSISALaIyK04F/Y+re2gqnXutvNxHg19XlXXichDwHJVnQv8G/AbEbkDp9jomxpv50extPLNHYDHRzvFQc11Hwi3/PmEX/baa69l9uzZ7N69m+nTpzNr1izKyspYsWIFoVCIwsLCqN1Pd1YNDcr8dbt5fNFmNu85zPC+OTxz01lMPr1xYBgRsZHBjEkB8SaC24EuwA+B/4tTPPSN4+3ktgl4q9my+yOm1wMT4g22XUy6v2kdAUAoy1l+EqZPn853vvMd9u7dy5IlS3j11Vfp06cPoVCId955h+3bt59k4B2DqrJoQymPLdzMhl0HOaV3Nv954zi+ckb/Fq2Bp43Ltwu/MSnguInAbQ9wvareCRzGqR/ouMIVwu381NDpp5/OoUOHyM/Pp3///nz961/nqquuoqioiLFjxzJixIh2CD51qSpLNpfx+MLNfFJcQWGvLjx+wxiuHpNP0LqDMCalHTcRqGq9iJwd2bK4wzvz+oQ8LrpmTWMldV5eHu+//37U7TpTGwJV5b1P9/HYws2s2H6Agh5Z/Pwfz+Sas/JtaEhjOoh4i4ZWAm+6o5MdCS9U1dcTEpXpED7cto9fLtzMR5/tp3/3TP7fV0dz3dkDSU+zBGBMRxJvIugJ7AMujlimgCUCH1qx/QCPL9zMu1v30icng59cfTo3nDOQzJB1CGdMRxRvy+KOXS9g2sXq4nIeW7iZv24qo1d2Ovd9ZSQ3jR9sCcCYDi7elsW/o2WrYFT1W+0e0QlS1U7fWMurKpr1JQd5fNFmFq7fQ26XEHdPGcH/OW8w2dYjqDGdQryf5D9FTGcCXyWFOojLzMxk37599OrVq9MmA1Vl3759ZGZmHn/jdrJ5zyF+tWgzb63ZTU5mGv966WncMqGQnMxQ0mIwxiRevEVDr0XOi8hLwKKERHQCCgoKKC4upqwssY2SvZaZmUlBQUHCX+fTssM8uXgLcz8pITs9jR9efCrfvmAo3bMsARjTGZ3ovf0w4CR7f2s/oVCIIUOGeB1Gh7d93xGeXLyVOSuLyUgL8t1/OIUZFwylR/ZxxyAyxnRg8dYRHKJpHcFunDEKTAcUOSDMgNwsvv3lQjbvOczsFcUEA8K3JgzhuxNPsUFhjPGJeIuGchIdiEmO5gPC7Cyv5KE/bSAocNP4wXzvolPp2y159RDGGO/F1fJHRL4qIt0j5nNFZFriwjKJ8rMYA8Lk5WTwk6mjLQkY40Px1hE8oKpzwjOqWi4iDwBvJCYs0552VVSyeEMpizfsYVeMAWFKD554N9zGmI4t3kQQ7c7BHiJPUQ0NytqSCha5F/91JQcBGNSzC9kZQY5Ut7wjsAFhjPGveC/my0XkMeApnErjHwArjreTiEwBnsAZj+A5VW0xWICIXI8zdrECn6hq88FrTBwqa+r5+9a9LN64h8UbSik9VE1A4KxBPbh7ygguGdmHU/t05c1VJU3qCMAGhDHG7+JNBD8Afgy84s4vAO5rbQe3++qngEtxhq1cJiJz3TEIwtsMA+4FJqjqARFpdbAb09Seg1XHinze3bqX6roGumakceFpeUwa0ZeLRvShZ7NHP8P9/9uAMMaYsHifGjoC3NPGY58LbFXVbQAi8jIwFVgfsc13gKdU9YD7OqUtjmKOUVXWlRxk0QbnW/+anRUADOyZxY3nDuKSkX05d0jP4/b+aQPCGGMixduOYCFwnaqWu/M9gJdVdXIru+UDkWNCFgNfarbNae7x/o5TfPSgqs6LM3ZfqKqt571P97JoQylvbyhl98EqxC3yuWvKcC4Z2Zdhfbp22q41fGX1q+0+YJIx8Yi3aCgvnAQA4izGiXZlat5rWhpOK+WJQAHwNxEZHflaACIyA5gBMGhQyjRoTpjSg1W8vbGURRtKeXdrGVW1DWSnB7nwtN5MGtmXi4b3ppc19mo/qXABXv1q0yFUK3Y485DcWFLhXKSSVDkfCY4j3kTQICKDVPULABEpJEpvpM0UAwMj5gto2VFdMfCBqtYCn4nIJpzEsCxyI1WdCcwEKCoq6rCjpDVv0Rsumw8X+SzeUMrijXtYXewU+eTnZnFD0UAmjezLl4b2JCPNuntud225ADc0QH1N409ddbP5Gqivjph25+tqYu8Xnl41q+k42uDM/+kO2L0agukQCEEw5EwHI6ajLQ+E59MhmNY4HUiLsTyYOskIUuMCnCrnIwlxSDxdG7tP/8wElriLLgRmqOr8VvZJAzYDk4CdOBf3r6nqumbHvVFVvyEieTgjoY1V1X2xjltUVKTLly8/bsyppnmLXoD0YICiwlw+23uUXRVOkc/YgblcMrIvk0b2YXjfHCvySbTHRsHBnS2XSxCy85pe1Btq2/e1Iy/KVeWxt0vLcl67oa59X78JIeZ3u0AI+p0BaRlOrGmZkJYOwQxnWVqGOx1jWVqmu1/k8laOteGP8Od/bZoYQ1lw5a9g1DT3/6IO6mud81JfA/V1UaZrI7apjbJf8/XNple9CDVHWp6PtCw45SLQBmiod35rvTutEdPttLyusmUMAN0Hwh1r4/8fFlmhqkXR1sVbWTxPRIpwimdWAW8CMaI7tk+diNwKzMcp/39eVdeJyKzwKTAAAAuFSURBVEPAclWd6667TETWA/XAna0lgY7s0fmbWrToralv4P1P93PZ6X2549LTuGh4H3rnJKHIJxW+bXlJFba/B6v+J3oSAOdDeNqUxgtW+Cd80To2nd54kYs23do+gYg7vMdHO9/0mov8sDc0uBeymoiLWk3jxS/u5ZEXxojppY9GPxcNtdClp3P3UlcFVRXunYx7t1NXFXF3k6CGibWVMOefnZ9EOnYnlRY9CYBzYS7fASLO/6EEQQKN04E052+g+fJY20sAAoHYy9/7z+hxVBS329uOt7L4n4DbcIp3VgHjgfdpOnRlC6r6FvBWs2X3R0wr8K/uT6dUeqiKBev2sLM8dt589uaoSToxUuV21wvlX8AnLzvFMAc+h/QcCGVDbZQPfPeBcPWTyYtt0v1N/1/A+RY86f7G+UAAAm4ySYRPXo6djG56reXyaFSbFnmFk0eTZVWNxWh11S2TysIfxz7+pAciir7SmhWNhRqnj91tRZuOLFKLOEbAvViHtZac/+Xd+M5He1j3Row42q9L+njrCG4DzsEpz79IREYAP2m3KBItyd+Aiw8cZf66Pcxbu4vl2w+gCmkB4Qr+xl1przJA9lKiefy87npWdLv0xF9IFWoOQ9VB51taVQVUR0w3mXd/f/4358MYqbYSFv+kcyaCmqNOUcOqF+Gzpc6yIf8AE38EI6+EjX8+/gU4GcLn3ss7tXiS0fGINBbxnKiPZsa+AF+QxO+M7XE+Okgc8SaCKlWtEhFEJENVN4pIx2iKmqRvwNvKDvOXtbuZv273screEf1yuH3SaVx+Rj8qPpzF6BXPkSXORbhA9vKz0HNsPK037Bvc8oLdfP7YsvKm22hD64EFMyCzO2R2c343TwJhFcUw+1swbDKceglk92q3c5N0qvDFB843/3VvQM0h6FEIF/07jJkOuRFPnqXCBTgyFi+Tcaqci1S5AKfK+UhCHPFWFs8BbgFuxykOOgCEVPWKdoskTm2uLI51e5fRHc77vlshU+dW0LiVM8em6xorcRrceXcbbajjcGUNew8dZf+ho1RV1xKUBnLSA/TqEqBHZpCMoDYeb9+WE6vsy3Av4OHfkRf1Y8si1mVEbJPRDULNehONdT5CXSA9G46UAQIF58Cwy+C0y6DfmU1vmVNV+Q5Y/bJT9r9/m1Psc/pXYezXYPD5HeM9GIff67ESoLXK4rgSQbOD/QPQHZinqjG+XiZOmxPBg7kc90lXCTrlg4E0dzrQWOlzrAIoiAaCVNfDoRqlolqpqoMGAmRmpNM9O5Pc7Ewy0tMbj3fsGAGneCKWac9Ev8hn5DStUGwPze+QwPm2ddWTMPpa2LUSNi+ALQug5GNnfU5/GHapkxiGTnTiShU1R53inVUvwrYlgELhBTD26zDyKsjo6nWExqSEdk0EXmu3O4LuBXDbaqdmvpVvivUNyrLP9zPPLfbZVVFFWkA4/9Q8ppzej8tO7xvfSF7xPBWSLPF+2zpcClsWwpb58Ok7TlFUIASFE5wipNMmQ69Tkhs7OEU/xctg5Yuwbo4TV+4g5+I/ZrpTDGSMacLfiaC1b8AxbjVr6hp4f9s+5q3dxYJ1e9h3pIaMtAAXntaby0f3Y9KIvnTv0saB3E8gjpRSX+uUu2+Z79wx7N3kLO851E0Kl8HgCSdXSXg8FTsbi372bXWKs0ZNc4t+Jjh3XsaYqPydCCCub8BVtfUs3VzGvLW7WbRhDwer6shOD3LRiD5cPro/E4f3JjvjJIdg6Ezlngc+d+4WNs93nsapr3bK5IdOdJLCsMug24CTf53aSrfo539g2ztOfc3gCc7Ff9TU1CqmMiaF+T4RxOra4XB1HW9vLGX+2t28s6mUozX1dM8KccnIvlw+uh9fHpZHZsi6dTiumqNOMtji1i2Ei8D6nuEmhclQUBR/fYcqFC93nvpZ+zpUVzhFaGNuhLE3Onchxpg28XUiiNa1QygonNa3K1tKj1BT10Be13QuO70fl4/ux/ihvQgFrYjhhKlC6YbGIqQdHzpPWmX1dB5LHXYZnDrJaana/A7p/B86jbtW/Q/s3ew05R811fn2X3iBFf0YcxJ8nQgmPPJ21Fa9QYFvnD+EKaP7cfbgHgQD9mhhQlQegE/fdpLC1oVwdJ9TQd9jiNPSN1r/PYPOc4t+pjlPURljTtpJ9zXUkZXE6NqhQeH+q0YlORofyuoBo//R+Wmoh5KVTr3Cu49HTwI5/eFbNiSFMcnU6e+1Yw3KboO1eyAQdOoKLv732I3rDu1ObkzGmM6fCO6cPJysZhW+Nlh7CojVYVY7dqRljIlPp08E08bl8/A1Z5Cfm4XgDPby8DVn2Ji9Xpt0v9OOIpIX/ckYYzpeZbGIlAHbT3D3PGBvO4bT0Xl6PvK6SM8BOZIfCpBe20BNySHdufeo7vcqHuzvI5Kdi6Y6w/kYrKq9o63ocIngZIjI8li15n5k56MpOx+N7Fw01dnPR6cvGjLGGNM6SwTGGONzfksEM70OIMXY+WjKzkcjOxdNderz4as6AmOMMS357Y7AGGNMM75JBCIyRUQ2ichWEbnH63i8JCIDReQdEdkgIutE5DavY/KaiARFZKWI/MnrWLwmIrkiMltENrp/I+d5HZOXROQO93OyVkReEpHM4+/VsfgiEYhIEHgKuBwYBdwoIn7uaKgO+DdVHQmMB77v8/MBcBuwwesgUsQTOEPRjgDG4OPzIiL5wA+BIlUdDQSB6d5G1f58kQiAc4GtqrrNHWf5ZWCqxzF5RlV3qerH7vQhnA+6b5tai0gB8BXgOa9j8ZqIdAMuBH4LoKo1qlrubVSeSwOyRCQN6AKUeBxPu/NLIsgHIgcMLsbHF75IIlIIjAM+9DYST/0KuAto8DqQFDAUKAN+5xaVPSci2V4H5RVV3Qn8AvgC2AVUqOoCb6Nqf35JBNEGG/D941Ii0hV4DbhdVQ96HY8XRORKoFRVV3gdS4pIA84CnlbVccARwLd1aiLSA6f0YAgwAMgWkZu8jar9+SURFAMDI+YL6IS3d20hIiGcJDBLVV/3Oh4PTQCuFpHPcYoMLxaRF70NyVPFQLGqhu8QZ+MkBr+6BPhMVctUtRZ4HTjf45janV8SwTJgmIgMEZF0nMqeuR7H5BkREZwy4A2q+pjX8XhJVe9V1QJVLcT5u3hbVTvdN754qepuYIeIhPtpnwSs9zAkr30BjBeRLu7nZhKdsPK8049QBqCqdSJyKzAfp9b/eVVd53FYXpoA3AysEZFV7rIfqepbHsZkUscPgFnul6ZtwC0ex+MZVf1QRGYDH+M8bbeSTtjK2FoWG2OMz/mlaMgYY0wMlgiMMcbnLBEYY4zPWSIwxhifs0RgjDE+Z4nAmCQSkYnWw6lJNZYIjDHG5ywRGBOFiNwkIh+JyCoRedYdr+CwiPxSRD4WkcUi0tvddqyIfCAiq0Vkjts/DSJyqogsEpFP3H1OcQ/fNaK//1lui1VjPGOJwJhmRGQkcAMwQVXHAvXA14Fs4GNVPQtYAjzg7vIH4G5VPRNYE7F8FvCUqo7B6Z9ml7t8HHA7ztgYQ3FaehvjGV90MWFMG00CzgaWuV/Ws4BSnG6qX3G3eRF4XUS6A7mqusRd/gLwvyKSA+Sr6hwAVa0CcI/3kaoWu/OrgELg3cS/LWOis0RgTEsCvKCq9zZZKPLjZtu11j9La8U91RHT9djn0HjMioaMaWkxcK2I9AEQkZ4iMhjn83Ktu83XgHdVtQI4ICIXuMtvBpa44zsUi8g09xgZItIlqe/CmDjZNxFjmlHV9SJyH7BARAJALfB9nEFaTheRFUAFTj0CwDeAZ9wLfWRvnTcDz4rIQ+4xrkvi2zAmbtb7qDFxEpHDqtrV6ziMaW9WNGSMMT5ndwTGGONzdkdgjDE+Z4nAGGN8zhKBMcb4nCUCY4zxOUsExhjjc5YIjDHG5/4/SAgBTloIse4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# read python dict back from the file\n",
    "with open(log_file, 'rb') as pfile:\n",
    "    test_dict = pickle.load(pfile)\n",
    "\n",
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(test_dict['loss_history_0'], 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(test_dict['train_acc_history_0'], '-o')\n",
    "plt.plot(test_dict['val_acc_history_0'], '-o')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "\n",
    "plt.tight_layout(pad=3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3444b34933e404b93c7680b23df8a00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=165.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74a7b5d08c604609acaeb2ab039f30e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=165.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99fca8ffdc6641ed95f3bda577adebe2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=165.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9275567493b74320af82335805cfbd92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=165.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "preds, targets = [], []\n",
    "for fold in range(CFG.nfolds):\n",
    "    model_fd = test_dict[f'best_mode_{fold}']\n",
    "    _, loader_val = data_loader(fold)\n",
    "    for inputs, labels in tqdm(loader_val):\n",
    "        inputs = inputs.to(device=CFG.device, dtype=CFG.dtype)\n",
    "        labels = labels.to(device=CFG.device, dtype=torch.long)\n",
    "                \n",
    "        # Forward, track history if only in training\n",
    "        with torch.no_grad():\n",
    "            outputs = model_fd(inputs)\n",
    "            pred = torch.argmax(outputs, 1)\n",
    "        preds.append(pred)\n",
    "        targets.append(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kappa: 0.7162803076844155\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f167e7da610>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuoAAAGbCAYAAACS8bVfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3hUxdvG8e9JJUAKIfQmCKKgyCu9g/Teq3SQqnRUEKUXpYn0XqSDdBAIvTcBQaRakA6BFJpCkvP+sTGSH2mSwO5m78917cVmzuyZGSY5eXb2ORPDNE1ERERERMS2OFm7AyIiIiIi8jwF6iIiIiIiNkiBuoiIiIiIDVKgLiIiIiJigxSoi4iIiIjYIJeX3cDTgN+0rYwNK5GvjbW7ILE4HnDJ2l0QsVtOhmHtLkgcwrXznE0LfXLNZn+IEjO+dPXLYbPj1Iq6iIiIiIgNeukr6iIiIiIiiSo8zNo9eCW0oi4iIiIiYoO0oi4iIiIi9sUMt3YPXgkF6iIiIiJiX8IdI1BX6ouIiIiIiA3SirqIiIiI2BVTqS8iIiIiIjZIqS8iIiIiImItWlEXEREREfui1BcRERERERukP3gkIiIiIiLWohV1EREREbEvSn0REREREbFB2vVFRERERESsRSvqIiIiImJX9AePRERERERskVJfRERERETEWrSiLiIiIiL2RakvIiIiIiI2SH/wSERERERErEUr6iIiIiJiX5T6IiIiIiJig7Tri4iIiIiIWItW1EVERETEvij1RURERETEBin1RURERERErCXJB+o3bt2hzUefUrNZB2p/0JHvlq+Jse7ps+fJV6o6W3fuTXC7wSH3ad+9P9Uat6N99/4Eh9wHYMfeg9Rt2Zn6rbrSqG03jv/0c4Lbsmdu7m7M3TiNRf6zWbpzHh/2aRNtvQo1y7F013yW7pzH0MlfJLhdLx9PJi4dy8p9i5i4dCye3ikBqFy3Aou2zWHRtjnMWjeZXHleT3BbjmrmjLFcv/oTJ09sj1LetUsbzvy8h59O7mDUyM+t1DuB6Odo8aKpHDu6lWNHt3LpwiGOHd1qxR46Lnd3d/bv28Cxo1s5eWI7X37RG4By5Upw+NAPHD2yhZ07VvH6669Zt6MOKnPmjGzbuoLTp3bx08kdfPxROwC+GjmAn0/v5viP/qxcMQtvby8r9zTpMs2wRHvYMsM0zZfawNOA315uA3G4E3CPO3fvkSd3Th4+fESjdt34duQXvJ49W5R6YWFhfNjjc9zdXKlboxKVypWK1/mPHD/F2k3+DB/QO0r52Mmz8fbypH2LRsz6bjkh9+/Tq0s7Hj16jIdHMgzD4Pyl3+nzxQjWL5mZaOP9r0rkiz4wfpU8knvw+NFjnF2cmblmEuO+nMjPx3+JPJ4leyZGTB9Ml4Y9uB/8gFSpfQi8GxSvc79XLD81GlVhSM9RUco/HtCJ4KAQFkxaTMuPmuHl7cmk4dN5p2Be/rh4mfvBDyhWrggf9m5N2xqdE3W8/8XxgEtWazuhSpUswoMHD5k7dwL5/688AGXLFKffZ92oWbslT548IU2a1Ny5c9fKPXVc0c3Rs0Z/9SXBISEMG/6NFXqXcE6GYe0uJEiKFMl5+PARLi4u7Nq5ml69BzJ3zjfUb9CWc+cu0bFjSwoVzE/7D3tZu6svLPwlxyAvS/r0acmQPi0nTv5MypQpOHJ4M/UbtCVzpgzs2LmfsLAwRo7oD0C//iOs3NsXF/rkms3+EP11ckOiffMky1/DZscZ54q6YRhvGobxqWEY3xqGMSHi+VuvonOJIY2fL3ly5wQsF70c2bJwK5rAYPHKdVQsWwLfVD5RyucsWknjdt2o27Izk2Z9F+92d+49SO2qFQCoXbUCO/YcBCB5cg+MiF8ej//6C+z8F0liePzoMQAuri64uLrwv28e63xQk5XzVnM/+AFAlCC9eecmzNs0nUXb5sS4Gh+d0pVLsHH5ZgA2Lt9MmSolATh97ExkOz8fP0PaDGlefGAObu++w9wLjPqGqmPHlnw9ejJPnjwBUJBuZdHN0bMaNKjJ0mVrX2GP5FkPHz4CwNXVBdeIa6Npmnh6egLg7eXJjRu3rNlFh3Xz5m1OnLR8Iv7gwUPOnbtIpozp8d+2h7AwywrtocPHyZQpgzW7KUlArIG6YRifAksBAzgCHI14vsQwjM9efvcS17Ubtzh78Vfy5c0dpfzWnQC27zlAozrVopTvP/wjf169xtJZE/h+3mR+OX+JYydPx6utu4FBpPHzBSxvFu4FBUce27Z7PzWbfkiXPl8ytH/PBI7K/jk5ObHQfxZbTq3hyJ5jnDlxNsrxrDkykzVHFmauncTs9VMoWrYwAEXKFCRL9sy0rtaR5hXb8dY7b/B/RfLFq01fv1TcvX0PgLu375Eqdarn6tRqWp2DOw8ncHTyrFy5clCyZGEO7FvPjm0rKVjgXWt3SWJQqmQRbt2+w6VLv1u7Kw7LycmJo0e2cO3qT2zfvpejR0/QsVNf1q1dwG+/HuWDD+rz9ejJ1u6mw8uWLTP5332bw0dORClv07oJm7fstFKvHEB4eOI9bFhcu760A/Kapvn02ULDMMYBZ4BR0b3IMIwOQAeAKWOH0b5l00ToasI8evSYnp8P49NuHUmZIkWUY19NmE7Pzm1xdnaOUn7g6HEOHDlOg9YfWc7x+DGXr1ynYP53aPphD548ecqjx48JDrlP/VZdAejVpS0lihSItS8VypSgQpkSHDt5mkkzFzBrwshEHKn9CQ8Pp3nF9qT0SsnXs4eRI3d2fjv/b3Dg7OxMluyZ6VS/O+kypGH66ok0fb8NRcoUokiZgiz0nwVYUmiy5MjMicOnmLNhKm7urngk98DLxyuyzqRh0zm0+2icfSpQ/P+o1bQ6Hep89HIG7aBcXJzx8fGmeMmaFCqYnyWLp5ErdzFrd0ui0bhxHZZpNd2qwsPDKVS4Mt7eXqxYPou8eXLTvduH1KrdkqNHT9CrVydGfz2QTp37WrurDitFiuQsXzaTXn0Gcv/+g8jyfp91IzQ0lMWLV1mxd0mctmcEIBzICFz+n/IMEceiZZrmDGAGWD9HHeBpaCg9Ph9G9UrlqFi2xHPHz5y7SN+BlvccgcEh7D141BK0m9C+RePnVtoBlsy05GzGlKOeOpUPdwLukcbPlzsB9/D18X7uHAXzv8OVazcIDAomVTTHHc2DkAccP3iCYuUKRwnUb9+4w+njvxAWGsb1Kzf589crZMmeGQOD+RMXsXrh+ufO9U9eeUw56vcCAkmd1pe7t++ROq0vgXcDI4/lfCsHn4/pS4/mnxAcGPKSRuuYrl29wZo1PwBw9NhJwsPD8fPzJSDgnpV7Js9ydnambp2qFC5a1dpdESA4OIQ9ew5SuUo53sn3FkePWlZuV6xYx4b1C63cO8fl4uLCimUzWbJkdeR1DaBFi4ZUr1aBipUbWbF3DiDctm8CTSxx5aj3ALYbhvGDYRgzIh6bge1A95ffvYQzTZMvR35DjmxZaNWkXrR1tqycx9bv57P1+/lUKluSAX26Ur50cYoXfo/VG7fyKCKH+tadAO7Gks/5rLIli7L2h20ArP1hG+VKWVYN/7x6PTIH+5fzl3j6NBQfB74r3MfXm5Relh1X3JO5UbhUQS5f+jNKnV2b91Gw+P8B4O3rTdbXs3D9z+sc2n2Emk2q4ZHcA4A06f1IlTrqPQYx2bN1P9UbVQGgeqMq7NmyH4B0mdLy1ayhDOw2nD9/u5ooY5R/rV23hXLlLG+Wc+XKgZubm4J0G1ShfCnOn7/EtWs3rN0Vh+Xn5xu5Y0iyZMl4//2SnDt3EW8vL3Llyg5A+fKlOXfOfm84t3czZ4zl7LlLfDNhRmRZ5Upl6dunC3Xqtebx47+s2DtJKmJdUTdNc7NhGG8AhYFMWPLTrwJHTVvfzybCiVNnWL95O7lefy0yPaV7x1bcuHUHgMZ1q8f42hJFCvDb5St80NFyR31yj2SM/LIvqVPFHQy2b9GI3l+MYNWGLWRIl4Zxwyzb0Pnv2se6H7bj4uJCMnc3xgz5LPLmUkfkly41Ayf0x8nJCScng23rd7Fv20E69G3L2Z/OsXfrAQ7tOkLRMoVYums+4WHhfDt0KsGBIRzefYzXcmZj9vopADx++JgvPx4Wrx1hFkxazIhpg6jVpDq3rt2iX8eBALTv2QrvVN58OtJy70BYaBitqnZ8ef8BSdjC7yZTpnQx/Px8+eO3YwweMoa585Yya+ZYTp7YzpMnT2nbroe1u+nQYpqjRo1q6yZSK8uQPh2zZ4/H2dkZJyeDlSs3sGnTdjp3/oRlS2cSHh5OYGAwHTr2jvtkkuhKFC9Ei+YNOHX6l8gtTL/4YhTjxw3B3d2dzT8sBeDw4eN0/cjubumzDw6S+pLkt2eU2NnC9owSM3venlHE2ux9e0ZHYK/bMzoKm96e8dCyxNuesWhjmx1nkv+DRyIiIiIi9iium0lFRERERGyLg6S+KFAXEREREfti4/ufJxalvoiIiIiI2CCtqIuIiIiIfXGQFXUF6iIiIiJiV+xkl/AEU+qLiIiIiIgN0oq6iIiIiNgXpb6IiIiIiNggB9meUakvIiIiIiI2SCvqIiIiImJflPoiIiIiImKDlPoiIiIiIiLWohV1EREREbEvSn0REREREbFBSn0RERERERFr0Yq6iIiIiNgXpb6IiIiIiNggBwnUlfoiIiIiImKDtKIuIiIiIvbFQW4mVaAuIiIiIvZFqS8iIiIiImItWlEXEREREfui1BcRERERERuk1BcREREREbEWraiLiIiIiH1R6kvi6Fmw38tuQhJgcyFr90BiU2Cfn7W7ILG48zjE2l2QWISGh1m7CxIH09QcyQtS6ouIiIiIiGMzDCOLYRg7DcM4axjGGcMwukeU+xqG4W8YxsWIf1NFlBuGYXxrGMYlwzBOGYbx3jPnahVR/6JhGK3ialuBuoiIiIjYl/DwxHvELRTobZrmW0BRoKthGHmAz4DtpmnmArZHfA1QFcgV8egATAVLYA8MBIoAhYGB/wT3MVGgLiIiIiL2xTQT7xFnU+YN0zSPRzy/D5wFMgG1gfkR1eYDdSKe1wYWmBaHAB/DMDIAlQF/0zTvmaYZCPgDVWJrW4G6iIiIiDgswzA6GIZx7JlHh1jqvgb8H3AYSGea5g2wBPNA2ohqmYArz7zsakRZTOUx0q4vIiIiImJfEvFmUtM0ZwAz4qpnGEZK4Hugh2maIYZhxFg1umZiKY+RVtRFRERExL682hx1DMNwxRKkLzJNc1VE8a2IlBYi/r0dUX4VyPLMyzMD12Mpj5ECdRERERGRGBiWpfPZwFnTNMc9c2gd8M/OLa2Atc+Ut4zY/aUoEByRGrMFqGQYRqqIm0grRZTFSKkvIiIiImJfXu0fPCoBtABOG4ZxMqKsPzAKWG4YRjvgT6BhxLFNQDXgEvAIaANgmuY9wzCGAkcj6g0xTfNebA0rUBcRERER+/IK/+CRaZr7iD6/HKB8NPVNoGsM55oDzIlv20p9ERERERGxQVpRFxERERH7Eo/9z5MCBeoiIiIiYl9eYeqLNSn1RURERETEBmlFXURERETsi4OsqCtQFxERERH78mq3Z7Qapb6IiIiIiNggraiLiIiIiF0xw7Xri4iIiIiI7XGQHHWlvoiIiIiI2CCtqIuIiIiIfXGQm0kVqIuIiIiIfXGQHHWlvoiIiIiI2CCtqIuIiIiIfXGQm0kVqIuIiIiIfVGgLiIiIiJig0zlqIuIiIiIiJVoRV1ERERE7ItSX5IOD6/kNBvVkQy5s4AJiz6Zyu/HL0YeT+bpQavxH5Mqkx/Ozk5sn7mBQyt2JajN5N4paDupB76Z03Dv6h1md/2GxyEPKVi7JBU71QLg70d/sWzAbK6dvZygtuxZio8/xbVgMcKDAwnp1ua54y5v5ydl/+GE37oBwJNDe/lr2fyENeriSoqe/XF5/Q3M+yE8GD2Y8Ns3cc71Jim69LHUMQweL53H00N7E9aWncuQMR1jpwwnTbrUhIebLJm/knkzFkepU7tBNTpFzN3Dh4/4os9wzp65kKB23dxcGTtlOG+/+xZBgcF81O4Trl25TsmyRfnki+64urny9MlTRg4az8G9RxLUliPr0qU1rds0wTAM5s5dypTJc0mVypv5CyaRNVsm/rx8jZYtuhIUFGLtrjqk8+f3c//+Q8LCwggNDaNEiRq8885bTJw4gpQpU3D58lVat+7G/fsPrN1Vh+Tu7s7OHd/j7u6Os4szq1ZtZMiQscyYPoYCBd7FMODCxd9p164HDx8+snZ3kx4H2Z7RMF9yjs9HrzW2+v9ki7FduHTkHAeX7cDZ1Rk3D3ceh/z7Q1OpSx08vJKzdtRiUvp68sWOb+hfqANhT8PiPHeuonko0qAMC/tMjVJe+7MPeBT8AP+pa6nYuTbJvVOwdtRisr/3BjcvXeNxyEPylM1PtR4NGFNnQKKPOb6GvHvTam0DuOTJh/nXY1L06B9joJ6sTmMeDOv3n8/tlDY9Kbp9xv0BPaKUu1etg/NrOXg0dRxupd7HtWgpHo4eDG7uEBoK4WEYqXzx/mYOQW3qQ3jc3wcvS4F9QVZrGyBNOj/SpvPjzKlzpEiZnPXbl9KhZQ8unf8tss57hd7l0oXfCAm+T5nyJejxaWfqVmoer/NnypKRMZOG0LR2+yjlzds24s08bzCgzzBq1K1C5erv83H7T8jzzpsE3LnL7Zt3eOPNnMxfOZVib1dM1DH/F3ce228AmyfPG8yb/y1lStfhyZOnrFk7jx7dv6B1myYEBgYxbuw0evXuhI+PN19+8ZW1u/tCQq34s5sYzp/fT/HiNbh7NzCybN++9fTrN4y9ew/TqlUjXnstC4MHj7ViLxMmzM7nKEWK5Dx8+AgXFxd271pNr14D+eXshcg3T6O/HsjtOwGMHj3Zyj19MU+fXDOs3YeYPBrTPtHiy+R9ZtnsOJN8jnqylB68XvgtDi7bAUDY07AoQfo/3FN4WP5NnoxHQQ8ID7V8pFK+Q036rh1Bvx++plrPhvFuN1/FghxeuRuAwyt3k69iIQB+P36BxyEPI55fxCd96hcfXBIQ+sspzAf3X+i1bmUq4jV6Gl7jZ5G8c29wit+3s1uREjzZsQWAJ/t345rvPcuBJ39HBuWGqxtg9feYVnfnVgBnTp0D4OGDR1y6+BvpM6SNUuf40Z8ICbbM4Yljp0ifMV3ksToNq7PGfxEbdy1j+NgvcIrnHFWsWo7vl64D4Id1/hQvXRiAX06f4/bNOwBcOHcJd3c33NxcEzZIB5U7d06OHD3J48d/ERYWxr59R6hZqzLVa1Rk0aLvAVi06Htq1Kxk5Z7Ks954Iwd79x4GYPv2vdSpU83KPXJs/6yUu7q64OrqimmaUT7h8PBIxsteEHVYZnjiPWxYkg/UU2dNy4O7ITQf05lPN46i2aiOuHm4R6mze/5m0ufMxPAj0+i/ZQwrB8/DNE3eLJWPtK+lZ3Tt/oyq9ilZ387O64Xfile7nmm8CbljWQ0NuROEp5/Xc3WKNy7HL7tOJnyQSZxL7rx4fTOblF9+jXOW1wBwypwNt5LvE/JZV0J6tofwcNzKxG9l1fD1IyzgtuWL8DDMhw8xPL0BcH7jLbwmzsP727k8nDrOqqvptiZTlozkeedNTv54OsY6jZvXZfe2fQC8/kZ2atSpTIOqrahetjFh4WHUaRi/oCJdhrTcuG75tCcsLIz7IQ9I5esTpU7VmhU4c/ocT548fcERObZffjlPiRKF8fX1wcMjGZUqlyVz5gykTevHrYg3Q7du3iFNGsdeTLAm0zTZsGEhBw5spF27ZgCcOXOeGjUs17p69aqTOXMGa3bR4Tk5OXHs6FauXzvFtu17OHL0BACzZo7j6pWT5M6dk8mT51i5l0lUuJl4Dxv2wjnqhmG0MU1zbgzHOgAdAMr6FiCv5+sv2kyCOTs7k+Xt7KwYNJfLJy9Rf2ArKnauzcZxyyPrvFX6Xa7+8gffNh2CX7Z0fLRwAL9W/YS3SuXjzdL5+GyT5WNf9+TJSPtaen49cpY+a4bh4uaKe/JkJPdJGVln7ajFnN3zU5z9ylUsL8Uav8/4Bl++nIEnEaG/XiDow8bw12NcCxQhZf/hBHf+ANd87+GS8w28xkwHwHB3xwy2fDycst8wnNKmx3B1xckvLV7jZwHw14bvebL9BzCi+4TL8oMaduEsIR+3xilzNlJ278fTHw/D0yevZKy2LHkKD6bOG8vQz0fz4P7DaOsULVmIRs3r0rBaawBKlC7C2/nfYu22RQAk80jG3Tv3AJi2YDxZsmbE1c2VjJkysHHXMgDmzljMysVrMaKZo2dXpXLlfp1PB/agZYNOiTlMh3L+/K+MHzeNdRu+4+GDR/x8+iyhoaHW7pY8o1y5+ty4cYs0aVKzceMizp+/RMeOfRk3bjD9+/dg40Z/vVG1svDwcAoWqoS3txcrV8wmb97cnDlznvYf9sLJyYkJ3wyjUcNazF+wPO6TiUQjITeTDgaiDdRN05wBzADr56gH3rxL0M27XD55CYCTmw5TsXPtKHWKNiyL/9S1AARcvsXdK7dJ93pGMAy2TlnL/sXbnjvvP3nlMeWo378TjFcaH0LuBOGVxof7Af/msmZ8MyvNRnVgautRPAzSTUCxevxvmtLTHw9DR2fL6rdh8PeOzTz+buZzL3kw0jI3MeWom3fv4OyXltC7d8DJGSNFCsz7UXONw69exvz7L5yzZSfs0vmXMDD74eLiwtR541i7chNbNmyPts6beXIx6puBtGnclaDAYAAMw+D7pesZPfTb5+p3atkTiDlH/eb1W2TImJ6b12/j7OyMp1fKyPOmz5iW6QvG07vLAP7842piDtXhLJi/nAXzLQHEwMF9uH7tJrdvB5AufRpu3bxDuvRpuHPnrpV76bhu3LgFwJ07d1m3bgsFC+bnm29mUKOG5R6QnDmzU6XK+9bsokQIDg5h954DVKpUljNnLL8zwsPDWb5iHb17dVag/hKYDrLrS6ypL4ZhnIrhcRpIF9trbcX9O8EEXr9L2hyWjwdzl3ibmxej/nIPvB5A7hJvA+Dp5026HBkJ+PM2Z/f8RLFGZXFLbkmV8U6XipSpn09hic7pbcco0qAMAEUalOGU/zEAUmVMzYfTerOg52Ru/34jUcaYlBk+vpHPnXO9CU5OmPeDeXrqR9yKl8XwtqRDGCk9cUoTv2/JJ0f24/Z+ZQDcSpTh6SnLR5VOadODk7PleZp0OGfKQvgt695sawu++nYQly78xuyp30V7PGOm9EydP45enT/n91//3cFo/57DVK1ZgdR+ljn09vEiUzw/pt+2eRf1m1h2R6paq2Lkzi6eXp7MWTKJr4dN4McjShtLqH/SWjJnzkjtWlVYsXwdmzZu44MP6gPwwQf12bjB35pddFjJk3uQMmWKyOfly5fizJnzkXNmGAb9+nVj1qyF1uymQ/Pz88Xb2xITJEuWjPLvl+LChd94/fXXIuvUqF6R8+cvWamHSZxSXwBLMF4ZCPyfcgM48FJ69BKsGDSX1t98jLOrCwFXbrOwz1RKflABgH2LtrH521U0H9OZ/ptHg2GwdtQiHgbe59zeU6TPmYk+q4YBlu0U5/eYxIO7ce/04D91LW0n96BYo3IEXg9gdpfxAFTt1oAUqVLSeFg7AMJDw/i6Vv+XNHLbl6L3l7i+nR/Dyxuf2St4tGQuhovl2/LvzetwK14G96q1ISwMnvzNwzGDAQi/cpnHi2bhOWiM5SbS0FAeTv8G7tyKs82//TeRsufneE9bhHn/Pg8izumSJx/J6jez7PximjycNh7zfvDLG7wdKFjk/6jXuCbnzlyITE8ZPWwiGSMC7sXzVtCtb0dS+fowdLTl+zg0LIza5Ztx6fxvjB0xmQUrp+Lk5MTTp6F8+ekIrl2N+w3qsoWrGT91ODuPric4KISP238CQKsPm5Ate1Y+7t2Bj3t3AKBlg87cDbj3Moaf5C1aPBVfXx+ePg2lV88vCQoKYdzYqSz4bhItWzXi6pXrtGje1drddEjp0qVh2bIZgOVTrWXL1uDvv5uuXdvSqVNLANas2cz8+VqptZYMGdIxZ/Y3ODs7YTg5sXLlejZt2saunavx8koJhsHpU7/Q9aP/vmuZyD9i3Z7RMIzZwFzTNPdFc2yxaZrN4mrA2qkvEjtrb88osbP29owSO3ventER2Pv2jI7A3rdnTOpseXvGh8OaJ1p8mWLAQpsdZ6wr6qZptovlWJxBuoiIiIhIorPxlJXEkuS3ZxQRERERsUcJ2fVFREREROTVc5BdXxSoi4iIiIh9UeqLiIiIiIhYi1bURURERMS+mEp9ERERERGxPUp9ERERERERa9GKuoiIiIjYFVO7voiIiIiI2CClvoiIiIiIiLVoRV1ERERE7IuDrKgrUBcRERER++Ig2zMq9UVERERExAZpRV1ERERE7ItSX0REREREbI/pIIG6Ul9ERERERGyQVtRFRERExL44yIq6AnURERERsS8O8pdJlfoiIiIiImKDtKIuIiIiIvZFqS8iIiIiIjbIQQJ1pb6IiIiIiNggraiLiIiIiF0xTcdYUVegLiIiIiL2RakvIiIiIiJiLVpRFxERERH74iAr6i89UP8+6PTLbkIS4Nfj2azdBYlFNg9Xa3dBYvEo9G9rd0FiEfLksbW7IHFwdnK2dhfETpkOEqgr9UVERERExAYp9UVERERE7IuDrKgrUBcRERER+xJu7Q68Gkp9ERERERGxQVpRFxERERG74ig3kypQFxERERH74iCBulJfRERERERskFbURURERMS+OMjNpArURURERMSuOEqOulJfRERERERskFbURURERMS+KPVFRERERMT2KPVFRERERESsRivqIiIiImJflPoiIiIiImJ7TAcJ1JX6IiIiIiL2JTwRH3EwDGOOYRi3DcP4+ZmyQYZhXDMM42TEo9ozx/oZhnHJMIzzhmFUfqa8SkTZJcMwPovPMBWoi4iIiIjEbB5QJZry8aZp5o94bAIwDCMP0ATIG/GaKYZhOBuG4QxMBqoCeYCmEXVjpdQXEREREbErrzL1xTTNPYZhvBbP6rWBpaZp/urrR/oAACAASURBVA38bhjGJaBwxLFLpmn+BmAYxtKIur/EdjKtqIuIiIiIfUnE1BfDMDoYhnHsmUeHePbiI8MwTkWkxqSKKMsEXHmmztWIspjKY6VAXUREREQclmmaM0zTLPjMY0Y8XjYVeB3ID9wAxkaUG9E1EUt5rJT6IiIiIiJ2xdq7vpimeeuf54ZhzAQ2RHx5FcjyTNXMwPWI5zGVx0gr6iIiIiJiV8zwxHu8CMMwMjzzZV3gnx1h1gFNDMNwNwwjO5ALOAIcBXIZhpHdMAw3LDecrourHa2oi4iIiIjEwDCMJUBZwM8wjKvAQKCsYRj5saSv/AF0BDBN84xhGMux3CQaCnQ1TTMs4jwfAVsAZ2COaZpn4mpbgbqIiIiI2JVXvOtL02iKZ8dSfzgwPJryTcCm/9K2AnURERERsS9mdPdmJj3KURcRERERsUFaURcRERERu2LtXV9elSQfqI+bNIyKlcsQcOce5YrXfu5454/bUq9RDQBcnJ3JlTsHb79ekqCg4Bdu083NlW+njSJf/rwE3guiY9teXP3zOqXLFuPzQb1wdXXl6dOnDPlyDPv3HH7hdpIKJycnvt34LQE3AxjUZlCUYx0GdiBfsXwAuHu445Pah4ZvN0xQeyl9UtJvcj/SZUnHrSu3GNllJA+CH1CuTjkadrGc+/HDx0zqP4nfz/6eoLaSCicnJ6ZvmkLAzQD6tR4Q5VjDD+tTvWk1wsLCCLobxNe9x3Dr2u0Etefp48nAKQNInyUdN6/cYlDnoTwIfkCFuu/TtEsTwDJH4/tN4NezvyWoLXv2zaThVKxSloA7dylTrFa0dYqXLMzQkf1wcXXh3t0g6lZvkaA23dxcmTT9q8jrW4c2vbjy5zVKlyvOgEG9cXN15cnTpwz54mv26fqWIN7eXkyd+hV58ryBaUKnTn15/PgvJk4cjru7O6GhYfToMYBjx36ydlcd1vnz+7l//yFhYWGEhoZRokQN8uXLw8SJI0iWzDJH3bt/rjl6Ccxwpb4kCcsXr6ZZg5j/wNTUiXOoWKoeFUvVY8SQ8RzcfzTeQXrmrBn5fsO858qbtqhPcFAIxd+rwowp8xkwqDcA9+4F0bJJF94vUYdunfsxcdqoFxpTUlO7XW3+vPRntMdmDJ7BR1U+4qMqH7F+7noObD4Q7/O+U/Qdeo3r9Vx5oy6NOLn/JO1Lt+fk/pM06tIIgJtXbvJJw0/oUqkLSyYsodtX3V5sQElQ/XZ1uRzDHF08c4mO1brQrmIHdm/cS8fP4/sH3SB/sXf5bFzf58qbdW3C8f0naF6qNcf3n6BZV0twfuPPm3Rv0It2FTuwYMJCen/d88UGlEQsXbyaJvU/jPG4l7cno8Z+ScumXShTtCYftuoe73NnyZqJVRsWPFferGUDgoJCKPp/lZk+ZT5fDI64vt0NpEXjzpQtXotunT5j0vSv//uAJIoxYwaydetu8ucvT+HCVTh37hLDh/dj+PAJFC1ajaFDxzF8eD9rd9PhVa7cmCJFqlKihGXRb8SI/gwf/g1FilRlyJCxjBjR38o9FHsWZ6BuGMabhmGUNwwj5f+UV3l53Uo8hw78SGBg/ALvOvWrsWblvzfj1m9Uk03bl+K/dxVfjx+Ek1P83tdUqfY+y5esAWDD2q2UKlMUgJ9PneXWzTsAnD97Cfdk7ri5uf6X4SQ5fun9KPx+YbYs2RJn3TK1y7Br7a7Ir+t3rM+EDROYsnUKzXs1j3ebxSoVY9vKbQBsW7mNYpWLAXD2x7M8CH4AwLkT5/DL4PcfRpJ0pcngR9HyRdi4OPob1U8e+Im///obgF+OnyXNM/9vjTs1YtqGycz2n0Hr3i3j3WaJSsXZvGIrAJtXbKVk5RIAnPnxl8g5srSV5oXGlFQcOnCMoFiub/Ua1mDTen+uXb0BQEDAvchj9RvVZPOO5Wzfu5rR3wz+D9e38ixfbLm+rV+zhZJlLD8/luub5ZOUc2cv6vqWQJ6eKSlZsgjz5i0F4OnTpwQHh2CaJl5ell/H3t6e3LiRsE+vJPFZ5sgT+GeObsXxCnkR1t5H/VWJ9cpsGEY3YC3wMfCzYRjP5o6MeJkde9U8PJJRrkIpNq7zByDXGzmoVa8KtSo3p2KpeoSFhVE/IkUmLukzpOP6tZsAhIWFERJyH19fnyh1qteqxM+nzvLkydPEHYid6TioI7NHzCY8PPaflLSZ0pI+S3p+2m/5+PC90u+RKXsmutfoTtfKXcn5Tk7eLvJ2vNr08fMh8HYgAIG3A/FO7f1cncpNKnNs57H/OJqk6aNBXZg+fCamGedfOqZ60yoc2XkUgIKlC5A5eyY61ehK+0odyf3OG+Qr8k682vT1S8W925ag8t7te6RK7fNcnepNqnJk55H/MBLH8/rrr+Ht48WqDQvYuvt7GjaxXMJzvZGDOvWqUaNSM8qXqhtxfasZr3NmyJCWa9csgX9YWBj3o7m+1ahdmZ9P/eLw17eEyJ49KwEBd5kxYwwHD25iypSvSJ7cg759hzBiRH8uXjzIyJGf8+WXX1m7qw7NNE02bFjIgQMbadeuGQB9+gxm5Mj+XLp0iJEjB/DFF5qjl8E0jUR72LK4ctQ/BAqYpvnAMIzXgJWGYbxmmuYEIMaRGYbRAegA4OWRnuRuqRKpuy9PxSplOXr4eGTaS8kyRcn3bl5+2LkcgGTJ3CNXo+Ys/JYs2TLj5upKpswZ8N+7CoBZ075j2aLVGMbz/zXPBjlvvJmTAYN70aRuzB9ZO4LC5QsTdDeIS6cv8U7R2AO4MrXKsG/TvsiA/r3S7/Fe6feYtHkSAB4pPMj4WkZ+Pvwz49eNx9XNFY8UHnj6eEbWmTNyDsd3H4+zX/mK5aNS40r0qdcngSO0f8XKFyEwIIgLpy+Sv9i7sdatWK88ufPlpnsDS7pRodIFKFS6ALO2TAMsc5Q5eyZOHT7NlPUTcXtmjv6pM33ELI7ujvsNUv7i71KtSRU+ruvYqS9xcXZx4d38eWlQqw3JkrmzcdtSfjz6E6XKFCNf/rxs2bkCgGQeyQi4Y7m+zV04kazZMuPq5krmzBnYvnc1ADOnfcfSRasg2uvbv89zv5mTLwb3plHddi9/gEmYi4sz+fO/Ta9eAzl69CRjxgykT58ueHt78sknQ1mz5gfq16/O1KlfU736B9bursMqV64+N27cIk2a1GzcuIjz5y9Rt251+vYdEjFHNZg2bTTVqjWzdlfFTsUVqDubpvkAwDTNPwzDKIslWM9GLIG6aZozgBkAGXzyxL0MZwP+N+3FMAxWLFnLiCHjn6vbtrkldzlz1oxMmDKC+jVaRzl+4/pNMmZKz43rt3B2dsbLyzMy/SZDxnTMWfgt3Tr14/IfV17egOxAnoJ5KFqxKIXKFcLV3ZXknsnpO6Evo7uPfq5umVplmDxg8r8FBiybvIwfFv3wXN2etSzB2ztF36Fio4qM6zUuyvGggCBSpU1F4O1AUqVNRfDdf1MHXnvzNXqM7sEXLb7gftD9RBqp/Xq70NuUqFSMou8Xxs3djeSeyfn8288Y3i3q/RUFSr5H84+b0b1Bb57+s4pqGCyatIT1izY+d94uNT8GLDnqVRpWYlSvqHN+LyAQ37S+3Lt9D9+0vgTeDYo8luOt7PT9ujeftuhHSFBIIo84ablx/Sb37gby6NFjHj16zKEDx8j7Tm4Mw2D5kjUMHzzuude0aW6ZmyxZMzFhykjq1Wj5P+e8RaZMGSKvb55engQGWuYnQ8Z0zF00iY86fsrl3x37+pZQ167d5Nq1Gxw9ehKA1as30bt3F4oXL0jv3oMA+P77jUyZotVaa/onreXOnbusW7eFggXz07x5fXr3HgjA999vYOpUzdHLYOspK4klrqTEmxF/HhWAiKC9BuAHxO8zbDvg6ZWSoiUKsXnTjsiyfbsPUb12JVL7+QLg4+NN5iwZ43W+LT/spFHTOgDUqF0pcucDL29Pvls+lZFDxnP08IlEHoX9mffVPFoUbkHr4q0Z1XUUP+3/KdogPVOOTKT0TsnZH89Glh3ffZxKjSuRLHkyAFKnTx1tCkt0DvkfokKDCgBUaFCBg1sPApAmYxq+mPkFo7uP5trv1xI6vCRh5qjZNCzUlCbFmjOk63BO7D/5XJCeM29Oeo3qQf+2XxL0TEB9dPcxqjapgkfEHPmlT41PNCks0Tngf5AqDSsBUKVhJfZvtdxEnDZjWobOHMSI7qO4qjmK0+aN2ylavADOzs54eCTjvQL5uHj+N/buPkiN2pXw++f6luo/XN827aBRM8v1rWadyuzbcwiwXN8WLZ/O8MHjdH1LBLdu3eHq1RvkypUDgLJlS3Du3EVu3LhNqVJFI8suXfrDir10bMmTe5AyZYrI5+XLl+LMmfPcuHGL0qUtc1SunOboZTHDjUR72LK4VtRbAqHPFpimGQq0NAxj+kvrVSKaMms0xUsWxje1Dz+e2cGYUZNwdbHc4LRg7jIAqtaowO4d+3n86HHk6y6c/5Wvhk1g6epZODkZhD4NpV+foVy9cj3ONpd89z0Tp3/FgeObCQoMolNbSwpF2w+bkT17Vnr07UyPvp0BaFK3PXefucFLoEXvFlw4dYHD/pY3OGVrl2X3ut1R6hzfc5wsObMwbq1lRfCvh38xuvvoKKvjMVk+eTn9p/ancpPK3Ll2h+GdLX/lt1mPZnj6eNJ1eFfAkn/bvXr8d8lwJG36tOL8Txc44H+QzgM64JHCg8HTvgDg1rXbfN72S47t+ZFsubIyed1EwLKd4vBuI6ME8zFZPGkpA6cNoFqTKty6dptBnYYC0Kpnc7x8vOg5wvKpVlhoGB2rd31Jo7R902aPpXjJQvimTsWJX3YxeuREXFwtl/UFc5Zx8cJv7Ni2l50H1mKGh7NowUrOnb0IwKhhE1i2ejZOTk48DQ2lX+8h8bq+Lf5uJZNmfM2hE1sICgymY1tLqlO7Dz8ge46s9OrbmV4R17fGddtFuYFV/ptevQYyd+4E3Nxc+eOPP+nQoQ8bNmxl9OhBuLg48/fff/PRR59Zu5sOK126NCxbNgMAFxcXli1bg7//brp0+YwxYyxz9Ndff9O1q+ZIXpwRnxvEEsJeUl8cVX7PbNbugsTisamb8WzZ2ftXrd0FiUXIk8dxVxKretkxiCTMX3/9abPLzX8WLJ9o3zxZj2232XEm+T94JCIiIiJJi62nrCSWJP8Hj0RERERE7JFW1EVERETErjjKiroCdRERERGxK45ye4NSX0REREREbJBW1EVERETErij1RURERETEBpmmYwTqSn0REREREbFBWlEXEREREbtihlu7B6+GAnURERERsSvhSn0RERERERFr0Yq6iIiIiNgVR7mZVIG6iIiIiNgVR9meUakvIiIiIiI2SCvqIiIiImJXTNPaPXg1FKiLiIiIiF1R6ouIiIiIiFiNVtRFRERExK44yj7qCtRFRERExK44yvaMSn0REREREbFBWlEXEREREbuiXV9ERERERGyQo+SoK/VFRERERMQGaUVdREREROyKo9xMqkBdREREROyKo+SoK/VFRERERMQGvfQV9b/Dnr7sJiQBTt6/bO0uSCxMHGTJwE695ZnZ2l2QWJx/cN3aXZA4OBmOkb4gic9RbiZV6ouIiIiI2BVHyVFX6ouIiIiIiA3SirqIiIiI2BWlvoiIiIiI2CBHuYNLgbqIiIiI2BVHWVFXjrqIiIiIiA3SirqIiIiI2BVH2fVFgbqIiIiI2JVwa3fgFVHqi4iIiIiIDdKKuoiIiIjYFROlvoiIiIiI2JxwB9mfUakvIiIiIiI2SCvqIiIiImJXwpX6IiIiIiJiexwlR12pLyIiIiIiNkgr6iIiIiJiVxxlH3UF6iIiIiJiV5T6IiIiIiIiVqMVdRERERGxK0p9ERERERGxQY4SqCv1RURERETEBmlFXURERETsiqPcTKpAXURERETsSrhjxOlKfRERERERsUVaURcRERERuxKu1BcREREREdtjWrsDr4hSX0REREREbFCSD9QnThnJ+d8Osf/wxmiPN2hUi70H17P34Ho2b1tG3rffTHCbbm5uzJ73DcdObsN/x0qyZM0EQNlyJdixZzX7Dm1gx57VlCpdNMFt2btxk4Zx+uJedh5YG+3xzh+3xX/vKvz3rmLngbVcvXsaHx/vBLXp5ubKtDljOXB8Mxu3LSVz1owAlC5bjC27VrBj/xq27FpBidJFEtROUjB+0jB+vriPXQfWxVineMlCbNu7it0H17N644IEt+nm5sr0OeM4eHwzm7YtJUvk/BRny66V7Ny/li27Vmp+nuHk5MTMzdMYOW/Yc8caflifeTtmM9t/BmOXfk26TGkT3J6njydjFn/Fwr3zGLP4K1J6pwSgQt33me0/g9n+M5i0ZgKvv5UjwW3ZM13fbN/YiUP56cIeth9YE+1xT6+UzFsyGf+9q9hxYC2NmtVJcJs+Pt4sWTWTfcc2sWTVTLy9vQCo27A6/vtW4b9vFWu3LCTP27kT3FZSFp6ID1uW5AP1xYtW0bBu2xiP/3n5CjWqfkCpYjUZ89Vkvvn2+V90McmSNRPrNi18rrx5ywYEBYVQMH8Fpk6ey6AhfQG4ezeQZo06UrJoDbp2/ISpM0f/9wElMcsXr6ZZgw4xHp86cQ4VS9WjYql6jBgynoP7jxIUFByvc2fOmpHvN8x7rrxpi/oEB4VQ/L0qzJgynwGDegNw714QLZt04f0SdejWuR8Tp416oTElJcsWr6FpLPPj5e3JqDFf0qppV8oUq8mHrXrE+9xZsmZk1Yb5z5U3a9GAoKBgir1XhelTFjBgUB8A7t0LpGWTzpQrUZvunfsxadpX/31ASVT9dnW5fOnPaI9dPHOJjtW60K5iB3Zv3EvHz2Oez/+Vv9i7fDau73Plzbo24fj+EzQv1Zrj+0/QrGsTAG78eZPuDXrRrmIHFkxYSO+ve77YgJIIXd9s3/Ila/igQccYj7du35QL53+lYql6NKjZmi+HfYKrq2u8zl2sRCHGTx7+XHnXnu3Zt+cwJQtWY9+ew3Tt2R6AK5ev0aB6ayqWrMc3o6fx1fhBLzQmRxFuGIn2sGVxBuqGYRQ2DKNQxPM8hmH0Mgyj2svvWuI4uP8ogYExX/iOHD5BcFAIAEePniRDpnSRxxo2roX/zpXs3r+OcROG4uQUv/c11apXYOniVQCsXbOZ0mWLAXD61C/cvHkbgLNnL5IsmTtubm4vNK6k4tCBH2Odn2fVqV+NNSs3RX5dv1FNNm1fiv/eVXw9flC856dKtfdZvsSyerJh7VZKlbF8svHzqbPcunkHgPNnL+GezB03t/hdkJOqQweOERQYFOPxeg1qsHH9Nq5dvQFAQMC9yGP1G9Xkh+3L2PYf56dytfdZvsSyArlh7RZKRjM/585e1PxESJPBj6Lli7Bx8aZoj5888BN///U3AL8cP0uaDH6Rxxp3asS0DZOZ7T+D1r1bxrvNEpWKs3nFVgA2r9hKycolADjz4y88CH7wTFtpXmhMSYWub7bv8IEfCYpljkzTJGXKFACkSJGcoMBgQkNDAej0cRs2bl+G/75V9P6sa7zbrFy1HCsi5mjFkjVUqfY+AMeOnCQ42BKPHD96igwZ08V4DnEcsf7kG4YxEPgWmGoYxkhgEpAS+MwwjM9fQf9eqRYtG7Ldfw8Ab+R+nbr1q1O1YhPKlKhFWFgYDRvXitd5MmRMx7WrNwEICwsjJPgBvqlTRalTq3YVTv30C0+ePEncQSRRHh7JKFehFBvX+QOQ640c1KpXhVqVm1OxVD3CwsKo36hGvM6VPkM6rl97Zn5C7uPr6xOlTvValfj51FmePHmauANJYnLkfA0fHy9WbZjPll0radikNmCZn9r1qlKz8gdUKFWP8LBw6jeqGa9zZsiQjuvXLIF/WFgY96OZnxqan0gfDerC9OEzMc24b62q3rQKR3YeBaBg6QJkzp6JTjW60r5SR3K/8wb5irwTrzZ9/VJx77blTdm92/dIldrnuTrVm1TlyM4j/2EkjkvXN9s1d+Zicr2Rg+Nnd7F9/xoG9huJaZqULlec7DmyUb18YyqVqk++/HkoUrxAvM7plzY1t28FAHD7VgCp0/g+V6dJi3rs3LY3UceS1JiJ+LBlce360gDID7gDN4HMpmmGGIYxGjgMPP+ZDmAYRgegA0By9zS4uyYs5+5VKFmqCM1bNqRqJctHuKXLFOPd/HnZvtuyMp7Mw52AO3cBWLB4MtmyZcHNzZVMmTOwe78lf3f61PksXvg9RjQfozz7S/TNN3MycEhf6tdp87KHlWRUrFKWo4ePR34sXLJMUfK9m5cfdi4HIFky98jV3DkLvyVLtsy4uVrmx3+vZQ5nTfuOZYtWxzk/b7yZkwGDe9Gk7ocve1h2z8XZmXz589KwdhuSJXNng/9Sfjz6E6Ui5mdz5PwkIyDA8vMzZ+FEsmbLFDk/256Zn6Uxzs+/z3O/mZMBg3vTuG77lz9AG1esfBECA4K4cPoi+Yu9G2vdivXKkztfbro36AVAodIFKFS6ALO2TAPAI4UHmbNn4tTh00xZPxE3N1c8Unjg6eMZWWf6iFkc3X0szn7lL/4u1ZpU4eO6jp36El+6vtmusu+X5MzpczSs1YbXsmdlyeqZHD74I2XKFafM+8XZuud7AJKnSE72HNk4fOBH1vsvwd3djeQpkuOTyjuyzvBB49i9Y3+cbRYvWZimzetRt2qLlzo2e/cqc8sNw5gD1ABum6b5dkSZL7AMeA34A2hkmmagYfkhnABUAx4BrU3TPB7xmlbAgIjTDjNN8/n8z/8RV6AeappmGPDIMIxfTdMMATBN87FhGDH+H5mmOQOYAeDrmcvW36yQJ29uJkwaQaP67Qi8Z/mY3zAMli5ezdBBY5+r37KZ5SOuLFkzMXnaV9Sq1jzK8evXbpIpc3quX7+Js7MzXt4pI8+bMWN6FiyZQpeOffnj9+hzSuV5//uxsGEYrFiylhFDxj9Xt23zboAlh3PClBHUr9E6yvEb12+SMVN6bly/ZZkfL8/Ij6czZEzHnIXf0q1TPy7/ceXlDSiJuH79JvfuBfLo0WMePXrMoQPHyPt2bgzDYPmSNTHMz8eAJUd9wpSR1KvR6rlzZsyUIXJ+PL08CYxIv7HMz0Q+7vSZ5gd4u9DblKhUjKLvF8bN3Y3knsn5/NvPGN4tav5xgZLv0fzjZnRv0Jun/6yiGgaLJi1h/aLnb7TvUtMyR/mLvUuVhpUY1Svq/TT3AgLxTevLvdv38E3rS+Ddf9OjcryVnb5f9+bTFv0IiUgrlNjp+ma7Gn9Qh0nfzALgj9//5Mrla+TMlQPDMJg0fiYL56147jU1KzYFLDnqjZrVoWfXqAkIAbfvkjadH7dvBZA2nR937/ybMvhW3jcY/e1gWjTsFO+0KXkl5mHJKnl2x4TPgO2maY4yDOOziK8/BaoCuSIeRYCpQJGIwH4gUBDLQv6PhmGsM00zMLaG40p6e2IYRvKI55Gf6RiG4Y3t3ygbL5kyZ2DBosl07tCHXy/9EVm+Z9dBatWugp+f5SMpn1TeZM6SMV7n/GHTdpo0qwdA7TpV2Lv7EGC58W7pyhkMHTiWw4eOJ+5AkjBPr5QULVGIzZt2RJbt232I6rUrkfqf+fGJ//xs+WEnjZpa7tyvUbsS+/YcBizz893yqYwcMp6jh08k8iiSpi2bdlCkWAGcnZ3x8EjGewXycfHCb+zdfYgatSv/+/PzH+Zn6w87adTUkkJTo3Zl9u/59+dn4fJpjBgyTvMTYeao2TQs1JQmxZozpOtwTuw/+VyQnjNvTnqN6kH/tl8S9ExAfXT3Mao2qYJH8mQA+KVPjU80KSzROeB/kCoNKwFQpWEl9m89AEDajGkZOnMQI7qP4urv1xJjiEmerm+27drVG5SM2KHNL01qcuR8jct/XGHXjv00/qAeyVNYQqT0GdJGzldctm7eScOIOWrYtA5bftgJQMbMGZi5YALdO/Xjt18vv4TRJC3hRuI94mKa5h7g3v8U1wb+WRGfD9R5pnyBaXEI8DEMIwNQGfA3TfNeRHDuD1SJq+24VtRLm6b5d0Qnnw3MXYFW0b/EtsycM54SpQqTOnUqfj63l1EjJuDiYrmBZt6cJXzy2Uf4+vowetxgAEJDQylfph7nz19ixNDxfL92Hk5OBk+fhvJJ78FcvXI9zjYXLljBtJljOHZyG4GBQbRvY/n498MOLcieIxt9Pu1Kn08tq/L1a7eOcgOeo5kyazTFSxbGN7UPP57ZwZhRk3CNmJ8Fc5cBULVGBXbv2M/jR48jX3fh/K98NWwCS1fPwsnJIPRpKP36DI3X/Cz57nsmTv+KA8c3ExQYRKe2ll1F2n7YjOzZs9Kjb2d69O0MQJO67bnrwPMzddaYyPk5fmYno0dNwtXFctlYMHcZFy/8xs5t+9i5fw3h4SaLvlvJubMXAZ6ZHyee/of5WfzdSiZN/4qDxzcTFBhMx7aWXSvafvgB2bNnpWffzvR8Zn4c+ecnJm36tOL8Txc44H+QzgM64JHCg8HTvgDg1rXbfN72S47t+ZFsubIyed1EAB4/fMzwbiOjBPMxWTxpKQOnDaBakyrcunabQZ2GAtCqZ3O8fLzoOcKy6hsWGkbH6vG/yS6p0fXN9k2eNZpiJQrhm9qHYz9vZ8z/t3fncVFV/x/HXwcFUVxYNERc0jL9ZZaZuYGJ+4Yr7ltqpZWZplZWlmmmaS6ZZmZ9XSr3XXNfcSN3c8kW+2alKIKCptZXgfv7Y4gkEFCwmWHezx7zYObeO3M/p+O98+Hczz289xHu7rZz3BczF/LB+9OY+NG7bNplKysaNXwCsRfj2L51N2UfKMPKDXMAuHblGv36DMnU/8+PJn7GtJkT6NS1DWdOn6VPD1s52ksvP4uPbyFGjbMdq/Hx8TSt2+Eutdz5ZedfJr25ZDvJ9KTqkPT4W5Z1RpCQgQAAIABJREFUFsCyrLPGmL/mvg0Ebr5kdTpp2a2Wpx9bZm5AygpnKH1xZXly6a5/R2Y5/G0urq18/gzPsWJH31/JOLEV+3Jz8KnxXN2Z2OMO20FzinXNti/ILpFfZthOY8y9wFc31ajHWZblfdP6WMuyfIwxq4HRlmXtTFq+GXgFqAvksSxrZNLyN4FrlmWlrrG+SY6fR11EREREchYHmPUlKqmkhaSf55OWnwZK3LRdcSAyneXpUqIuIiIiIk7l36xRv4WV/F0G/iSw4qbl3Y1NdeBSUonMeqChMcbHGOMDNExalq6MatRFRERERFyWMWYeEAIUNsacxjZ7y3vAQmPMU8CvQLukzddgm5rxJLbpGXsCWJZ10RjzDrAvabsRlmVleFODEnURERERcSr/5tSDlmV1usWqemlsawFp3kVvWdYMYMbt7FuJuoiIiIg4FVeZakE16iIiIiIiDkgj6iIiIiLiVLJwE6hTUaIuIiIiIk7l36xRtyeVvoiIiIiIOCCNqIuIiIiIU3GVEXUl6iIiIiLiVCwXqVFX6YuIiIiIiAPSiLqIiIiIOBWVvoiIiIiIOCBXSdRV+iIiIiIi4oA0oi4iIiIiTsWydwD/EiXqIiIiIuJUXOUvk6r0RURERETEAWlEXURERESciqvcTKpEXURERESciqsk6ip9ERERERFxQBpRFxERERGnollfREREREQckKvM+qJEXUREREScimrURURERETEbjSiLiIiIiJORTXq2SSXmwbtHVm8lWDvECQdV67/ae8QJB3fE2nvECQdjxUsbe8QJAMnrp21dwjipBJdJFVXFi0iIiIi4oBU+iIiIiIiTsVVbiZVoi4iIiIiTsU1Cl9U+iIiIiIi4pA0oi4iIiIiTkWlLyIiIiIiDshV/jKpSl9ERERERByQRtRFRERExKm4yjzqStRFRERExKm4Rpqu0hcREREREYekEXURERERcSqa9UVERERExAG5So26Sl9ERERERByQRtRFRERExKm4xni6EnURERERcTKuUqOu0hcREREREQekEXURERERcSqucjOpEnURERERcSqukaar9EVERERExCFpRF1EREREnIqr3EyqRF1EREREnIrlIsUvKn0REREREXFAGlEXEREREaei0hcREREREQfkKtMzqvRFRERERMQBaURdRERERJyKa4ynK1EXERERESej0hcREREREbGbHJ+oT5oyim9P7mZ7xKp0t6tUuSLnLn5L85aNsrxPb59CLFo+gz0H17No+QwKeRcEIKxdc7btWsm2XStZvWEeFR4ql+V9ObuM+qdmcFV++nU/W3csZ+uO5Qx6pW+W9+nh4c6nMyey99AG1m1eSImSgQDUrlOTTeFLCN+9kk3hSwh+onqW9+XqChUqyNy5H3P48GYOHdpMtWqVk9cNGNCbP/74BT8/HztG6NwmTBnJ0R93sHX3ijTXP9evFxt3LGXjjqVs3b2C0xeO4u1dKEv79PBwZ9qM8ew+uI7Vm+ZTvGQxAJ4IqcH6bYvYsms567ctIuiJalnaT07i5ubGpDUf8tbMYanWFQkswrvz3mXy+imMXjAav6J+Wd5f/kL5eWfOSKaHT+edOSPxKpQfgJBWIUxeP4XJ66fw/tJxlP6/0lnelzMLKObPnOWfsH73EtbuXESP3p1uuW3FRx/kh6h9NG5eL8v7LeRdkNmLp7J573JmL55KwUIFAGjRtgmrwxewOnwBi9bMpHyFslneV06WmI0PR5bjE/X5c5fSMezpdLdxc3PjreGD2bp55219ds3gqkyeOjrV8hdf6s2O8AiqVW7EjvAIXnypNwC//nKals26EhLUggljP2b8pHdua385UWb65+uI/dSp1Yo6tVoxfuxHmf7sEiUDWf7V56mWd+nejri4y1R9tCHTps7ireGDAbh4IZYuHZ6jds0WvPDsEKZ+Mvb2GiOpjBs3jA0bwqlUqR5Vqzbmu+9OAlC8eAB16wbz66+n7Ryhc1s4dxmd2/a+5fqPJ8+gQa02NKjVhlEjJhKxax9xcZcy9dnFSxZjyVezUi3v1C2MS3GXqVm5MdOnzmbo24MAuHgxju4dn6duUCtefO41Jk97747alBO16NWC307+lua6p4Y+zeYlW+jX6AXmTZrHk0N6ZPpzK1avyIDxL6Va3q5vO77Z9Q29a/fmm13f0O75dgCc+y2KIe2H0K/RC8z/cB4vvNfvjtqTU8QnJDDqrYk0qhlG28ZP0vWp9tz/QOpfXtzc3Hj1rf7s2BJxW59fLegxxk5+O9XyZ/v3ZPf2vdSr2ord2/fybP+eAJz+5QydWjxNs9odmDL+U96dMPSO2uUqrGz8z5HddqJujEmd+TiwiN37iY1N/4vpmT7d+GrFemKiL6RY3vfFp9iwdTHbdq3kldcyf0Jr0rQeC+YuB2DB3OU0bVYfgH17D3Ep7jIA+/cfplixorfTlBwpM/1zK23bt2D9lkVs3bGccR8Mx80tc/+cmzSty4K5ywBYtXw9tWrXAODokRNEnTsPwHcnfiSPpwceHu53FJtAgQL5CQ6uxqxZ8wG4ceMGly7Z/v2PHfsWb7wxGsty7BOko/t694FMHz+twpqyfPGa5Ndh7ZuzZvN8Nu5YytiJb2f6+GnctC4L59nOb1+t2ECt2rYrT8eOnCDqXDQA3584SR7PPDp+AL+ifjxe73E2zF+f5voSZUvwzc7DABzZfYTqDf6+ktemTxsmrJrI5PVT6DywS6b3Wa1BdTYv3gTA5sWbqN7Q9pnfHTjB1UtXbM8PfU/hgKyP3juz6KgYjh/5DoCrV65x8oef8Q+4J9V23Z/pyLpVm7kQczHF8mde6M6yjV+wOnwB/V99NtP7rd+kNksXfAXA0gVf0aBpCAAH9x3h8qXfATi0/yhFi/nfSbMkh0n3zGyMWfmPxyqgzV+v/6UY76qiAffQNLQ+s2bMT7E8pG4QZe4rRcM6bakT3JJHKlWgRs0qmfrMIkX8iIqyfWFFRUVTuIhvqm26dGvL5k3bs94AF1ClaiW27lzB/MWfUq78/QCUfaAMrdo0oVnDTtSp1YqEhETatm+eqc8rGuDPmTNnAUhISODy5d/x9U1ZftG8ZSOOHjnB9es3srcxLqR06ZLExFxg+vRxRESsYerUMeTLl5dmzeoTGXmOo0dP2DtEl5E3ryd16tdi9cqNgO34adGmMS0adaVBrTYkJCQQ1j40U59VNMCfyDPngJuPH+8U2zRr0ZBjOn4A6P12b2aMmomVmPYvpT9/+zNBTYMAqNG4JvkK5KOAdwEerfUoxUoHMrD5S7zYuB/3V7yfClUrZGqf3oW9iT0fC0Ds+Vi8C3un2qZhh4bs33rgDluV8wSWCKBCxXJ8c+BYiuX+RYvQsFkd5s5anGJ5cEh17i1TktYNuhEa0pGHHvk/Hq9RmcwoXMSP6KgYwPbLgl/h1DlC+66tCN+86w5b4xpcpfQlo1lfigPfAp9hmwnHAFWA8em9yRjTG+gNkN/zHjw9Up8kHMW7773BiGHjSExM2VUhdYMIqRPE1h22kSOv/Pkoc9+9ROzez7rNC8nj4YFX/nx4+xRK3mbE2+MyVT4TVKsaXbq1JbRR5+xvUA5z5JvjVH6oLlevXqN+gyf4fO5HVKvciCdq1+CRSg+xcavt5OmZ1zP5isisL6dQqlRx3D3cKV48ILl/pk/7nHlzlmKMSbWfm0d2y5W/nzeHD6Z9617/Qgtzrty5c1Gp0kMMHDiMffsOM27cMIYOfYng4KqEhnazd3gupUHjEPbtOZhc9hJcuzoPP1KBtVsXAuDpmYeYpNHCGV9+SIlSxfFwdyeweAAbdywF4LNpX7BgzrIMj58Hyt/P0OED6dj6mbvdLIf3eL3HiYu5xE9HT1KxesU0t5nx7n94dsRz1Gtbn+N7jxFzNoaEhAQefaIyj9Z6lA/XTgbA08uTYqUDOb73OONXTMDdwx1PL08KeBdI3mbW6Jkc3H4ww7gq1niYhh0a8krYy9nXWCeWzysvU2eN4503xnPlytUU64a+O5ixwz9MlSPUqlOd4JDqrNo6DwAvr3zcW6YE+yIOsmT9bDw8PPDyykchn4LJ24wd8SE7tmZcPlM9uArturSiQzN9B6XH0UtWsktGiXoVoD/wBvCyZVmHjTF/WJYVnt6bLMuaDkwHKFKonEP/n3zk0YeYPmMCAH5+PtRrWJv4+HgMhkkTp/P5zAWp3tO4XnvAVqPeqXNr+j3/Wor10dEX8PcvQlRUNP7+RYiJ/vty2YMVyjFx8kg6hj1DbGzcXWxZznDl979Pmps2bmfM+GH4+vpgjGHBvGWMHD4h1Xt6dH0BsNWoT546mlah3VOsPxt5jsDAAM5GRpErVy4KFiyQ3BcBxfyZPWcKL/R5lVM/p11TKplz5sw5zpw5y759tsv6y5at4Y03XqJUqRLs3bsWgMDAACIiVlOrVsvkq1CS/f5Z9mKMYdG8FYwaMTHVtr26vgjYatQnTR1FWGiPFOvPRp6jWGDRfxw/tl8AAor5M+PLD3nx2df45ZSOnwerPEi1BtWoUqcKHnk8yFsgL4M+GMz4AeOSt7kYdZFRfd4FwDOfJzWbBHHt92sYA4umLmTdnHWpPndQy4GArUa9Xrv6fDAoZT/GxcThc48Psedj8bnHh7iYv79r7i1/Ly+OfZFh3d/i97jf70aznUru3Ln5aOY4Vixew4bVW1Ktr1jpQSZ9arsXzcfXm5D6wSTEJ4AxTJs0k3mzl6R6T1ijJwFbjXpYx+a80u/tFOtjoi9QxL8w0VExFPEvnKKkptyDZRk18U16dexH3B2WhUrOkm7pi2VZiZZlTQR6Am8YY6aQw+Zer/JwPR5LeqxasZ5XBw1n7erNbN2yk85dw/DyygfYSmQKp3F5Ki3r1m6hQ+dWAHTo3Iq1azYDEFg8gFlfTqZv71f470+n7kp7cpp77imc/PzRyhVxc3Pj4sVYtodH0Lxlo+Q+8fYpRPESxTL1mevWbKFD59YANG/ViJ3bvwagYKECzF04nZHDJ7B3T8ajUpK+qKhoTp8+S9myZQAICQni8OFjlCr1GOXLB1O+fDBnzpylRo1mStLvogIF81M96HHWrfk7CdkZ/jXNWjZMvuTu7Z3542f92q2072Q7v4W2bMjO7XsA2/HzxcKPGT1iIvv2HMrmVjin2WNm06PakzwV1IuxL4zhyO4jKZJ0gII+BZOvUrTr256NC2zlSQfDD9KgfUM883kC4OfvRyG/zM3Ys2fjHuq1td0bVa9tffZstJ3jihQrwuvT32D8gPFE/hyZLW10du9NeouffviZGR/PSXN9yGPNqV05lNqVQ1m3ahNvvTKajWu3sWNLBG07tyCfV17AViLjVzhzM1htXredNh1spWZtOoSyaa1t7DMgsCgfzxrH4Off5NRPv2ZD63I2lb7cxLKs00A7Y0wz4PLdDSl7ffKf8QQFV8XXz4dvvg1n7OjJ5Ha3NXv2P+rSb7Ztyy4eeOA+1my0bXP16jWe7/1y8uXh9Hw4YTqfzf6ALt3acvr0WZ56sj8Ag1/ti4+vN2PH26boik9IoEFIWFab6NQy6p/mLRvR46lOxMcn8Oeff9K7l20k6Yfvf2L0yA9YtGwGxs2N+PgbvDpoBKd/y/jLZ84Xi5k6/X32HtpAbOwleveyzZrw9DNdKV2mJINefp5BLz8PQLvWvTLV55K2gQOHMXPmJDw83Dl16ld69x5s75BylKmfvU/N4Kr4+nlz4PgWxr03Bffcths4/7oa2CS0PuFbdvHHtT+S3/fD9z8xZuQk5i/7DDc3Q/yNeF4b/E6mjp95Xyxh8idj2H1wHXGxcTzby9anvZ7pTOnSJRnw8nMMePk5ADq2fjrVDXgCXQZ25cejP7J34x4q1qjIk68+iWXBsT3H+PjNqQAc2nGIEmVLMG65rdL0z6t/MG7AOC5dyHiUdfHURQz5eAgNOzQgOjKa0c/aRoQ79u9EQZ+CPD/Sdn5LSEjgpdABd6mVju+xapVo3SGU747/mFyeMv7dKQQUt030MG9W6tHyv+zc9jX3P1CaxWtnAXD16h8Mem4oF2JiM9zvtEkzmfyfMbTv2orI0+d4odcrAPR7+Rm8fQsxfKztKn1CQgKt6nfNShNztEQXmYzA3O1ZFxy99EXEkV25/qe9Q5B0eOfxsncIko7HCrr2POHO4MS1s/YOQdLxU8zB1DelOIhupdpkW375xS9LHbadOaqMRURERERyPlcZBVaiLiIiIiJOJdFFUvUc/5dJRURERESckUbURURERMSpaB51EREREREH5OjTKmYXlb6IiIiIiDggjaiLiIiIiFNxlZtJlaiLiIiIiFNxlRp1lb6IiIiIiDggjaiLiIiIiFPRzaQiIiIiIg7Isqxse2SGMeaUMeaoMeawMWZ/0jJfY8xGY8yPST99kpYbY8yHxpiTxpgjxpjKd9pOJeoiIiIiIhmrY1lWJcuyqiS9HgJstiyrLLA56TVAE6Bs0qM38PGd7lCJuoiIiIg4lUSsbHtkQUtgdtLz2UCrm5Z/btl8DXgbYwLuZAdK1EVERETEqSRm48MY09sYs/+mR+80dmkBG4wxB25a729Z1lmApJ/3JC0PBH676b2nk5bdNt1MKiIiIiJOJTunZ7QsazowPYPNgizLijTG3ANsNMZ8l862Jq3d3ElsGlEXEREREUmHZVmRST/PA8uAqkDUXyUtST/PJ21+Gihx09uLA5F3sl8l6iIiIiLiVP7NGnVjjJcxpsBfz4GGwDFgJfBk0mZPAiuSnq8EuifN/lIduPRXicztUumLiIiIiDiVzE6rmE38gWXGGLDlznMty1pnjNkHLDTGPAX8CrRL2n4N0BQ4CVwDet7pjpWoi4iIiIjcgmVZ/wUeSWP5BaBeGsstoG927FuJuoiIiIg4FVf5y6RK1EVERETEqWTnrC+OTDeTioiIiIg4II2oi4iIiIhTyeJfFHUaStRFRERExKn8y7O+2I1KX0REREREHJBG1EVERETEqaj0JZskJLrKBDrO6eqNP+0dgqTDVS7tOav/JdywdwiSjv2X/2vvECQDnb1TTU0tkima9UVEREREROxGpS8iIiIi4lQSXeSKsxJ1EREREXEqrpGmq/RFRERERMQhaURdRERERJyKZn0REREREXFArpKoq/RFRERERMQBaURdRERERJyKq/ydESXqIiIiIuJUVPoiIiIiIiJ2oxF1EREREXEqlouMqCtRFxERERGn4io16ip9ERERERFxQBpRFxERERGn4io3kypRFxERERGnotIXERERERGxG42oi4iIiIhTUemLiIiIiIgDcpXpGVX6IiIiIiLigDSiLiIiIiJOJdFFbiZVoi4iIiIiTkWlLyIiIiIiYjcaURcRERERp6LSFxERERERB6TSFxERERERsRuNqIuIiIiIU1Hpi4iIiIiIA3KV0pccn6hPnjqaho3rEBN9gaBqzVKtb9KsHq8PHUBiokV8fDyvD3mXPREHsrRPb59CzJg1iRIlA/nt1zP0fPJFLsVdpm37FvR/6RkArl69xqABwzh+7Lss7UvAzc2NiN2riYw8R+s2Pdm8eQkF8nsBUKRIYfbvP0y79k/bOUrX5ebmxtcRazgTeY7WrXtw770l+PKLqfj4enP40FF69OzPjRs37B2mU8ro/HY3zjkeHh58PH0sj1R6iNiLcfTq0Z/ffj1DSJ0g3ho+GA8Pd65fv8GwoWPYsf3rLO3L2U2cMpIGjUKIib5ISM0WqdY/368XbdqHApA7V27KlitDhfuCiIu7dMf79PBwZ/K0MTxc6UFiL8bRp9dAfvs1kidCavLG2wPxcHfn+o0bjHjrfXZt33PH+8kJipQJoPuU/smv/Urcw7qJi9g+Y23yssotg6j7rK3v/nftfywZ+hmRJ37N0n5zeeSm84S+lHioNFfjrvD5C5OIPR3NA8EVafZqJ3K75yb+RjyrRs3hZMTxLO1LnF+Or1GfO2cp7Vr3uuX67dsiqFWjObWDWtDv+deYNOXdTH92UHBVpkwbk2r5gIF9CA/fzeOPNiA8fDcDBvYB4NdffiO0SRdq1WjOuDEf8cGHI2+/QZJKvxee4rvvTya/rlcvjKrVGlO1WmP27DnA8hVr03m33G39+j3Fd9/93T+j3n2dDz/8lAoVahEbd4mePTvaMTrnltH5LSvnnBIlA1m55stUy7t2b0tc3GWqVKrPxx/N5O0RLwNw4UIsndv3Ibh6KH37vMLHn75/+w3KYRbMXU6ntr1vuX7q5BnUr9WG+rXa8O6ICUTs2pfpJL1EyWIs/Wp2quWdu7UlLu4SNSo35pOpnzP07cEAXLwYS/eOz1EnqCX9n3stze8uVxP937OMbzqE8U2HMCH0Na7/eZ2j6/el2Obib9F81GEE45q8ysbJS2k3+tb9+U8+xYvw/Py3Ui2v1r4Of1y6wqiQAYT/ZzWhQzoDcDX2d/7z1Pu83/gV5g2aSpeJfbPWwBwu0bKy7eHIbitRN8YEG2MGGmMa3q2AslvErn3Ext76xHf16rXk515eebn5Skq//k+zadsSdkSsYsjrL2Z6n02a1WP+nGUAzJ+zjKah9QHYu+cQl+IuA7Bv32ECAv1vpymShsDAojRpUpeZM+elWpc/vxchITVZuXK9HSITgMDAAJo0qceMmXOTl4WEBLFk6WoAvvhiES1aNLJXeE4vo/Nbeuecdh1asHHrYsJ3rWTCpHdwc8vc10HTZvWZP3cpACuWr+OJkBoAHD3yLefOnQfgxIkf8fTMg4eHxx21K6f4evd+4mLjMrVt67BmLFu8Jvl1WPvmrN28gE07ljJ24tuZ7p9GTeuycN4KAL5asZ7g2tUBOHbkBFHnogH47sSP5PHMg4eH++00J0crG1SRC79EEXsmJsXyUwd/4I/LVwH45eCPeBf1TV73WKtgBiwfyaA179Fu1NMYN5OpfT3UsAr7lmwH4MiaPZStWQGAM8dPcfl8LADnfjhN7jzu5PLI8YUPd8zKxv8cWbpHvjFm703PnwGmAAWAYcaYIXc5tn9Ns+YN+PrAOuYv+pR+z9uaVaduMGXuK0X9kDCeqNmCRx59iBpBj2fq8+4pUpioKNsJMSoqmiKF/VJt0617OzZv3J59jXBR495/m9deH0ViYmKqdS1bNmbr1l38/vsVO0QmAOPHvc1rr71LYqLtROjn50PcpcskJCQAcObMWQKLFbVniC7j5nPOA+Xuo3VYM5o06EjtoBYkJCTQrkPq0oy0BBTz58zpcwAkJCRw+dIVfP18UmzTomVjjnzzLdevX8/eRuRQefN6Uqd+MKtXbgCg7ANlaNmmCc0bdaF+rTYkJiQS1r55pj4rIMCfyDNnAVv//H75d3x9vVNsE9qiIceOnOD6dZWc/eXR5jU4tHJ3uttU61CHE9sOA3DPfcWoFFqDD9sOY3zTISQmJPJYq+BM7auQvy9xkRcASExI5M/f/8DLp0CKbR5uUo0zx0+RcD3+DlojOUlGv6rd/Ot2b6CBZVnRxphxwNfAe2m9yRjTO2l78uUpQh73QtkR612zetVGVq/aSI2gx3lt6ADatOhBnXpB1KkbTPiulQB4eeXjvvtKEbFrHxu3LMYjjwdeXvnw8SmUvM3wt8ayZfPODPcXXKsaXbu3o0lDXfLPiqZN6hEdfYFDh47yxBPVU63v0L4lM9IYaZd/R9Om9TgfHZPUP7ZRV2NSjzg5+FXHHOGf55wnatfgkUoV2BxuGxn3zJuHmGhb4vD53I8oVaoEHh7uBBYPSD6/ffLxbOZ+ueQWffh3J5Yvfz/DRrxMWKued7tZOUbDxnXYt+dQctlLrdrVefiRCqzbuhAAT09PYmJs/TPjy8mULBWIh7utfzbtsPXhZ9O+YP6cZRkeY+XK38/Q4YPo0Fr37fwll3suKtR/jNVj599ym/trPEi1DnWY3HYYAA8EVaR4xdK8tNJWLuuex4MrF2xXr3p+MhDfEveQyz03PsUKM2iNLVXaPnMt+xaFk0YXpTiG/MsWJ3RIZz7pNiq7mpgjWVbqAbqcKKNE3c0Y44Nt5N1YlhUNYFnWVWPMLX/NsyxrOjAdwLdAWaf5Go7YtY/SpUvi6+eDMYaJ4z9h9szUB26Dum0BW416p65hvPDsqynWn4+Owd+/CFFR0fj7FyE66QQL8GCFckyaMor2YU8RezFzl0QlbTVqVqFZswY0alwHzzx5KFiwADNnTqJnz/74+npTpUol2rV/xt5huqyaNR4ntFlDGjeqi6enrX/GjxuOd6GC5MqVi4SEBAIDA4g8e87eoeZoaZ1zjDHMn7uMd94en2r77p1tdbElSgby0bQxtGjaNcX6yDPnCCxelMjIc+TKlYuChfInf26xYkX5fN5Unu/zMqd+ztoNd66kZVhTli1enfzaGMPCecsZNWJiqm17de0H2GrUJ00dTZvQJ1Osj4w8R7HAAM5GRpErVy4KFCxAbFL5TUAxf2Z8OZl+zw7hl1O/3cUWOZfyIZU4c+wUV2LSLiMLKF+S9u/14dMe73EtLukKrYH9S7anmdzP7DMBsNWodxr3HFM7jkixPu7cRbyL+XHp3EXccrnhWSBv8ucWKupLz08GMXfgR1z4NSobW5nzJDp4yUp2yajorRBwANgP+BpjigIYY/IDmSvGcnCly5RMfv7wIw/i7uHOxQuxbNm0k67d2uLllQ+wXU4sXNj3Vh+Twro1W+jYpTUAHbu0Zu3qzQAEFg/g8zkf8Vzvwfx08lT2NsQFvfnmGO67vyrlytWkW/e+bNu2i549bXfwh7UJZc3aTfzvf/+zc5Sua+ib71Hmvsd5oFwNunbry9Ztu3iyRz/Cw3cT1sY2Q0m3bu1YtWqDnSPNuW51ztm+LYIWLRsnn9O8fQpRvESxTH3m2jWb6di5DQAtWzVmR7htZpeChQowf/F03hk2nj1fH8zehuRgBQrmp0ZQFdYsn8mnAAAHE0lEQVSv2ZK8bEf414S2bPR3/3hnvn82rN1K+04tAQht2Yhd2//uny8XTmPUiAns23Mom1vh3Cq3COLgql1prvMu5kfPaQOZ+9JHRP98Nnn5j7uO8XCTauT3KwhAvkJe+AQWztT+jm88wONhTwDwcNNqnNxtm9nFs2A+npn5KmvGzuPUgR+y0iTJQdIdUbcs695brEoEWmd7NHfBpzMmElSrKn5+Phz7bgfvjZpE7ty2ip5ZM+bRvGVjOnZqxY0b8fz555881WMAAFu37OSBcvexfrPt0uPVq9fo8/RgYmIuZrjPDyZ8wozZk+jarR2nT0fSs7vtRtRXhryAr683708YDkB8fDz1are5G812ee3at2Dc+1PtHYak4fU3RvHlF1N5e/grfHP4GDPTuGolmZPR+e1W55zvvz/JqHcmsmTFLNzcDDduxPPKoOGc/i0yw31++fkipn06jv2HNxEbG8fTPV8C4Jne3ShdphSDX+3L4Fdto/JhLXtk6pyZU3382ThqBlfF18+bg8e38v57U3DPbfva/XzmAgCahtYnfMturl37I/l9P3z/E2NGTmL+ss9wc3Pjxo14Xhv8Tqb6Z+4Xi5nyyRgiDq4jLvYSfXoNAqDXM10oXbokL738HC+9/BwAHVs/7dL9A+Du6cEDwRVZ9PqnyctqdLFNABExZxMNXwwjn09+wkbaZldKjE9gYos3iDp5hrXjF9Lni9cxxpAQn8DSt2akuhk1LXsWbqXzhL68vu0DrsVd4fN+HwIQ3L0RfqX8afBiGxq8aMsNPuk2KrmkRlKyXKRu0tzthjpT6YsrunrjT3uHIOlwlRORs8rvkdfeIUg6PHJpxgxH19n7EXuHIOmYcGq+w1ZPFPd9KNu+IE9fPOaw7czx86iLiIiIiDgjDTeIiIiIiFNxlSvOStRFRERExKk4+l8UzS4qfRERERERcUAaURcRERERp2K5yDzqStRFRERExKmoRl1ERERExAHpL5OKiIiIiIjdaERdRERERJyKSl9ERERERByQpmcUERERERG70Yi6iIiIiDgVlb6IiIiIiDggzfoiIiIiIiJ2oxF1EREREXEqKn0REREREXFAmvVFRERERETsRiPqIiIiIuJULBe5mVSJuoiIiIg4FZW+iIiIiIiI3WhEXUREREScimZ9ERERERFxQK5So67SFxERERERB6QRdRERERFxKip9ERERERFxQK6SqKv0RURERETEAWlEXUREREScimuMp4NxlUsH2cUY09uyrOn2jkPSpv5xfOojx6b+cWzqH8em/pHsptKX29fb3gFIutQ/jk995NjUP45N/ePY1D+SrZSoi4iIiIg4ICXqIiIiIiIOSIn67VPtmWNT/zg+9ZFjU/84NvWPY1P/SLbSzaQiIiIiIg5II+oiIiIiIg5IibqIiIiIiANSon4bjDGNjTHfG2NOGmOG2Dse+ZsxZoYx5rwx5pi9Y5HUjDEljDFbjTEnjDHHjTH97R2T/M0Y42mM2WuM+Sapf4bbOyZJzRiTyxhzyBjzlb1jkdSMMaeMMUeNMYeNMfvtHY/kDKpRzyRjTC7gB6ABcBrYB3SyLOtbuwYmABhjngCuAJ9blvWQveORlIwxAUCAZVkHjTEFgANAKx0/jsEYYwAvy7KuGGPcgZ1Af8uyvrZzaHITY8xAoApQ0LKsUHvHIykZY04BVSzLirF3LJJzaEQ986oCJy3L+q9lWdeB+UBLO8ckSSzL2g5ctHcckjbLss5alnUw6fnvwAkg0L5RyV8smytJL92THhrFcSDGmOJAM+Aze8ciIv8eJeqZFwj8dtPr0yjRELltxph7gUeBPfaNRG6WVFZxGDgPbLQsS/3jWD4AXgES7R2I3JIFbDDGHDDG6C+USrZQop55Jo1lGnESuQ3GmPzAEmCAZVmX7R2P/M2yrATLsioBxYGqxhiVkDkIY0wocN6yrAP2jkXSFWRZVmWgCdA3qSRTJEuUqGfeaaDETa+LA5F2ikXE6STVPi8B5liWtdTe8UjaLMuKA7YBje0civwtCGiRVAM9H6hrjPnSviHJP1mWFZn08zywDFvJrEiWKFHPvH1AWWNMaWOMB9ARWGnnmEScQtLNiv8BTliWNcHe8UhKxpgixhjvpOd5gfrAd/aNSv5iWdZrlmUVtyzrXmzfPVssy+pq57DkJsYYr6Qb5THGeAENAc1CJlmmRD2TLMuKB14A1mO7EW6hZVnH7RuV/MUYMw+IAMoZY04bY56yd0ySQhDQDdtI4OGkR1N7ByXJAoCtxpgj2AYlNlqWpSkARTLPH9hpjPkG2AustixrnZ1jkhxA0zOKiIiIiDggjaiLiIiIiDggJeoiIiIiIg5IibqIiIiIiANSoi4iIiIi4oCUqIuIiIiIOCAl6iIiIiIiDkiJuoiIiIiIA/p/tScnqYSR0ZkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x504 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.cat(preds).cpu()\n",
    "t = torch.cat(targets).cpu()\n",
    "kappa = cohen_kappa_score(t, p, weights='quadratic')\n",
    "print(f'Kappa: {kappa}')\n",
    "conf_mat = confusion_matrix(t,p)\n",
    "#plt.matshow()\n",
    "plt.figure(figsize=(14,7))\n",
    "sn.heatmap(conf_mat, annot=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}