DkNN_CIFAR-10_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading faiss with AVX2 support.\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import os\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import foolbox\n",
    "import torch\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "from lib.dataset_utils import *\n",
    "from lib.cifar_resnet import *\n",
    "from lib.adv_model import *\n",
    "from lib.dknn_attack import DKNNAttack\n",
    "from lib.cwl2_attack import CWL2Attack\n",
    "from lib.dknn import DKNNL2\n",
    "from lib.utils import *\n",
    "from lib.lip_model import *\n",
    "from lib.cifar10_model import *\n",
    "\n",
    "# from lib.cifar10_dcgan import Discriminator, Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f5c017c4270>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set all random seeds\n",
    "seed = 2019\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_valid, y_valid), (x_test, y_test) = load_cifar10_all(\n",
    "    '/data', val_size=0.1, seed=seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "Load Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_name = 'cifar10_basic_rn.h5'\n",
    "model_name = 'cifar10_at_rn.h5'\n",
    "# model_name = 'adv_cifar10_exp%d.h5' % exp_id\n",
    "# model_name = 'train_cifar10_vae_exp%d.h5' % exp_id\n",
    "# model_name = 'rot_cifar10_exp%d.h5' % exp_id\n",
    "# model_name = 'ae_cifar10_exp%d.h5' % exp_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = PreActResNet(PreActBlock, [2, 2, 2, 2], num_classes=10)\n",
    "net = torch.nn.DataParallel(net)\n",
    "cudnn.benchmark = True\n",
    "net.load_state_dict(torch.load('saved_models/cifar10/' + model_name))\n",
    "net = net.module\n",
    "net = net.eval().to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "net = PreActResNet(PreActBlock, [2, 2, 2, 2]).eval()\n",
    "config = {'num_steps': 8,\n",
    "          'step_size': 0.05,\n",
    "          'random_start': True,\n",
    "          'loss_func': 'xent'}\n",
    "net = PGDL2Model(net, config)\n",
    "\n",
    "net.load_state_dict(torch.load('saved_models/cifar10/' + model_name))\n",
    "net = net.basic_net\n",
    "net = net.eval().to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = CIFAR10_VAE((3, 32, 32), latent_dim=128)\n",
    "net.load_state_dict(torch.load('saved_models/' + model_name))\n",
    "net = net.eval().to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = PreActResNet(PreActBlock, [2, 2, 2, 2], num_classes=4)\n",
    "net.load_state_dict(torch.load('saved_models/' + model_name))\n",
    "net = net.eval().to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = CIFAR10_AE((3, 32, 32), latent_dim=128)\n",
    "net.load_state_dict(torch.load('saved_models/' + model_name))\n",
    "net = net.eval().to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreActResNet(\n",
       "  (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "  (layer1): Sequential(\n",
       "    (0): PreActBlock(\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    )\n",
       "    (1): PreActBlock(\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): PreActBlock(\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (shortcut): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "      )\n",
       "    )\n",
       "    (1): PreActBlock(\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): PreActBlock(\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (shortcut): Sequential(\n",
       "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "      )\n",
       "    )\n",
       "    (1): PreActBlock(\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): PreActBlock(\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (shortcut): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "      )\n",
       "    )\n",
       "    (1): PreActBlock(\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    )\n",
       "  )\n",
       "  (linear): Linear(in_features=512, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8045\n"
     ]
    }
   ],
   "source": [
    "ind = np.zeros(x_test.size(0))\n",
    "with torch.no_grad():\n",
    "    num = 0\n",
    "    for i in range(x_test.size(0) // 100):\n",
    "        begin = i * 100\n",
    "        end = (i + 1) * 100\n",
    "        y_pred = net(x_test[begin:end].to('cuda'))\n",
    "        ind[begin:end] = (y_pred.argmax(1).cpu() == y_test[begin:end]).numpy()\n",
    "        num += (y_pred.argmax(1).cpu() == y_test[begin:end]).sum().numpy()\n",
    "    print(num / y_test.size(0))\n",
    "ind = np.where(ind)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind = ind[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    step: 0; loss: 30.836; l2dist: 0.000\n",
      "    step: 100; loss: 2.474; l2dist: 1.385\n",
      "    step: 200; loss: 2.265; l2dist: 1.348\n",
      "    step: 300; loss: 2.224; l2dist: 1.343\n",
      "    step: 400; loss: 2.202; l2dist: 1.337\n",
      "    step: 500; loss: 2.192; l2dist: 1.337\n",
      "    step: 600; loss: 2.185; l2dist: 1.330\n",
      "    step: 700; loss: 2.181; l2dist: 1.337\n",
      "    step: 800; loss: 2.168; l2dist: 1.329\n",
      "    step: 900; loss: 2.157; l2dist: 1.325\n",
      "binary step: 0; number of successful adv: 89/100\n",
      "    step: 0; loss: 51.983; l2dist: 0.000\n",
      "    step: 100; loss: 2.878; l2dist: 1.433\n",
      "    step: 200; loss: 2.404; l2dist: 1.359\n",
      "    step: 300; loss: 2.346; l2dist: 1.352\n",
      "    step: 400; loss: 2.303; l2dist: 1.346\n",
      "    step: 500; loss: 2.298; l2dist: 1.352\n",
      "    step: 600; loss: 2.263; l2dist: 1.339\n",
      "    step: 700; loss: 2.253; l2dist: 1.338\n",
      "    step: 800; loss: 2.237; l2dist: 1.336\n",
      "    step: 900; loss: 2.202; l2dist: 1.332\n",
      "binary step: 1; number of successful adv: 100/100\n",
      "    step: 0; loss: 30.527; l2dist: 0.000\n",
      "    step: 100; loss: 2.580; l2dist: 1.362\n",
      "    step: 200; loss: 2.316; l2dist: 1.334\n",
      "    step: 300; loss: 2.255; l2dist: 1.327\n",
      "    step: 400; loss: 2.201; l2dist: 1.319\n",
      "    step: 500; loss: 2.173; l2dist: 1.320\n",
      "    step: 600; loss: 2.167; l2dist: 1.318\n",
      "    step: 700; loss: 2.152; l2dist: 1.314\n",
      "    step: 800; loss: 2.154; l2dist: 1.311\n",
      "    step: 900; loss: 2.161; l2dist: 1.314\n",
      "binary step: 2; number of successful adv: 100/100\n",
      "    step: 0; loss: 19.551; l2dist: 0.000\n",
      "    step: 100; loss: 2.371; l2dist: 1.290\n",
      "    step: 200; loss: 2.239; l2dist: 1.276\n",
      "    step: 300; loss: 2.210; l2dist: 1.277\n",
      "    step: 400; loss: 2.195; l2dist: 1.278\n",
      "    step: 500; loss: 2.175; l2dist: 1.275\n",
      "    step: 600; loss: 2.164; l2dist: 1.273\n",
      "    step: 700; loss: 2.163; l2dist: 1.272\n",
      "    step: 800; loss: 2.166; l2dist: 1.275\n",
      "    step: 900; loss: 2.156; l2dist: 1.275\n",
      "binary step: 3; number of successful adv: 100/100\n",
      "    step: 0; loss: 16.960; l2dist: 0.000\n",
      "    step: 100; loss: 2.297; l2dist: 1.237\n",
      "    step: 200; loss: 2.174; l2dist: 1.234\n",
      "    step: 300; loss: 2.145; l2dist: 1.237\n",
      "    step: 400; loss: 2.130; l2dist: 1.233\n",
      "    step: 500; loss: 2.132; l2dist: 1.236\n",
      "    step: 600; loss: 2.124; l2dist: 1.234\n",
      "    step: 700; loss: 2.124; l2dist: 1.240\n",
      "    step: 800; loss: 2.120; l2dist: 1.239\n",
      "    step: 900; loss: 2.112; l2dist: 1.235\n",
      "binary step: 4; number of successful adv: 100/100\n",
      "    step: 0; loss: 15.216; l2dist: 0.000\n",
      "    step: 100; loss: 2.273; l2dist: 1.231\n",
      "    step: 200; loss: 2.165; l2dist: 1.227\n",
      "    step: 300; loss: 2.138; l2dist: 1.229\n",
      "    step: 400; loss: 2.133; l2dist: 1.230\n",
      "    step: 500; loss: 2.130; l2dist: 1.227\n",
      "    step: 600; loss: 2.129; l2dist: 1.226\n",
      "    step: 700; loss: 2.123; l2dist: 1.232\n",
      "    step: 800; loss: 2.123; l2dist: 1.238\n",
      "    step: 900; loss: 2.115; l2dist: 1.235\n",
      "binary step: 5; number of successful adv: 100/100\n",
      "    step: 0; loss: 14.526; l2dist: 0.000\n",
      "    step: 100; loss: 2.256; l2dist: 1.236\n",
      "    step: 200; loss: 2.171; l2dist: 1.230\n",
      "    step: 300; loss: 2.140; l2dist: 1.236\n",
      "    step: 400; loss: 2.130; l2dist: 1.235\n",
      "    step: 500; loss: 2.128; l2dist: 1.238\n",
      "    step: 600; loss: 2.131; l2dist: 1.242\n",
      "    step: 700; loss: 2.122; l2dist: 1.236\n",
      "    step: 800; loss: 2.118; l2dist: 1.237\n",
      "    step: 900; loss: 2.117; l2dist: 1.239\n",
      "binary step: 6; number of successful adv: 100/100\n",
      "    step: 0; loss: 14.199; l2dist: 0.000\n",
      "    step: 100; loss: 2.250; l2dist: 1.230\n",
      "    step: 200; loss: 2.176; l2dist: 1.225\n",
      "    step: 300; loss: 2.134; l2dist: 1.231\n",
      "    step: 400; loss: 2.131; l2dist: 1.233\n",
      "    step: 500; loss: 2.128; l2dist: 1.230\n",
      "    step: 600; loss: 2.125; l2dist: 1.232\n",
      "    step: 700; loss: 2.124; l2dist: 1.229\n",
      "    step: 800; loss: 2.116; l2dist: 1.232\n",
      "    step: 900; loss: 2.115; l2dist: 1.238\n",
      "binary step: 7; number of successful adv: 100/100\n",
      "    step: 0; loss: 14.023; l2dist: 0.000\n",
      "    step: 100; loss: 2.257; l2dist: 1.233\n",
      "    step: 200; loss: 2.179; l2dist: 1.228\n",
      "    step: 300; loss: 2.142; l2dist: 1.233\n",
      "    step: 400; loss: 2.133; l2dist: 1.233\n",
      "    step: 500; loss: 2.129; l2dist: 1.235\n",
      "    step: 600; loss: 2.114; l2dist: 1.231\n",
      "    step: 700; loss: 2.115; l2dist: 1.230\n",
      "    step: 800; loss: 2.109; l2dist: 1.231\n",
      "    step: 900; loss: 2.103; l2dist: 1.234\n",
      "binary step: 8; number of successful adv: 100/100\n",
      "    step: 0; loss: 14.131; l2dist: 0.000\n",
      "    step: 100; loss: 2.253; l2dist: 1.237\n",
      "    step: 200; loss: 2.167; l2dist: 1.231\n",
      "    step: 300; loss: 2.137; l2dist: 1.237\n",
      "    step: 400; loss: 2.126; l2dist: 1.237\n",
      "    step: 500; loss: 2.118; l2dist: 1.238\n",
      "    step: 600; loss: 2.112; l2dist: 1.236\n",
      "    step: 700; loss: 2.116; l2dist: 1.235\n",
      "    step: 800; loss: 2.116; l2dist: 1.237\n",
      "    step: 900; loss: 2.103; l2dist: 1.239\n",
      "binary step: 9; number of successful adv: 100/100\n"
     ]
    }
   ],
   "source": [
    "attack = CWL2Attack()\n",
    "\n",
    "def attack_batch(x, y, batch_size):\n",
    "    x_adv = torch.zeros_like(x)\n",
    "    total_num = x.size(0)\n",
    "    num_batches = total_num // batch_size\n",
    "    for i in range(num_batches):\n",
    "        begin = i * batch_size\n",
    "        end = (i + 1) * batch_size\n",
    "        x_adv[begin:end] = attack(\n",
    "            net, x[begin:end], y[begin:end], targeted=False,\n",
    "            binary_search_steps=10, max_iterations=1000,\n",
    "            confidence=0, learning_rate=1e-2,\n",
    "            initial_const=1e1, abort_early=False)\n",
    "    return x_adv\n",
    "\n",
    "x_adv = attack_batch(x_test[ind].cuda(), y_test[ind].cuda(), 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "ind_adv = np.zeros(x_adv.size(0))\n",
    "with torch.no_grad():\n",
    "    num = 0\n",
    "    for i in range(x_adv.size(0) // 100):\n",
    "        begin = i * 100\n",
    "        end = (i + 1) * 100\n",
    "        y_pred = net(x_adv[begin:end].to('cuda'))\n",
    "        ind_adv[begin:end] = (y_pred.argmax(1).cpu() != y_test[ind][begin:end]).numpy()\n",
    "        num += (y_pred.argmax(1).cpu() == y_test[ind][begin:end]).sum().numpy()\n",
    "    print(num / x_adv.size(0))\n",
    "ind_adv = np.where(ind_adv)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "& 0.0083 & 0.0063 & 0.0033\n"
     ]
    }
   ],
   "source": [
    "pert = (x_adv.cpu() - x_test[ind]).view(x_adv.size(0), -1).norm(2, 1)\n",
    "d1 = (len(ind) - (pert[ind_adv] < 0.5).sum().numpy()) / y_test.size(0)\n",
    "d2 = (len(ind) - (pert[ind_adv] < 1).sum().numpy()) / y_test.size(0)\n",
    "d3 = (len(ind) - (pert[ind_adv] < 1.5).sum().numpy()) / y_test.size(0)\n",
    "print('& %.4f & %.4f & %.4f' % (d1, d2, d3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.2168, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pert[ind_adv].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## DkNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "layers = ['layer4']\n",
    "\n",
    "dknn = DKNNL2(net, x_train, y_train, x_valid, y_valid, layers, \n",
    "              k=5, num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7825\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    y_pred = dknn.classify(x_test)\n",
    "    ind = np.where(y_pred.argmax(1) == y_test.numpy())[0]\n",
    "    print((y_pred.argmax(1) == y_test.numpy()).sum() / y_test.size(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load x_adv from file\n",
    "# Skip this if you you want to generate attacks on the notebook\n",
    "\n",
    "x_adv = pickle.load(open('x_adv/x_adv_cifar10_resnet_exp2.h5.p', 'rb'))\n",
    "\n",
    "with torch.no_grad():\n",
    "    y_pred = dknn.classify(x_adv)\n",
    "    ind_adv = np.where(y_pred.argmax(1) != y_test[ind].numpy())[0]\n",
    "    print((y_pred.argmax(1) == y_test[ind].numpy()).sum() / y_pred.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-a2f7d302ab0f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0mnum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0mind\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mx_adv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mattack_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnum\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnum\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-14-a2f7d302ab0f>\u001b[0m in \u001b[0;36mattack_batch\u001b[0;34m(x, y, batch_size, layer)\u001b[0m\n\u001b[1;32m     13\u001b[0m             \u001b[0mguide_layer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinary_search_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0mmax_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitial_const\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m             abort_early=False, random_start=True, guide_mode=2)\n\u001b[0m\u001b[1;32m     16\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mx_a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/research/knn-defense/lib/dknn_attack_l2.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, dknn, x_orig, label, guide_layer, m, binary_search_steps, max_iterations, learning_rate, initial_const, abort_early, max_linf, random_start, guide_mode)\u001b[0m\n\u001b[1;32m    127\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mguide_mode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m                 x_guide = self.find_guide_samples_v2(\n\u001b[0;32m--> 129\u001b[0;31m                     dknn, x_orig, label, k=m, layer=guide_layer)\n\u001b[0m\u001b[1;32m    130\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid guide_mode (choose between 1 and 2)\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/research/knn-defense/lib/dknn_attack_l2.py\u001b[0m in \u001b[0;36mfind_guide_samples_v2\u001b[0;34m(cls, dknn, x, label, k, layer)\u001b[0m\n\u001b[1;32m    276\u001b[0m         \"\"\"\n\u001b[1;32m    277\u001b[0m         \u001b[0;31m# find nearest sample with different class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 278\u001b[0;31m         \u001b[0mnn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdknn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_nn_diff_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    279\u001b[0m         \u001b[0;31m# now find k neighbors that has the same class as x_nn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    280\u001b[0m         \u001b[0mx_nn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_nn_same_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdknn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/research/knn-defense/lib/dknn.py\u001b[0m in \u001b[0;36mfind_nn_diff_class\u001b[0;34m(self, x, label)\u001b[0m\n\u001b[1;32m    302\u001b[0m             \u001b[0;31m# least one sample of a different class is found\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfound_diff_class\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m                 \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mI\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_neighbors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m                 \u001b[0mI\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mI\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m                 \u001b[0mind\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mI\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/research/knn-defense/lib/dknn.py\u001b[0m in \u001b[0;36mget_neighbors\u001b[0;34m(self, x, k, layers)\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mrep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m                 \u001b[0mrep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrep\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m                 \u001b[0mD\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mI\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m                 \u001b[0;31m# D, I = search_index_pytorch(index, reps[layer], k)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m                 \u001b[0;31m# uncomment when using GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/envs/py36/lib/python3.6/site-packages/faiss/__init__.py\u001b[0m in \u001b[0;36mreplacement_search\u001b[0;34m(self, x, k)\u001b[0m\n\u001b[1;32m    149\u001b[0m         self.search_c(n, swig_ptr(x),\n\u001b[1;32m    150\u001b[0m                       \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswig_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdistances\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m                       swig_ptr(labels))\n\u001b[0m\u001b[1;32m    152\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdistances\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/envs/py36/lib/python3.6/site-packages/faiss/swigfaiss_avx2.py\u001b[0m in \u001b[0;36msearch\u001b[0;34m(self, n, x, k, distances, labels)\u001b[0m\n\u001b[1;32m   2021\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2022\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdistances\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2023\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_swigfaiss_avx2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIndexFlat_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdistances\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2024\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2025\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrange_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mradius\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from lib.dknn_attack_l2 import DKNNL2Attack\n",
    "attack = DKNNL2Attack()\n",
    "\n",
    "def attack_batch(x, y, batch_size, layer):\n",
    "    x_a = torch.zeros_like(x)\n",
    "    total_num = x.size(0)\n",
    "    num_batches = total_num // batch_size\n",
    "    for i in range(num_batches):\n",
    "        begin = i * batch_size\n",
    "        end = (i + 1) * batch_size\n",
    "        x_a[begin:end] = attack(\n",
    "            dknn, x[begin:end], y[begin:end],\n",
    "            guide_layer=layer, m=300, binary_search_steps=10,\n",
    "            max_iterations=500, learning_rate=1e-2, initial_const=1e-5,\n",
    "            abort_early=False, random_start=True, guide_mode=2)\n",
    "    return x_a\n",
    "\n",
    "num = 100\n",
    "ind = np.where(y_pred.argmax(1) == y_test.numpy())[0]\n",
    "x_adv = attack_batch(x_test[ind][:num].cuda(), y_test[ind][:num], 100, layers[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    step: 0; loss: 21.100; l2dist: 0.000\n",
      "    step: 100; loss: 2.900; l2dist: 1.321\n",
      "    step: 200; loss: 5.201; l2dist: 1.305\n",
      "    step: 300; loss: 2.720; l2dist: 1.291\n",
      "    step: 400; loss: 3.637; l2dist: 1.284\n",
      "    step: 500; loss: 2.557; l2dist: 1.299\n",
      "    step: 600; loss: 3.194; l2dist: 1.308\n",
      "    step: 700; loss: 2.531; l2dist: 1.281\n",
      "    step: 800; loss: 3.008; l2dist: 1.283\n",
      "    step: 900; loss: 2.393; l2dist: 1.273\n",
      "tensor(68., device='cuda:0')\n",
      "binary step: 0; number of successful adv: 86/100\n",
      "    step: 0; loss: 86.745; l2dist: 0.000\n",
      "    step: 100; loss: 3.584; l2dist: 1.420\n",
      "    step: 200; loss: 14.398; l2dist: 1.390\n",
      "    step: 300; loss: 3.533; l2dist: 1.397\n",
      "    step: 400; loss: 8.149; l2dist: 1.367\n",
      "    step: 500; loss: 3.216; l2dist: 1.376\n",
      "    step: 600; loss: 6.926; l2dist: 1.363\n",
      "    step: 700; loss: 2.996; l2dist: 1.353\n",
      "    step: 800; loss: 6.676; l2dist: 1.374\n",
      "    step: 900; loss: 3.086; l2dist: 1.388\n",
      "tensor(78., device='cuda:0')\n",
      "binary step: 1; number of successful adv: 97/100\n",
      "    step: 0; loss: 204.001; l2dist: 0.000\n",
      "    step: 100; loss: 3.491; l2dist: 1.367\n",
      "    step: 200; loss: 39.980; l2dist: 1.328\n",
      "    step: 300; loss: 3.528; l2dist: 1.374\n",
      "    step: 400; loss: 14.203; l2dist: 1.370\n",
      "    step: 500; loss: 3.485; l2dist: 1.370\n",
      "    step: 600; loss: 7.338; l2dist: 1.365\n",
      "    step: 700; loss: 2.937; l2dist: 1.337\n",
      "    step: 800; loss: 10.796; l2dist: 1.329\n",
      "    step: 900; loss: 3.074; l2dist: 1.355\n",
      "tensor(69., device='cuda:0')\n",
      "binary step: 2; number of successful adv: 100/100\n",
      "    step: 0; loss: 126.025; l2dist: 0.000\n",
      "    step: 100; loss: 3.368; l2dist: 1.292\n",
      "    step: 200; loss: 22.284; l2dist: 1.249\n",
      "    step: 300; loss: 4.296; l2dist: 1.290\n",
      "    step: 400; loss: 10.309; l2dist: 1.279\n",
      "    step: 500; loss: 3.140; l2dist: 1.316\n",
      "    step: 600; loss: 7.657; l2dist: 1.296\n",
      "    step: 700; loss: 3.473; l2dist: 1.314\n",
      "    step: 800; loss: 7.606; l2dist: 1.293\n",
      "    step: 900; loss: 2.909; l2dist: 1.301\n",
      "tensor(65., device='cuda:0')\n",
      "binary step: 3; number of successful adv: 100/100\n",
      "    step: 0; loss: 117.700; l2dist: 0.000\n",
      "    step: 100; loss: 3.093; l2dist: 1.231\n",
      "    step: 200; loss: 17.196; l2dist: 1.210\n",
      "    step: 300; loss: 3.100; l2dist: 1.221\n",
      "    step: 400; loss: 6.240; l2dist: 1.218\n",
      "    step: 500; loss: 2.781; l2dist: 1.227\n",
      "    step: 600; loss: 4.721; l2dist: 1.233\n",
      "    step: 700; loss: 2.868; l2dist: 1.209\n",
      "    step: 800; loss: 5.310; l2dist: 1.209\n",
      "    step: 900; loss: 2.556; l2dist: 1.221\n",
      "tensor(66., device='cuda:0')\n",
      "binary step: 4; number of successful adv: 100/100\n",
      "    step: 0; loss: 102.282; l2dist: 0.000\n",
      "    step: 100; loss: 3.180; l2dist: 1.217\n",
      "    step: 200; loss: 21.679; l2dist: 1.188\n",
      "    step: 300; loss: 3.307; l2dist: 1.225\n",
      "    step: 400; loss: 9.907; l2dist: 1.230\n",
      "    step: 500; loss: 2.842; l2dist: 1.228\n",
      "    step: 600; loss: 4.812; l2dist: 1.216\n",
      "    step: 700; loss: 2.742; l2dist: 1.238\n",
      "    step: 800; loss: 4.793; l2dist: 1.226\n",
      "    step: 900; loss: 2.720; l2dist: 1.212\n",
      "tensor(57., device='cuda:0')\n",
      "binary step: 5; number of successful adv: 100/100\n",
      "    step: 0; loss: 95.338; l2dist: 0.000\n",
      "    step: 100; loss: 3.076; l2dist: 1.197\n",
      "    step: 200; loss: 21.158; l2dist: 1.175\n",
      "    step: 300; loss: 3.808; l2dist: 1.225\n",
      "    step: 400; loss: 7.385; l2dist: 1.218\n",
      "    step: 500; loss: 2.813; l2dist: 1.225\n",
      "    step: 600; loss: 4.456; l2dist: 1.214\n",
      "    step: 700; loss: 2.705; l2dist: 1.232\n",
      "    step: 800; loss: 5.313; l2dist: 1.218\n",
      "    step: 900; loss: 2.744; l2dist: 1.225\n",
      "tensor(56., device='cuda:0')\n",
      "binary step: 6; number of successful adv: 100/100\n",
      "    step: 0; loss: 90.572; l2dist: 0.000\n",
      "    step: 100; loss: 2.942; l2dist: 1.184\n",
      "    step: 200; loss: 20.036; l2dist: 1.176\n",
      "    step: 300; loss: 2.852; l2dist: 1.196\n",
      "    step: 400; loss: 7.243; l2dist: 1.197\n",
      "    step: 500; loss: 2.621; l2dist: 1.199\n",
      "    step: 600; loss: 5.222; l2dist: 1.201\n",
      "    step: 700; loss: 2.561; l2dist: 1.208\n",
      "    step: 800; loss: 6.241; l2dist: 1.189\n"
     ]
    }
   ],
   "source": [
    "from lib.dknn_attack_exp import DKNNExpAttack\n",
    "\n",
    "attack = DKNNExpAttack(dknn)\n",
    "\n",
    "def attack_batch(x, y, batch_size):\n",
    "    x_adv = torch.zeros_like(x)\n",
    "    total_num = x.size(0)\n",
    "    num_batches = total_num // batch_size\n",
    "    for i in range(num_batches):\n",
    "        begin = i * batch_size\n",
    "        end = (i + 1) * batch_size\n",
    "        x_adv[begin:end] = attack(\n",
    "            x[begin:end], y[begin:end],\n",
    "            guide_layer=layers[0], m=6, binary_search_steps=10,\n",
    "            max_iterations=1000, learning_rate=1e-2,\n",
    "            initial_const=1e-2, random_start=False,\n",
    "            thres_steps=200, check_adv_steps=200, verbose=True)\n",
    "    return x_adv\n",
    "\n",
    "num = 100\n",
    "x_adv = attack_batch(x_test[ind][:num].cuda(), y_test[ind][:num], 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([93.,  6.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.]),\n",
       " array([0.0584 , 0.15256, 0.24672, 0.34088, 0.43504, 0.5292 , 0.62336,\n",
       "        0.71752, 0.81168, 0.90584, 1.     ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAMJklEQVR4nO3cb4xlhVnH8e+vbAnSFqHdocEFHZpstYTEQCZIbVK1VFOpAV5Qg7G6bTZu0mitpdGivsDoG/AfatKoa6muplYQG9m0atMgpGpk06HUFlgJSHG7gmWqgH8abbGPL+6xrMvdncPOvXd4Zr+fZDNzzj2X89yT2e8eztx7UlVIkvp50WYPIEk6MQZckpoy4JLUlAGXpKYMuCQ1tW2RO9u+fXstLy8vcpeS1N4999zzxapaOnr9QgO+vLzM6urqIncpSe0l+cdp672EIklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0t9JOYG7F83Uc3Zb+P3vDmTdmvJK3HM3BJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJampUwJO8O8n9Se5L8qEkpyU5P8mBJA8luSXJqfMeVpL0rHUDnmQH8OPASlVdCJwCXAPcCNxUVTuBJ4Hd8xxUkvT/jb2Esg34uiTbgNOBx4E3ALcNj+8Drpr9eJKkY1k34FX1T8AvA4eYhPtp4B7gqap6ZtjsMLBj2vOT7EmymmR1bW1tNlNLkkZdQjkLuBI4H/gG4CXA907ZtKY9v6r2VtVKVa0sLS1tZFZJ0hHGXEJ5I/C5qlqrqq8AHwa+HThzuKQCcC7w2JxmlCRNMSbgh4BLk5yeJMBlwAPAncDVwza7gNvnM6IkaZox18APMPll5aeAzw7P2Qu8F7g2ycPAK4Cb5zinJOko29bfBKrqeuD6o1Y/Alwy84kkSaP4SUxJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaGhXwJGcmuS3J3yc5mOS1SV6e5ONJHhq+njXvYSVJzxp7Bv7rwF9U1bcA3wocBK4D7qiqncAdw7IkaUHWDXiSM4DXAzcDVNWXq+op4Epg37DZPuCqeQ0pSXquMWfgrwLWgN9Ncm+S9yd5CfDKqnocYPh69rQnJ9mTZDXJ6tra2swGl6ST3ZiAbwMuBn6zqi4C/pPncbmkqvZW1UpVrSwtLZ3gmJKko40J+GHgcFUdGJZvYxL0LyQ5B2D4+sR8RpQkTbNuwKvqn4HPJ/nmYdVlwAPAfmDXsG4XcPtcJpQkTbVt5HbvBD6Y5FTgEeDtTOJ/a5LdwCHgLfMZUZI0zaiAV9WngZUpD10223EkSWP5SUxJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1NTogCc5Jcm9ST4yLJ+f5ECSh5LckuTU+Y0pSTra8zkDfxdw8IjlG4Gbqmon8CSwe5aDSZKOb1TAk5wLvBl4/7Ac4A3AbcMm+4Cr5jGgJGm6sWfgvwb8FPDVYfkVwFNV9cywfBjYMe2JSfYkWU2yura2tqFhJUnPWjfgSb4PeKKq7jly9ZRNa9rzq2pvVa1U1crS0tIJjilJOtq2Edu8DrgiyeXAacAZTM7Iz0yybTgLPxd4bH5jSpKOtu4ZeFX9dFWdW1XLwDXAX1bVDwJ3AlcPm+0Cbp/blJKk59jI+8DfC1yb5GEm18Rvns1IkqQxxlxC+Zqqugu4a/j+EeCS2Y8kSRrDT2JKUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLU1LoBT3JekjuTHExyf5J3DetfnuTjSR4avp41/3ElSf9nzBn4M8B7quo1wKXAjya5ALgOuKOqdgJ3DMuSpAVZN+BV9XhVfWr4/t+Bg8AO4Epg37DZPuCqeQ0pSXqu53UNPMkycBFwAHhlVT0Ok8gDZx/jOXuSrCZZXVtb29i0kqSvGR3wJC8F/gT4iar6t7HPq6q9VbVSVStLS0snMqMkaYpRAU/yYibx/mBVfXhY/YUk5wyPnwM8MZ8RJUnTjHkXSoCbgYNV9atHPLQf2DV8vwu4ffbjSZKOZduIbV4H/BDw2SSfHtb9DHADcGuS3cAh4C3zGVGSNM26Aa+qvwZyjIcvm+04kqSx/CSmJDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDW1bbMHeKFbvu6jm7bvR29486btW9ILn2fgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmNhTwJG9K8mCSh5NcN6uhJEnrO+F7oSQ5BXgf8N3AYeCTSfZX1QOzGk6SZmmz7m00r/sabeQM/BLg4ap6pKq+DPwRcOVsxpIkrWcjdyPcAXz+iOXDwLcdvVGSPcCeYfE/kjy4gX12tB344ok8MTfOeJLNccKvfwvxGJzkxyA3bvj1f9O0lRsJeKasq+esqNoL7N3AflpLslpVK5s9x2Y52V8/eAzAYzCv17+RSyiHgfOOWD4XeGxj40iSxtpIwD8J7ExyfpJTgWuA/bMZS5K0nhO+hFJVzyT5MeBjwCnAB6rq/plNtnWctJePBif76wePAXgM5vL6U/Wcy9aSpAb8JKYkNWXAJakpAz4j691WIMm1SR5I8pkkdySZ+r7OrsbeViHJ1UkqyZZ7S9mYY5Dk+4efg/uT/OGiZ5y3EX8PvjHJnUnuHf4uXL4Zc85Lkg8keSLJfcd4PEl+Yzg+n0ly8YZ2WFX+2eAfJr/E/QfgVcCpwN8BFxy1zXcBpw/fvwO4ZbPnXuTrH7Z7GfAJ4G5gZbPn3oSfgZ3AvcBZw/LZmz33JhyDvcA7hu8vAB7d7LlnfAxeD1wM3HeMxy8H/pzJ52guBQ5sZH+egc/GurcVqKo7q+pLw+LdTN43v1WMva3CLwC/CPzXIodbkDHH4EeA91XVkwBV9cSCZ5y3MceggDOG77+eLfbZkar6BPCvx9nkSuD3a+Ju4Mwk55zo/gz4bEy7rcCO42y/m8m/wlvFuq8/yUXAeVX1kUUOtkBjfgZeDbw6yd8kuTvJmxY23WKMOQY/B7w1yWHgz4B3Lma0F4zn24rj2shH6fWsUbcVAEjyVmAF+I65TrRYx339SV4E3AS8bVEDbYIxPwPbmFxG+U4m/wf2V0kurKqn5jzboow5Bj8A/F5V/UqS1wJ/MByDr85/vBeE0a0YwzPw2Rh1W4EkbwR+Friiqv57QbMtwnqv/2XAhcBdSR5lcu1v/xb7ReaYn4HDwO1V9ZWq+hzwIJOgbxVjjsFu4FaAqvpb4DQmN7o6Wcz0FiQGfDbWva3AcAnht5nEe6td+zzu66+qp6tqe1UtV9Uyk98BXFFVq5sz7lyMubXEnzL5ZTZJtjO5pPLIQqecrzHH4BBwGUCS1zAJ+NpCp9xc+4EfHt6NcinwdFU9fqL/MS+hzEAd47YCSX4eWK2q/cAvAS8F/jgJwKGqumLThp6hka9/Sxt5DD4GfE+SB4D/AX6yqv5l86aerZHH4D3A7yR5N5NLB2+r4e0ZW0GSDzG5RLZ9uM5/PfBigKr6LSbX/S8HHga+BLx9Q/vbQsdOkk4qXkKRpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmvpfIZGo+G/mto8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cred = dknn.credibility(y_pred)\n",
    "plt.hist(cred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on x_adv: 0.0000\n",
      "Mean distance: 1.0605\n"
     ]
    }
   ],
   "source": [
    "y_pred = dknn.classify(x_adv)\n",
    "acc = (y_pred.argmax(1) == y_test[ind][:num].numpy()).sum() / len(y_pred)\n",
    "print('Accuracy on x_adv: %.4f' % acc)\n",
    "\n",
    "ind_adv = np.where(y_pred.argmax(1) != y_test[ind][:num].numpy())[0]\n",
    "dist = np.sqrt(np.sum((x_adv.cpu().detach().numpy() - x_test.numpy()[ind][:num])[ind_adv]**2, (1, 2, 3)))\n",
    "print('Mean distance: %.4f' % np.mean(dist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfHElEQVR4nO2da3Bd13Xf/+uc+8KLAAiSAPgWKcqR/Ahl06pdW4mcVxU1HdnTJrE749EHT5Tp2DP1TPpB487U7kw/OJ3aHn/ouKVrJkrqWlJtq1IbTRJHTuzGTSXBokQ9KEskxTcIvvDGBe7jrH7AVUor+78BEsAFnfP/zXAA7nX3Pevuc9Y9F/t/11rm7hBC/P0nWW8HhBDtQcEuRE5QsAuRExTsQuQEBbsQOUHBLkROKKxkspndC+CrAFIA/8Xdvxh7fE9HwQc2lK77OInZdc+JSYoObrPIsfhT3ph8GX1ZHnsfjvkfthmiB7sBS3yNOZH1jczKIsYbeWWxOVn0dUVmRpfj+tc49pqZj5OzDVQXmkEnbzjYzSwF8B8B/CqAswCeM7Mn3f1VNmdgQwmf/+c/FzZmTXqscjHspiUpnVOrL1Bbo1mntmKxSG1NdhFEr8QGNSVp5MJpdHGb8+cslueD4wXw1wXLqKkZOS/1Jp+XURO/5BoZXw9y/QJYInARdiSJfKhdqPPrI8siIdPka5VG1niBrPFsg7+yai0854//4hyds5KP8XcBOObuJ9y9BuARAPev4PmEEGvISoJ9G4Az1/z/bGtMCHETspJgD33G+DufZ83sQTMbMbORmSr/+CmEWFtWEuxnAey45v/bAZx/+4Pc/aC7H3D3A90dK9oPFEKsgJUE+3MA9pnZLWZWAvBxAE+ujltCiNXmhm+17t4ws88A+DMsSm+H3P2V6BwYFsj7i2OOzjMP72SWvDsyh+/UFwp81zSywQ8j06wY2dmtxXa6+bxCRHpLUu5kKSXzIgqEN8I7+ACQkN1sAPCI/zV0BMezQoXOWajx11Vv8mOZ8/OJLPy6OwqRtY/cAz3lykuTHAsAHPxPWEfY/5gCmCTXf59e0edqd38KwFMreQ4hRHvQN+iEyAkKdiFygoJdiJygYBciJyjYhcgJbf6WiwMZkSAynrjizfCcpMGlmqxeo7a0IyLj8AwOmnHRJP4BQLHAlziLJKc0IzJUQqQaAKg3iC8N/rpS48dKI1JZhjK1VRth29gVLk/NLHCtaWaGzytE1qOnEj7XxUgWWm9nWDYEgEqZn+um8Ws4jaTrFIhcWnI+p0F0uVgmpe7sQuQEBbsQOUHBLkROULALkRMU7ELkhLbuxptnKDRJ0kUkwSAhSRzllO/CeiTRwSKJJElkHtvAzSLllDy2PZryXd+hXbdR2/TkZWq7dGU2OF5I+K564nxXvVbjl8g8Oqnt6KlLwXGvbKRzGilPbKr3cP+vTlyhtnNjE8Hxrgp/Xc0L4TkAsGuQ11AciPhYKfBdfCNlxiIV0lAiyWGxWoO6swuRExTsQuQEBbsQOUHBLkROULALkRMU7ELkhDYnwhhta2SF/tisIA0iPwBAmnCpYyHSLaaccBmqUQ9LfVmkEwgiUl5nhUuAH/zVX6G2kR/9DbWdHw/LcnORTib1Ju8+c+pMWEIDgFPnzlJbpX9rcHx4aDed4+UN1FYrclmr0L2Z2hrz08HxK2OjdE5XP78Wz8yOUVs10iFnqJdLdp2k41FW43UZ2ZUT646jO7sQOUHBLkROULALkRMU7ELkBAW7EDlBwS5ETliR9GZmJwFMA2gCaLj7gdjjM0swn/YEbZOzPIMqa4alsr5uLq/1plwGKWRcKms2eO06Jms4yVoC4vXd5qpXqe37//MJahsb5+2aLk6H379Pnh+nc06NnqG2tMJluSzhUll396bgeLkzfP4BIKnwa6AYWcdKwn28tBCWr4Z37qRzFqoz1PbmmxeobXySn5dCwjP6dm0J29KYpMtqDUa0t9XQ2T/i7jznUghxU6CP8ULkhJUGuwP4czP7sZk9uBoOCSHWhpV+jP+Qu583sy0Avmdmr7n7D699QOtN4EEA6O/hXxkUQqwtK7qzu/v51s+LAB4HcFfgMQfd/YC7H+juaPNX8YUQf8sNB7uZdZlZz1u/A/g1AC+vlmNCiNVlJbfaQQCP22JBxQKA/+bufxqb0MgMl+bCEsp4o4/O+8GPfhAcv+M2Lrncc8cAtfUXuSznkcylpBD2PSXtewCg4VzKSyJvtW+eOUltl2Z4Zl69HF7HtLeX+9GcorZSD5eMmvM8e7Bq4XXc0MfPWU83P9bYGJe8rl7lBSd7S+FLvNzJs+hOjVeprdAzSG0XL5ymtu6L4ew7ABjcEJYcO5JI6zAi9xqv23rjwe7uJwD8/I3OF0K0F0lvQuQEBbsQOUHBLkROULALkRMU7ELkhPb2ekvLKPTuDtpmr0Tkq3K4oOCVWT6nWufSSm8p3DsOABoZtxVIs7c00rOt0eC2SzxJChcneCZd58ZwMUcA2DK0KzhePTdJ5/Q1eFOxYhf3v1bmsuL8dLjn3OwMl6B2DoYz5QBgjhRlBIDLC1wqS9Lwtzan5ngxR4tkm1VnIkUgCzxr7+IU7x93YSrs/85NPNPPMqKxRbLedGcXIico2IXICQp2IXKCgl2InKBgFyIntHU3vtLRhXe85x8EbWef+Qmd17UhvEt71z8MPxcAdCYnqW1hjh8rKfCcey+Gd6ZrzpN4evp3UNsLL75Bbd19fGd6x853Ulvdwjvr5UKkPhqrZwagNhXeVQcAkJ1uACh4+NI6+uIROqe3zJ+v0sUTaDojde0ujJIEmkjCSBJJQBno5urEeJ2rE+NXue3EaDgRaeuWITqnwBQl4/dv3dmFyAkKdiFygoJdiJygYBciJyjYhcgJCnYhckJbpbckTdHZuzFo273nNjqvWgvrJDt376VzNtV5LbnJk6eoreGRJIh6WHZ5/y/eT+fs3MM7Yu1+N/dj5PkXqK2/m0syY6QeW+o82aUQkZo8cj+YjiSFTIyHW1v1d8WOxfUwfjaBgcFwohQAzNfCEtXkFK+7ZxH5qrvEk13SAn9ttRpP1jlx5mxwfEsfl/n2bQ/LjbHzpTu7EDlBwS5ETlCwC5ETFOxC5AQFuxA5QcEuRE5YUnozs0MAfgPARXd/V2tsI4BHAewGcBLAb7n7+JJHS1KklbBkcO7CUTpt//veHxzv6uWZUIWZ89TWbHCJJy3xul/HT4frp324/xY6xzq3U1t3F5euOou8FVJnmdfXS9OwxNZ0/pqHd+yktldef53aihE/pmbC0tbuHVxive3226nt6lVew627ZwO1nRsdC443Ii27+vrD8jAATEVqyaUpLwDX0cEzI+dL4evgjTNcHuwoh/2vNyLty6jl//OHAO5929hDAJ52930Anm79XwhxE7NksLf6rb/9GxL3A3i49fvDAD66yn4JIVaZG/2bfdDdRwGg9XPL6rkkhFgL1nyDzsweNLMRMxuZmuR/gwgh1pYbDfYxMxsGgNbPi+yB7n7Q3Q+4+4ENvXwjRQixttxosD8J4IHW7w8AeGJ13BFCrBXLkd6+BeAeAJvM7CyAzwP4IoDHzOxTAE4D+M3lHMwsRZFIbwsLvCBfbSGcuVQq8wykjowXIeys8GyiSsqLL3YXFoLjf/CfD9E5/+S3P01txTlSDBFAqRTJXjLu4y23hqW+8+P8WNOk/RAAbNvCC1+Oz3LpcKEWXqs9t/JMxb23cllu6vBhapuc5i2lpufDr60eybCbmw/7DgC9vb3U1si4H739POuwQTLz0pT7cWY0/GG6VufXxpLB7u6fIKZfXmquEOLmQd+gEyInKNiFyAkKdiFygoJdiJygYBciJ7S14KSZISFZWXMzvKfYfDUsnxQjfdlm5iIlChMuvRWdf8tvuC+8XG8cPUbnnDt3nLsxN0ptJ0+/SW3vG7yL2rbv3Boc33lhmM459Sr3sSuSUdbbN0BtJ06E/d+6dRudEysCWW/wQqAXL4eLbAIAUW3hkQKcc7Nc8jLn15VF7p1dkR5xyMJyWdG4JFq/EpZSY0U7dWcXIico2IXICQp2IXKCgl2InKBgFyInKNiFyAltld7gDs/C0kASK4i4KZx51VnhBQ+//yKXk/ojBSdvHeCSTKUcln9KxXk65/JF3s8tmw/3QwOA3Xt5EctCpUxtnZ39wfEtG8OSHABM9/FaodNV/trmnRdYHBgI919Li9z3+RrP2KpFbNUql8rY9dZo8OdDk8t8zQbP9Nu8iffgc+OhVrTwGlfAfWxm4WKrxZSfS93ZhcgJCnYhcoKCXYicoGAXIico2IXICW1PhCmm4UP2RhIFqC0jWQ4AppzXp7s8HtlF3sCXpKtMWislvH7eyfM8oWUoUs9s9228FdJcjasJzx1+NTh+dpTv/Hf2h3fwAQDdfKf79Imz1JYk4TZamfP7y8IC332eneNJIbF2TfUsfLxzF2hBZJS7eYJVRyRiOju5OhRrlYV6+Nxkc3xnfXBLuMZiocjbl+nOLkROULALkRMU7ELkBAW7EDlBwS5ETlCwC5ETltP+6RCA3wBw0d3f1Rr7AoDfAXCp9bDPuftTyzlgmoRlr6HNPImgQN6TskjLqOEdPJFkZJQnp0xaOMEAADydCY73beISYG9Eyit2dFPbrlu59NbVx1syHTr0X4Pj1UhCy9wCl+Um53jiR5LyhJEtpF7fwgRf+9kKf74Nvfy8nHvtDWobuxiuTzczHT6XAFAu8tp6pUjyVdLk10GpxtcxnTsXHN/SyaXI3o5wHBUit+/l3Nn/EMC9gfGvuPv+1r9lBboQYv1YMtjd/YcA+Fu/EOJngpX8zf4ZMztiZofMLPIVLCHEzcCNBvvXAOwFsB/AKIAvsQea2YNmNmJmIxMTEzd4OCHESrmhYHf3MXdvunsG4OsAaNcCdz/o7gfc/UBfX9+N+imEWCE3FOxmdm17kY8BeHl13BFCrBXLkd6+BeAeAJvM7CyAzwO4x8z2A3AAJwH87nIOlliCcjGcUdTTz6W3RhbO5ClH2j/tu2UXtT03Es4YAoCpwl5qy5Lp4PjgVr6Mr772DLV98O4HqO1Hf/Mstc3ORNokLVwOjl8eO0PneKT+31SDZ1GlxqWh/iScsbW1wn2fuswltGbCt4WGBnnWW5PUk5ub4Vl0MzNclium/JprVvm8ejUsrwHAFlLDcFsPzwSdr4f9N34qlw52d/9EYPgbS80TQtxc6Bt0QuQEBbsQOUHBLkROULALkRMU7ELkhLYWnEzSBF094UyvjZt4plGDtM6Zt0hhwK4N1Nbfxws9nj4zRm0fvuudwfHqdMb96A5nXQHA6HlesPH4sZ9QWzPjkldG2h3NV7nk1dHLs+imIq2VNnTxDLA9+94dHH/2Bf66Dr92ktruvieUi7VIscwz4k4cOxYcn5qepXOyyD2wNs/ltZ2D/JqrdHEZbWAgLAV7gWfRNRKisZGsUkB3diFyg4JdiJygYBciJyjYhcgJCnYhcoKCXYic0FbpzT1Dsx6WPHo38uKLc/NhaYsMAwCShL+P7di5ndreeIVnXk3MhTOourt5ht2OW7kUcur1k9Q2Onae2j7w/gPUNkd6ovVu4Os7sI0X5zz943DvOACYr/IUq1JHOBOtb8tOOufOnm3UdukSlzDfPP0itc1Ww0VJJ6a49LZ5E5cie32U2nb18AKog5HCowULy6L1Gs/M67bwdZWAnxPd2YXICQp2IXKCgl2InKBgFyInKNiFyAlt3Y3PGnXMXAnvZnYUy3RebSGcEOCR+mgo8F3Jzf086eb15Di1Xbwa3sG9wpISAPT28Np6P/dOnjhx8gyvGVfL+A7/xHS4zdDeffvonFv38Lp7p0Ynqe3lV16itsvjncHxciRpZWMnt708xhNoxq7yJB+QZKmkzNWJ4e1cndgZSTTZ2cMTgyoJT2qpEVnJm0U+p0FiIlKDTnd2IXKCgl2InKBgFyInKNiFyAkKdiFygoJdiJywnPZPOwD8EYAhABmAg+7+VTPbCOBRALux2ALqt9w93POnxcLCAk4cOxG07bztdj7PwgkGWSOcmAIAhZRLeeUKt/V089ZQ3d1hqez2O95B5/zFnz1FbbOTF6it0reF2o6fvURt27eHE032vONOOqdc5BLPnp08cWV8nJ/uV157PTjuzs/Z6FWeSDJR5fPm6/x8Ts2EpcjBoR10zukrPAGlfwevX3ilzP2wjL+28Tp5bcVI+6dauDZgHVyOXs6dvQHg99z9dgAfAPBpM7sDwEMAnnb3fQCebv1fCHGTsmSwu/uouz/f+n0awFEA2wDcD+Dh1sMeBvDRtXJSCLFyrutvdjPbDeBOAM8AGHRfTO5t/eSfO4UQ686yg93MugF8B8Bn3T3y/cS/M+9BMxsxs5HpaV5zWwixtiwr2M2siMVA/6a7f7c1PGZmwy37MICLobnuftDdD7j7gR7SIEIIsfYsGexmZljsx37U3b98jelJAA+0fn8AwBOr754QYrVYTtbbhwB8EsBLZvZCa+xzAL4I4DEz+xSA0wB+c6knmlto4PDxsGy049130XlNC2ebWZ23QUKRt4aK/TkxMcllrYFN+4Pj9/2jj9A5+9/DJcVHH3+c2sy4hNK3oZ/atm4N19fr6e2jc9IGr8e2cYhfIluneCbXREdYhjr84hE65/wMzyhDiUteG4Z4FuPm7vC8pMAz1JqRrMLXPZzNBwDHRnlRxHLKn7O6EJb65iKXd70ZPi9TTZ4duGSwu/tfA2Ce/vJS84UQNwf6Bp0QOUHBLkROULALkRMU7ELkBAW7EDmhrQUn55uGN6bCmTyXmzzbDIVwhk9S4MUQ6xmXrizl73Hbhvi3fu/+YDhzrFLiksstu7ZS2z/+p79Nbd/+H39CbZcvTFDb6CRplTXP21qVwDWeK5Fss2OneCskJov6wG10Tv9mXnAys0hbI+NZe81yWCprkkKUAFBv8vM5lfF5lYjcWy5w6W2uFs7MqxUi4ZmG1zeLSLa6swuRExTsQuQEBbsQOUHBLkROULALkRMU7ELkhLZKbwsNw+tXwxLEE//7ZTpv/65wVtNwiWcgdZT4SxseGua2TTy7au+ecEaZOy8mOHr5CrUdeoTLa8+/8Aq1sd53AFCvhyWqJCLJNOthaRMAmhW+HlnCJa8CwhJrI1IQsZ7wAouV2JUayVKbr4VtHrnNFYr8ukojslw2z89LA3xekTiTRu7FC3XymtXrTQihYBciJyjYhcgJCnYhcoKCXYic0Nbd+KYZZpJwbbKnD4fbBQHAG8fDbv76+3h9t71bec21k8d5Usjd738XtVVK4USH6QW+w/zonz5LbYePnqe2aoO3EvKU109LimFfssguclLiz4cCvx80mzyBZoHsMNcafE6S8N3sWpPv/GdZZKebJKckCX9dnZ08oaXo3P/MuSrQjKghC+TcNGv8WKUeUlsvUSKMELlHwS5ETlCwC5ETFOxC5AQFuxA5QcEuRE5YUnozsx0A/gjAEIAMwEF3/6qZfQHA7wB4q1/S59z9qejBCgVs3Lw5aLs6zr/Bf2FiPDj+f468Ruc067sjnnBpZfPQNmqzNCyHPfciT+J56vv/l9pqTZ74kRW49JZGZCPWu6eZ8WQdjyRPIOPyj0fkvAaR3kpFLqG58deVpPycFUoRuYlIUT09vOZhIdKqKYlk0GQxG/jrBlnHoUGehNTTG7adKPPrZjk6ewPA77n782bWA+DHZva9lu0r7v4flvEcQoh1Zjm93kYBjLZ+nzazowD47U8IcVNyXX+zm9luAHcCeKY19BkzO2Jmh8yMtxYVQqw7yw52M+sG8B0An3X3KQBfA7AXwH4s3vm/ROY9aGYjZjbSqE6vgstCiBthWcFuZkUsBvo33f27AODuY+7edPcMwNcBBBusu/tBdz/g7gcKHZFGEEKINWXJYDczA/ANAEfd/cvXjF9b2+ljAPiWtBBi3VnObvyHAHwSwEtm9kJr7HMAPmFm+7FY9eokgN9d6onMDEXSeqlY4NJKg0hlJ8f4nwULs0ep7e737qO2zn7ermmStEL6q2dG6JxqRLqqNXiWV6XCZblmxlsyVeeqwXGP6GtpJCPLnV8iMQWwTFoXJUnkkotkbKHE16Ozk9vSNHy8eiSjbHo23I4JALKITDnf4Oelrz8sOQPA4PCm4HhPpPBedXoqOO6RDMDl7Mb/NcLqbVRTF0LcXOgbdELkBAW7EDlBwS5ETlCwC5ETFOxC5IS2FpyEO7JGWBqwSMaQF8LSSq3J3R+bmae2wz/hhR7vm+PaylQ2Exw/Nx6WQQCg3NNNbc05LjXNz/OWTB0RqalQCGe3VWs86y3WGsoi94NYBltm4XPjkaKMpTJvuzQdkcpqk7PU1tkZfk43fp5rTW6bqfLz0t0XltAAoHfzILXV6+Fz89qbvDBqkciv9Rr3T3d2IXKCgl2InKBgFyInKNiFyAkKdiFygoJdiJzQZukNNG1oMS0+TEoypZpZpA9Zkb+0NyPZcn/w2J9Q20fuORB+vtFLwXEAqDa5rNWMyFCFDp4FmJKecwDQWQg/ZzGSkTU/FZYUAaBe5/M8IlGVSMZWocDXg0lQAFAg2WsA0Ixk9M2RgimR5DAUUu5jX/9Gats0OExtV65cpbaJyxfC46e59HbrLXvChkhWnu7sQuQEBbsQOUHBLkROULALkRMU7ELkBAW7EDmhrdJbmqboJz2qqlWeOTZbDUsylQLPkmo0ePZPWqpQ2w+efYnaTpwPZ8tNzvLCkVdnuMzX4EoTuju7qK3ZzeWwcinc66sYkeu8g69HV6QIZCHynE2E59Ujcp1FJLQsUmSzGZHsaiTbr6PMMwc3DwxQW//AED9W5N45H5GCy+XwOmYFnlU4Ox8uiplFJGzd2YXICQp2IXKCgl2InKBgFyInKNiFyAlL7sabWQXADwGUW4//trt/3sxuAfAIgI0AngfwSXeP7C8vtiCqLYRrw1WSWE2w8G53MeV1yeqR58sSnoCSVHjNuFPnLwfH0wJ/z2zU+O5oo85t8wtcTZid4zXXjOyel8vhXXoA6I7sFBc6+U59Ytz/YiV8vP5Ovr61SJ25K1fHqS2LtNjqKoXXo6+Pqx1b+vuobWiI79RPzPK6h1PjE9Q2PRlWono38mNdvnQlON6o87VYzp19AcAvufvPY7E9871m9gEAvw/gK+6+D8A4gE8t47mEEOvEksHui7yVA1ls/XMAvwTg263xhwF8dE08FEKsCsvtz562OrheBPA9AMcBTLj7W58ZzgLYtjYuCiFWg2UFu7s33X0/gO0A7gJwe+hhoblm9qCZjZjZSH2Of0tOCLG2XNduvLtPAPgrAB8A0Gf2t50AtgMIfpfU3Q+6+wF3P1Ds3LASX4UQK2DJYDezzWbW1/q9A8CvADgK4C8B/LPWwx4A8MRaOSmEWDnLSYQZBvCwmaVYfHN4zN3/l5m9CuARM/t3AA4D+MZST5RlGear1aCtlPL3nQ7ipTe41BHJ34i2/mmCy0kZSe5oRuQ1NGPvp5FjRRIasmasXl/Yx4lxLl1drYeTKgBgQzeXqHr7uTTUy2rNGU9oybJIDbqEz0tIvTsAqBEJs5xG6v9FjtWocgmtQRK2AGB2MiyVAYDXwz4WyzwRZj4h15VFXhe1vOWI+xEAdwbGT2Dx73chxM8A+gadEDlBwS5ETlCwC5ETFOxC5AQFuxA5wTxS92vVD2Z2CcCp1n83AQinkbUX+fHTyI+f5mfNj13uvjlkaGuw/9SBzUbcPdw8TX7ID/mx6n7oY7wQOUHBLkROWM9gP7iOx74W+fHTyI+f5u+NH+v2N7sQor3oY7wQOWFdgt3M7jWzn5jZMTN7aD18aPlx0sxeMrMXzGykjcc9ZGYXzezla8Y2mtn3zOyN1s/+dfLjC2Z2rrUmL5jZfW3wY4eZ/aWZHTWzV8zsX7bG27omET/auiZmVjGzZ83sxZYf/7Y1fouZPdNaj0fNjPffCuHubf0HIMViWas9AEoAXgRwR7v9aPlyEsCmdTjuLwB4L4CXrxn79wAeav3+EIDfXyc/vgDgX7V5PYYBvLf1ew+A1wHc0e41ifjR1jUBYAC6W78XATyDxYIxjwH4eGv8PwH4F9fzvOtxZ78LwDF3P+GLpacfAXD/Ovixbrj7DwFcfdvw/Vgs3Am0qYAn8aPtuPuouz/f+n0ai8VRtqHNaxLxo634Iqte5HU9gn0bgDPX/H89i1U6gD83sx+b2YPr5MNbDLr7KLB40QHYso6+fMbMjrQ+5q/5nxPXYma7sVg/4Rms45q8zQ+gzWuyFkVe1yPYQ6U01ksS+JC7vxfArwP4tJn9wjr5cTPxNQB7sdgjYBTAl9p1YDPrBvAdAJ9193WrThrwo+1r4iso8spYj2A/C2DHNf+nxSrXGnc/3/p5EcDjWN/KO2NmNgwArZ8X18MJdx9rXWgZgK+jTWtiZkUsBtg33f27reG2r0nIj/Vak9axr7vIK2M9gv05APtaO4slAB8H8GS7nTCzLjPreet3AL8G4OX4rDXlSSwW7gTWsYDnW8HV4mNow5qYmWGxhuFRd//yNaa2rgnzo91rsmZFXtu1w/i23cb7sLjTeRzAv14nH/ZgUQl4EcAr7fQDwLew+HGwjsVPOp8CMADgaQBvtH5uXCc//hjASwCOYDHYhtvgx4ex+JH0CIAXWv/ua/eaRPxo65oAeA8Wi7geweIby7+55pp9FsAxAP8dQPl6nlffoBMiJ+gbdELkBAW7EDlBwS5ETlCwC5ETFOxC5AQFuxA5QcEuRE5QsAuRE/4fkgo7MdSg5+wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcrklEQVR4nO2daYylZ3Xn/+futa+90d2x2wvEjsE2KVmOyCCHJNhBjAyTZeAD8gcmHUVBGqTMB4uRBkaaD2Q0gPgwYtQMVswMYUkAYUXWxJaHyBONZGiMN2iMF9rupdxbVXftdz3zoa41bef5n6qu5Vbbz/8ntbrqPfd5n3Of+5773nr+95xj7g4hxNufwk47IIToDQp2ITJBwS5EJijYhcgEBbsQmaBgFyITSpsZbGb3APgygCKA/+7un48ePzEx4QcPHkzacpQALbC9NVZjq59BdL6It8Zq9YITJ05gZmYmuZAbDnYzKwL4rwB+H8BJAD82s4fc/edszMGDB/Hoo48mbe12eyNeBJar/wIw4/731PtossDmwfp74cqfgXnwegY2WHDtWCd5OPLdwg+84YJsjOA6oFORm+Pdd99Nx2zmY/wdAF5095fdvQHgWwDu3cT5hBDbyGaCfT+AE5f9frJ7TAhxFbKZYE999vhnny3M7LCZHTWzoxcuXNjEdEKIzbCZYD8J4PLdtgMATr/5Qe5+xN2n3H1qYmJiE9MJITbDZoL9xwBuNLNDZlYB8DEAD22NW0KIrWbDu/Hu3jKzTwH4B6xKbw+4+8+iMQagVCxudMrECaMd1aufaDc+Hri1fkS7yNaJdrqDU5Ln1o7mii7HcDee37MKxiaMts4DlSFULqInF1yrxOYb2PmPrqlN6ezu/jCAhzdzDiFEb9A36ITIBAW7EJmgYBciExTsQmSCgl2ITNjUbvyV4uCCR5T1RqWJTjrJYXWuHuaUbUzF2TCxYnflE1qoJwVr7NH6E4m1wKXXRqtFbYUCv1RD/4n0FmdZhvpgMFUk2fG1KrAXNJir07nya1h3diEyQcEuRCYo2IXIBAW7EJmgYBciE3q6G29mdGc9+gI/tW04EWZrt8jDhIUNehFtFsc7sWkb3fEFEJ0u3HEPElDaZFh9uUnHvHT8V9S2Z89uPlerzseNjyePV6v80m8HC2LhdnxkiozMxv2IXk865opHCCHekijYhcgEBbsQmaBgFyITFOxCZIKCXYhM6G0ijDtNQNhIIkyUALEtHVXISTcqvcWjuLRSKG5ApgzWqmA8OcWDxJVO4OPcpbQcNndpiY45e36W2vqGBqhtcniI2jqt9PO2Mr/PWZD8E7MxWY4atziJSnd2ITJBwS5EJijYhcgEBbsQmaBgFyITFOxCZMKmpDczOw5gHkAbQMvdp9Z4PAqFtJ5QLETvO7TXDZ8rciSaaSOl6zbW9QeFIGssrD8WzNdqpds1FUtB7bcmr/12dm6O2i7NL1PbSj0tXy0t8Qw1VPr5XAsr1DbQP8htfennHcmeYUZZ1PIqLF135eeMMkE3IvZuhc7+O+5+fgvOI4TYRvQxXohM2GywO4BHzOwnZnZ4KxwSQmwPm/0Y/z53P21muwE8ama/cPfHL39A903gMAAcOHBgk9MJITbKpu7s7n66+/9ZAN8HcEfiMUfcfcrdpyYnJzcznRBiE2w42M1swMyGXv8ZwAcBPLdVjgkhtpbNfIzfA+D7XXmgBOBv3P1/RQM6nQ4Wl9JyjbMKhQDKpbSbUfunYpFLTYXAtpHWP4Ww7Q8/XSTxRO2rFle4DMWyB/vIGgLAcoMXgZw+z6W38xe5rdNO+18PXrOl+QVqOzdzgdpOnzlDbbfccF3y+PXX8j8pi87XI24bxe+doaS7gaS3IrFGYzYc7O7+MoBbNzpeCNFbJL0JkQkKdiEyQcEuRCYo2IXIBAW7EJnQ04KTrU4Hs0tp2WhogGcuFUrl5PF2m0sknaAoYzmwBSaauRT2qSNZfgDQbqYz1ACgWOQvzZnpU9Q2NpHubdbfV6Fj6iu8CGRflY/bG3xJqkUKPa40GnTMUpnP1WxwudEKXM6bW06Pa4aZikGRTdtgH7iA4gZ6vQUt+Ci6swuRCQp2ITJBwS5EJijYhcgEBbsQmdDT3XgrllAenkjaWlFyCmlBZOX0Lj0AdMDrqrWDZAwLCrwVyBZox4NWQsH5rBi81wamRp3Xfit4eoffOly5GB/mrZVaJKEFADzYte7vT7dkWg524zvOL8dihc9VrvFxJVJ7rxXd5jrRc+bDCtFOfaTykOSaYlCj0EmLqo2l6Qgh3lYo2IXIBAW7EJmgYBciExTsQmSCgl2ITOip9Hb+wgV87ev/M2kLVAuUimmJbWS4Rsdcf+jXqG3q3TfxuUrB+x+RSKJaeJ2ozlxQz6zT4dLh5K5d1FYiySTtQE4qlngCysRocIkYtxmRS6OEnHaR25aDFlUX52apbfbSxeTxxblLdEyTJGsBgAfi1vjECLW9k9TCA4DhGkv04tdVici2UVKW7uxCZIKCXYhMULALkQkKdiEyQcEuRCYo2IXIhDWlNzN7AMCHAZx191u6x8YBfBvAtQCOA/gTd+f6RxfvOOqkJthKIHeUiZy0QGQVIG531Pn1d1Hbcp37USKyRrlcpWOswN9Pg4QyeDBuZDxqkJk+adROqhXIScVKsI7hvYK0fwpaTZ08xWvrnTjzGrVdODdDbSuNevJ4cyl9HACadS7zNRu8Xt879u+mtmsO8nZTbbLGhaB+YYfJwHTE+u7sfw3gnjcdux/AY+5+I4DHur8LIa5i1gz2br/1N7913gvgwe7PDwL4yBb7JYTYYjb6N/sed58GgO7//POLEOKqYNs36MzssJkdNbOjy4uL2z2dEIKw0WA/Y2b7AKD7/1n2QHc/4u5T7j7VN8DLHwkhtpeNBvtDAO7r/nwfgB9sjTtCiO1iPdLbNwHcBWDSzE4C+CyAzwP4jpl9EsCrAP54PZONjY7hj/7VHyZt9SVeRHGgvz9tCIo59ld5MUoLqgbOz81RW6edlmRKRS69lft4Zl6HFEMEgJUWl6jaraD4Ijln1E6qEshrxaAflhXDHkrJw41gyGJQFLN/MF3AEgDGRtItrwCg1Uyfs2r8dZmb5dfAiZPHqe2GQzdSW5lkbgJAm8hoUQbbRlgz2N3948T0u1vqiRBiW9E36ITIBAW7EJmgYBciExTsQmSCgl2ITOhpwUm4w5vpInql4H2HKTxDVf4lnVotKF64zIsNLjXTvdIA4OXjx9NzlbmM82uHrqW2l17hWV4PP/JDamsGL1u1mn7eUaHHoYE+ahseHOa2MS6H3X7ru5PHd09ymeyGAzwzrBD00ysGElWjnu4tVw6kyIVF7uO+PWPUtv/AXmprt3gm3dJiWh4cGCCSM4BiIcpvS6M7uxCZoGAXIhMU7EJkgoJdiExQsAuRCQp2ITKhp9Lb7KU5fP+hR5K2TtDLq2hEmihxyWtklEtG19ywn9p2TXI5aXJfun/cxATvvVYdCLKrfvEKtT17jNtWgh5xpAUYysHb+mA/z9q74eA11PZbd/4mte0aSK//YJDpF7TMQ7POJdFmkC23cDFdB7XVDjLs+rkUOTbG5bDXpnlRzJlZXhy1QmTivfu4lNffz3r68WtDd3YhMkHBLkQmKNiFyAQFuxCZoGAXIhN6uhu/tLSMn/z0uaStSlo8AUCjPp88Xqnynd077uA7xTh1kprOneHDbrn5luTxco3vuC+TRAwAKAV18m7/zfdQW2OZty4qk7ZXN153iI65OWiH9Y7JUWobCRI1OitpH09M8wU+e5HvWE9fOE9tCwsL1DZ3KZ30VF/hNQ8rQdJQucRtgaCEdpurCf0koeiWoMbf6HB6TCtIuNGdXYhMULALkQkKdiEyQcEuRCYo2IXIBAW7EJmwnvZPDwD4MICz7n5L99jnAPwpgHPdh33G3R9e61ytRgPnTqUTPMZHucSz/2A6IeCmW99Jx5QrvC7Zc0//iNr29HE5adDTmRrnz0/TMQNDPCFncpjP9S8/+H5qKxW45DgynJ5vcmKCjpmZvUBtx199kdouzvA2SayN1sICl7xmAwntApHQAKDZ4EktNVKTrxy0vCosch+Hh0eobWyMX8P9QzzBqtafrqVYrvHrY4742A6yidZzZ/9rAPckjn/J3W/r/lsz0IUQO8uawe7ujwOY6YEvQohtZDN/s3/KzJ4xswfMjNfXFUJcFWw02L8C4HoAtwGYBvAF9kAzO2xmR83saKvFvzoqhNheNhTs7n7G3dvu3gHwVQB3BI894u5T7j5VCr5XLITYXjYU7Ga277JfPwognd0ihLhqWI/09k0AdwGYNLOTAD4L4C4zuw2AAzgO4M/WM1ljZRknn/950jY3PEjHffju30oev/vu36FjHv3f/0Btu8e4HLabyCAA0EcUrz7j2Um7iRQGACPjXKphdckAoM1VRVQq6XpyrUCSOf0Cb0N1/OxZams2glp4fel17B/irZV29/NroNHgfwJ2gpp8FZJNWWLF+gAUAttwIKENjXD/iyV+zvn5peTx16Z5pl+9nh7TaHIZcs1gd/ePJw5/ba1xQoirC32DTohMULALkQkKdiEyQcEuRCYo2IXIhJ4WnOx4ByvLacng3be+m477wAd+L3l8coRncv2LO++iNjMuQw0XefHI4aG0nFSs8jHlCm8l1C5wyajT4UUlZ+d4qsLAaFpqcuOZcte9K11IEwB27+fFKGdmeCbayHj6G9T1Jl97c+5jJWwbxc+5spTODouKVCK4PuaX0sVPAeDEL09zP4ICl435leTxqEjl4FBaYm23+Bjd2YXIBAW7EJmgYBciExTsQmSCgl2ITFCwC5EJPZXeqrV+XHPDrUnbv77v39BxS+10T7TnX+B9w9oF/tT6hnh20mzw/jczwzKK0nIiADSbi9RWqvG5Wp20HAMAc5d4ocfydNrHU0ETu3qdZ0p1lnjvsMEBvo7HXzqRPP6rV9LHAcCLvPfd+K5Jams0+FotkLU6e+Zc8jgAGLgkWjAubVmRr1V/lRePHKumJd1aX1peA4CFufQ1125vruCkEOJtgIJdiExQsAuRCQp2ITJBwS5EJvR0N35sbBx/+LFUlStgdNcBOu6pn6VrpDWDelv1ICGgEySFoM3f/4qFdPG3aPc2SmbwaNc3sFlQI63RSO8ynzsfKBdNriYUnM81Msxr6DXq6R3y2RmegOJBW6tZqoQAyw3uf3sl7UdrhScalao8LPqrXDGolXhCVKHJCwc22Q6/8efVP5ROsLKgPqHu7EJkgoJdiExQsAuRCQp2ITJBwS5EJijYhciE9bR/Ogjg6wD2AugAOOLuXzazcQDfBnAtVltA/Ym7z0bnWlpewtPPPpm0PfPsT+m4YiGdKBDVVasEiQflMpdI2pH0Vk7LLkH+Bgb6uB/FYuQ/r11XqPFzFtrpl3S4xtsuFUgiBgC0SlwCXGrylkxOZMpyP/e9vsDlsMUL/NJqtPi4YjudnBIoVKgX+DXQbPDEpqVFnghTC2roVUjbqGrQAqxMXpfNSm8tAH/p7jcBuBPAX5jZzQDuB/CYu98I4LHu70KIq5Q1g93dp939ye7P8wCOAdgP4F4AD3Yf9iCAj2yXk0KIzXNFf7Ob2bUAbgfwBIA97j4NrL4hANi91c4JIbaOdQe7mQ0C+C6AT7s7r57wz8cdNrOjZna0Uee1s4UQ28u6gt3MylgN9G+4+/e6h8+Y2b6ufR+AZCNvdz/i7lPuPhVtOgkhtpc1g93MDKv92I+5+xcvMz0E4L7uz/cB+MHWuyeE2CrWk/X2PgCfAPCsmT3VPfYZAJ8H8B0z+ySAVwH88VonWpifw+P/+EjStjLPWwmVy+lPBLX+ITrGjD+1YofbOkGWV4FkQ5W4QoK+4NNMlUh5AFCp8fpuxQFej62vls5EKwc1+UrG5bVSLRKpeL2zeiMth9WXuUzWbnLpqhO0ZELQRqvYSWeUWZBhZ0Htt8GgbmDfMB/XXwte63La/2qBr4e1yTo6X6c1g93d/wlclvzdtcYLIa4O9A06ITJBwS5EJijYhcgEBbsQmaBgFyITelpwslwuYO+u4aTt1FLyOzkAgA7Jahqe4JlcJeNSx/w5LvMtzPGCiM1OOsurw2QQAAgKToYEUlm1fx+frpRe31aQIVgo8ff8gQqXDgdq3MYKbXo7KKRZ4TJfZKtWglZffekMx4lBnn23fyC9hgDwjj0T1Nbfz31srPDrqujpopilApfyRofTa18MipHqzi5EJijYhcgEBbsQmaBgFyITFOxCZIKCXYhM6Kn05t5Bh/QVGxviRSDnSd+wRovLGe+86RbuyF6eNXbu7HlqO3M+3UdtaZ7La0uLXJZrtnjBRnT4+3C/8SIgv/6e25PHT83xeiPnLl2ktuWowOIy70VWIgUnq+UgM6zML8exQCrbNTJCbXsPpGXK6/bzwkp7AilvcZHLthdm09cHAJSCgpMDY2PJ40NEXgOA8Yn0mFKJ+647uxCZoGAXIhMU7EJkgoJdiExQsAuRCT3djW83mpg5fTJpY8kuALBI6motrLxKx4yXeCLM7jKv71Zt8J3uQbJajaD9U7vd5Mag9hsKfId/cYUrBr89dVPy+C03v4eOeeUVvo4XZmeordEInht5zYrB/WWgxBNJxit8kccGefuqtqXruE2f58/5+fOvUZsFteRGd3GVpzbEr7mBobT/40Gi1xBRIKKWYrqzC5EJCnYhMkHBLkQmKNiFyAQFuxCZoGAXIhPWlN7M7CCArwPYi9V+P0fc/ctm9jkAfwrg9W//f8bdHw4nKxexZ19aTjh54hQd16kTiafI2+O8+svnqW2hyBMMomZHC520HwukxRAAdJz7WCxGs3Ea9Xlqe+r/Ppo8ftcgr6v2G1ytQX2cj2sF7ZqM2OqNdFITAFxscdu5CzzJ5JXnuVR2YSWdALRU5mvfF9SZG9u7l9pqg/y6KvXzBKDaSLqNWXWQtzdDkYUuf17r0dlbAP7S3Z80syEAPzGz16+oL7n7f1nHOYQQO8x6er1NA5ju/jxvZscA7N9ux4QQW8sV/c1uZtcCuB3AE91DnzKzZ8zsATNLJ9gKIa4K1h3sZjYI4LsAPu3ucwC+AuB6ALdh9c7/BTLusJkdNbOj7XbQdlcIsa2sK9jNrIzVQP+Gu38PANz9jLu33b0D4KsA7kiNdfcj7j7l7lNRAXshxPayZvSZmQH4GoBj7v7Fy45fXu/nowCe23r3hBBbxXp2498H4BMAnjWzp7rHPgPg42Z2GwAHcBzAn611okqtggPvOpi0zS1xOWnxRFp2KRS4ZrTS4PXdZgs826wcvP81kZbYOuB/nhj4XJ2gFVIp8KMdnPOFp3+cPH5qjmeo7QraDLWCxLxO8EltwdJr8hppdQQAL9Z5TbsTbf56Lvfzy3joYLoG3e5D19AxtTEuN1rQRgvBegxEWW+j6fm8WKFjHIEfhPXsxv8T0uJdqKkLIa4u9Ee0EJmgYBciExTsQmSCgl2ITFCwC5EJPS04WSyVMEJa3ezes4uOe41Ib0QJAwA0A8nI23ygBTJam0pvnGiBec5Y7EdEYzldMHPx/Fk6plgdpbZSkKV2MngBnvJ0AdEXy/x5LQ7wYo79B3nxxYm9aXkNAHZNprPUqkE7qXonkFIL/P5YLXNbIchwLJBzloMim4USmcuCeahFCPG2QsEuRCYo2IXIBAW7EJmgYBciExTsQmRCT6W3ghXQV0n3tarWeOZVmUgarVYgkQS1HFtRj7Wo5CTNeOLna22wYEcj8oPJLgAWWmk57BdN3sNuqMoLJR5bOUNtP2/xLLWZobS0NXHwEB2z5xCX0MZIoVIAqA3wwoyFVnodmx3+mhUrPCxKlRq3BdewBa9Zh/TFi67hIrkWoxKmurMLkQkKdiEyQcEuRCYo2IXIBAW7EJmgYBciE3oqvTkcLdIXbSEoODk0lpaGlhfTmVUAlzMAoBUVc4yy5cg5C9GgDWYhEQWtO5AXG1wspX38P/WLdMyvCjz/7kI/97Ky+wC17du/O3n8ml28j9rkKLcV+tOSLQAsBYLTMpFFS0Gvt6Eal9cq/TxbrhxImNUaH1eppCW7Uolnvbmnn1ckKuvOLkQmKNiFyAQFuxCZoGAXIhMU7EJkwpq78WZWA/A4gGr38X/n7p81s0MAvgVgHMCTAD7h7rxHD1Z3EBut9A56qcr3EUd3pXdiBwZ5e5woSaYZ7HS3gvpjTnbdC1GuywZ349241Yt8l7ZcIokfA3yt6sM8yeSGsfSuOgCMjvCWRkPD6fmGhvglVwkSUOrR6xnV6yNrVaryuTx4YaIadNHuebkc2dK+lItReEb77mnWc2evA/iAu9+K1fbM95jZnQD+CsCX3P1GALMAPnnFswshesaawe6rLHR/LXf/OYAPAPi77vEHAXxkWzwUQmwJ6+3PXux2cD0L4FEALwG46O6vfxvjJID92+OiEGIrWFewu3vb3W8DcADAHQBuSj0sNdbMDpvZUTM7Wl/h33gTQmwvV7Qb7+4XAfwjgDsBjJrZ6zsIBwCcJmOOuPuUu09F1WiEENvLmsFuZrvMbLT7cx+A3wNwDMAPAfxR92H3AfjBdjkphNg860mE2QfgQTMrYvXN4Tvu/vdm9nMA3zKz/wTgpwC+ttaJzIAiSUAYHed1xAb70okf7aDHUzPQ19phtgs3/f8PMm828PfMYiDVGK1pBxSDmmXFMneyv5w+59AQTyTZPTBMbUM1ntwxUOE2JjW1akH9vCK3rXjwegbrXyOyViWQtUplLlMWivw1s8D/TiDpNprN5PE6kakBoFria89YM9jd/RkAtyeOv4zVv9+FEG8B9A06ITJBwS5EJijYhcgEBbsQmaBgFyITjNWy2pbJzM4BeKX76ySA8z2bnCM/3oj8eCNvNT+ucfddKUNPg/0NE5sddfepHZlcfsiPDP3Qx3ghMkHBLkQm7GSwH9nBuS9HfrwR+fFG3jZ+7Njf7EKI3qKP8UJkwo4Eu5ndY2bPm9mLZnb/TvjQ9eO4mT1rZk+Z2dEezvuAmZ01s+cuOzZuZo+a2Qvd/8d2yI/Pmdmp7po8ZWYf6oEfB83sh2Z2zMx+Zmb/tnu8p2sS+NHTNTGzmpn9yMye7vrxH7vHD5nZE931+LaZ8fS8FO7e038Ailgta3UdgAqApwHc3Gs/ur4cBzC5A/O+H8B7ATx32bH/DOD+7s/3A/irHfLjcwD+XY/XYx+A93Z/HgLwSwA393pNAj96uiYADMBg9+cygCewWjDmOwA+1j3+3wD8+ZWcdyfu7HcAeNHdX/bV0tPfAnDvDvixY7j74wBm3nT4XqwW7gR6VMCT+NFz3H3a3Z/s/jyP1eIo+9HjNQn86Cm+ypYXed2JYN8P4MRlv+9ksUoH8IiZ/cTMDu+QD6+zx92ngdWLDgAv2L79fMrMnul+zN/2Pycux8yuxWr9hCewg2vyJj+AHq/JdhR53YlgT5Wq2SlJ4H3u/l4AfwDgL8zs/Tvkx9XEVwBcj9UeAdMAvtCric1sEMB3AXza3ed6Ne86/Oj5mvgmirwydiLYTwI4eNnvtFjlduPup7v/nwXwfexs5Z0zZrYPALr/n90JJ9z9TPdC6wD4Knq0JmZWxmqAfcPdv9c93PM1SfmxU2vSnfuKi7wydiLYfwzgxu7OYgXAxwA81GsnzGzAzIZe/xnABwE8F4/aVh7CauFOYAcLeL4eXF0+ih6siZkZVmsYHnP3L15m6umaMD96vSbbVuS1VzuMb9pt/BBWdzpfAvDvd8iH67CqBDwN4Ge99APAN7H6cbCJ1U86nwQwAeAxAC90/x/fIT/+B4BnATyD1WDb1wM/fhurH0mfAfBU99+Her0mgR89XRMA78FqEddnsPrG8h8uu2Z/BOBFAH8LoHol59U36ITIBH2DTohMULALkQkKdiEyQcEuRCYo2IXIBAW7EJmgYBciExTsQmTC/wOOf//hOSLA+QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAd1klEQVR4nO2da4yc53Xf/2feue2Fy+WdFKVUF0uWJaWWXVZw4CBwnTZVjRSygSawPxj6YIRBEQM1kH4QXCB2gX5witqGPxQO6FqIUri+NLZhITBau4oL1y2qmFJkShZlS5R5EyleluTeZuf6nn6YUUEpz//sksudpf38fwDB3efM875nnnnPvLPPf8455u4QQvzqU9lsB4QQ40HBLkQmKNiFyAQFuxCZoGAXIhMU7EJkQnU9k83sYQBfAFAA+E/u/pno8Vu2bvcde/enjYEE2O/301PKks5pNBrUVhQFtZlREyrEaMGk4HChzZ0/t6K4jvMFJ7PA2B+k137oR7CO5JjRWkU4+PVxXUcMJpUDvvaR/5UKv3eWwbXK5O/oeIxTJ09ibu5i0snrDnYzKwD8RwD/BMBpAD82syfd/UU2Z8fe/fiTP3syfbxBj57r4vnXk+PdTpvOufPOt1Hb7Nat1FYLAqleS1/c9eCirxf8BasavwD6vRVq2zJVpzbiIqrB8yoKfhlcujRHbTMzW6itWq2lxyv8XJUgAPtll8/jyw8W1RXjr8vy0jK11cjzAoDmRJPaup0OtfW6advkxCSdA+L/+9/3m3TKej7GPwTgFXd/1d27AL4G4JF1HE8IsYGsJ9j3Azh11e+nR2NCiJuQ9QR76vPR3/njw8wOmtlhMzu8NH9pHacTQqyH9QT7aQC3XfX7rQDOvPVB7n7I3Q+4+4HprdvXcTohxHpYT7D/GMDdZnaHmdUBfBhAevdNCLHpXPduvLv3zezjAP47htLb4+7+02hOUalgaiK9k1w431LtTKZltIVei85p1vnW7uQEP1ct2BI2HyTHG1X+njnR4DYL5Ebr8536eo0fs0G24y2QrmpsCx9cgQCASBkypNdqosl3rC2QG5dbXK2J7li1evp6c4ueF78GalU+r1rl4dQLduOrZGd9IpCPnbgY+b4und3dvwvgu+s5hhBiPOgbdEJkgoJdiExQsAuRCQp2ITJBwS5EJqxrN/5aMTjqlbQkw2QtAKhX07JRzficRsGztZpFIEMFK9JtpaW+ouASSSTL9YJEnopxqQl9Pq8ESdQIZK1QXrNAHgxeM5b11uvxhJZOm0uply6ep7Y9O7ZxP0giUlHnCS3VQKaM1iNKouoFkliHZBYGKh+6XXJ9B3Ku7uxCZIKCXYhMULALkQkKdiEyQcEuRCaMfTee7aB7VHYI6Z3paoXvMNfAd+MrJd/1rVf4zroVad+rFe57zfiurxufNxgEZYxWAhViejo5vhIkYkxO8fJHRbCLXJLagAAv37Tc5krCM888S239Ni8Vte3AP6S2OklECjbOw4QcK/naV6JWasG8skyvY1i3jh0vcEF3diEyQcEuRCYo2IXIBAW7EJmgYBciExTsQmTCeKU3M9Srac3Do7pflbSeYCWXkwrnspD1+bwaePZBjySglCV/zyxmuJSHkie7RFKkDwJ9hSRVLC/O0ylbpiaoLWq2MgiSWga1tPR2pcU73Vxa5PLaZJAV0u3za6fbS8tXtQZfwzKQ3gZEJgOA/oCvR6/Hr7k6qV0XtTcrB0TCjpJ4qEUI8SuFgl2ITFCwC5EJCnYhMkHBLkQmKNiFyIR1SW9mdhzAIoABgL67H4geX4GjaWnpok/GAV5rLmqpUwmOhzKq/caXpEbq2lVJnTMAKKJacoF0aEGts14g8fQ9fb6lxQU652RQC8+C9DAPJKrbtqQz6eaCWnJHjhyhtl+//wFqK4mcCwAdkj3YdP46l87Xd6XF5cF6la9Hr8vnVWvpteoHWYWdTjpzM5LrboTO/o/c/eINOI4QYgPRx3ghMmG9we4Avmdmz5jZwRvhkBBiY1jvx/j3uvsZM9sN4Ptm9pK7//DqB4zeBA4CwJ59t6zzdEKI62Vdd3Z3PzP6/zyAbwN4KPGYQ+5+wN0PzM5uX8/phBDr4LqD3cymzGzLGz8D+B0AL9wox4QQN5b1fIzfA+DbZvbGcf6Lu/+3eEqJCpFCrMeLQBb9tBTSmueZXB4UKPSgQGQxybOr6oO0rFUv+BzrLVHbIPKxH7RkClpKsSKWS0tX6JzlJZ6JNjUzxc8VPG+cSV9a3eBczVqT2i5e4f4/8/xPqG26mfbjrjvvoHOi9k/d1iK1NYsgSy2QN7ski7EfRKd1iJQatOS67mB391cBvPN65wshxoukNyEyQcEuRCYo2IXIBAW7EJmgYBciE8ZacLICoEmyuaI+WRUivTWD/llbgiKQM8HTrswHWU3kfE3wrKvKCpearM3lxkbBZSiUQYHFhbS0OTPNj7dtB/+y0/HT56jt2KnXqe3nL/+P5PiVOS6XLq7wLMB276fUVkGQBdhKS3YPvP3tdM4//8A/pbZb9+6ktm6DX4+dIFuuu3w2OT6zZxedgxUiARJ5GNCdXYhsULALkQkKdiEyQcEuRCYo2IXIhLHuxne7XZw6fjxp63X5TuziQnrncRC0cXrttdPUdrlZp7aloE3S7p3pXevpqH1SwXdou0GNsWqd754XVd5SamklvevbDurkwXlCy4kzF6jtF6cuUdtyN+1jc2Y3nWOTPJFkOkhOmapz/8+e/Fly/LUzXGX40f/639R27913Uduu2Rlqay8HiUgLc8nx3r330DlL8+njtTtcmdCdXYhMULALkQkKdiEyQcEuRCYo2IXIBAW7EJkwVultaWkJP/o//5dYg7pqpM3Qygqv73b8bDq5AACM55GgVnCJZ9tMWlqZmuBSWCM4V9Q2qtqIpDduWybSS3Url4W8wqXIc3M8gaPv3P+pmdn0ucDlxu4STwyKLtQ2aYUEADNbtibHf+PdvJ3U0jyXFDtBLbmTpy5T27Fjx6htpZe+5k5c5ElU7NqfX+ZzdGcXIhMU7EJkgoJdiExQsAuRCQp2ITJBwS5EJqwqvZnZ4wB+F8B5d39gNLYdwNcB3A7gOIDfd3euO4xotTt47uW0BDE5sYXOK8u0XNPpc5lh67Yd1NYIZK1eIP9cWEzLLoXx98zp5iS1DcjzAgCr8WNWKtz/SjXdrqm+VKNzuj2e6Xf5EpehEGSisbJ8XdL+CwCWWvz17Ab16W4l2YgAsHP/3uT4cou0TwJw+QrP9Ns5y9f+wIP3U9trZ3gW5gIJw6On09lwAFCQ1ls90koKWNud/c8BPPyWsccAPOXudwN4avS7EOImZtVgH/Vbf+vb+yMAnhj9/ASAD95gv4QQN5jr/Zt9j7ufBYDR/7wigRDipmDDvy5rZgcBHASAxgT/+1UIsbFc7539nJntA4DR/+fZA939kLsfcPcD9Tr/DrkQYmO53mB/EsCjo58fBfCdG+OOEGKjWIv09lUA7wOw08xOA/gUgM8A+IaZfQzASQC/t5aTDRxYJN1pyjp/35mcSstyEwUv9Hjrr/HCgL0ub5Fz4XWeLXfhUloK2bOHb1k0dt1GbZeucGmlNF58ces2fr5mIy1DtflTRmvAZajGFM+WK/s8I66wdKHNRoV/uqs3eOHIbpPbHvoHPIPtnl+7JTne6XLff3GM+3js5y9S22889OvUdhvxAwBOHjmRHI9ktA6xlSWfs2qwu/tHiOm3V5srhLh50DfohMgEBbsQmaBgFyITFOxCZIKCXYhMGGvBSasUqDamk7Zdu7k0MUEkmQsXT9E5y8tcTkLJq0C2uzwTbeuudAbV/jvupnO2bN1GbTM7uYR26RJPIhw4f9m6RJJpkR5wALDcSvfSA4Bul2eimfG1qpP+a03y+gNAFbxP2S5SOBIAdm3j8mCzln6tdwfy5Uydr+/FUyep7fixtIQGAHu276K2+XPpIqy17dzHbiW9vsGlrTu7ELmgYBciExTsQmSCgl2ITFCwC5EJCnYhMmGs0ltRVDE7uzPtSMFdaZP+Wha8V12au0JtiwtchqpUed+zapn28cTpc3TOzAIvYDkbyHJRUck26ec2JC2HNeu84KSBFxUpJ7kfFlw95mkJcLLJMxXrQR+4/dvThTQBYIrIfADQWkgX0xy0+DUQcUcgsx59ifdzu+eee/lB++kMwbOv8SKV9W3p7MZywLMldWcXIhMU7EJkgoJdiExQsAuRCQp2ITJhvIkwZqjW07vdy22+w1yQNkNFle8UDwZ8h7Za8GSM0nkmQa1Oknh28iSe6Wm++9yc4LXOtga71tUaVwyY+2Wf1ybr94Pknxm+VpUK3/kty3TRu6rz19k7S9yPBn9dyh5vDdVn7ab6/D630uEF+yaDhJwTr/NWWS8e+x61tbtpxabbSe/SA4AvkESYAZ+jO7sQmaBgFyITFOxCZIKCXYhMULALkQkKdiEyYS3tnx4H8LsAzrv7A6OxTwP4AwAXRg/7pLt/d9WTVWvYvjNdx80DaWiqmXazHKQTZACgHiSS7Nq9j9qiRJh6M33MeiCTNYOWRkWNv9e6RcXEuM0q6WMWxs+1vMQlrwJcXmsEEmDJauFduUjnnDnxMrXNVflznp3gl/GeHbPJ8eYET6zpBHUIyyJIlJpMtykDgAunzlDbbbeka81t6fC1X+ykfSyCa2Mtd/Y/B/BwYvzz7v7g6N+qgS6E2FxWDXZ3/yEA/m0BIcQvBev5m/3jZnbEzB43M56YLYS4KbjeYP8igLsAPAjgLIDPsgea2UEzO2xmh9st/rehEGJjua5gd/dz7j5w9xLAlwA8FDz2kLsfcPcDzUn+PWshxMZyXcFuZldvZ38IwAs3xh0hxEaxFuntqwDeB2CnmZ0G8CkA7zOzBwE4gOMA/nAtJ6tUCkxNpbOGuis8G2pyKi15bZ3ZQ+c4T/5BtcazzSanuXxSEvnKgvp5Jbj0Vgky7Dx4G7aCzxsQqazf422cBgNej+3ypTlqK0idOQCokae9NH+ezjl7hstTe4MWT9um03UNAWCll/ZxEEh5ZRAWTFIEgP37b6O2e992F7W98747kuMv/4K3N/vbIy8lx5+p8VqDqwa7u38kMfzl1eYJIW4u9A06ITJBwS5EJijYhcgEBbsQmaBgFyITxlpwsixLLLfSmWrTQcZQQaStCxd4BtX8PG//5M7lsLfdfQ+1ze7YlRwvqvx4CKS3XlAcsNfjUuRyl0tlnU66eGG/u0DnoB8VgeTFHKebPANsdibdnmiill5DAKgFiX6zkzyzcGYLb1/VI8Uo28E10GnzbEqSVAgA2D7Li1FO1vnE06dOJMcLrvLh/re/LTn+V00uK+vOLkQmKNiFyAQFuxCZoGAXIhMU7EJkgoJdiEwYe6+3OsnKmbvAs6FevXwhOT4ouWQ0u40Xz9m7l2fLdfq8z1e3k84cK4IUu4UWl8naK2mZDAAGPS7/FMFbdI3oV5N1/lJPTAUFMwNZsdPmz21A1mQqyCosgiKbzSr3vwiyAGFpeXClx19nC56zBQU4u10uU85d5FJwqzWfHK9VuYy2d+/+5LgFa6g7uxCZoGAXIhMU7EJkgoJdiExQsAuRCWPdjR8M+pi/kq5pduY0rz82NZVOdLj3vgfonO07+Y77RJBU0V7hO8xXLqd7ZbBkCwBoIaitFyV3bOXtq6bqPAFlspG2RTvdPuA+drvc1utxFaKCtK0w3lqJJTwBQK/Pd8Gt5PNqpJ2Xl7wmX7vLlZC5IPlqbo7X61tYWKS2yyRpa2qSJ/jUp3ckx/tBjTzd2YXIBAW7EJmgYBciExTsQmSCgl2ITFCwC5EJa2n/dBuAvwCwF0AJ4JC7f8HMtgP4OoDbMWwB9fvufjk6VlGtYduutCS2beduPo8U/qo2uDy1uMiljsUlXo+t2eTtc3qkVls54EkV+3bzmmuNoF5YNShA5iWXr5ZIck1nkXfQvRK0eJq7lE5CAoBWmyfyvOPedC2/epCgNOwmlqYIsn/afb4e88vp1/r0udN0zoWLPCmr1+Wv9fISX+OFK/yaqxfpxJvF4DX76x/8NZnDz7OWO3sfwB+7+zsAvAfAH5nZfQAeA/CUu98N4KnR70KIm5RVg93dz7r7s6OfFwEcBbAfwCMAnhg97AkAH9woJ4UQ6+ea/mY3s9sBvAvA0wD2uPtZYPiGAIB/DhdCbDprDnYzmwbwTQCfcPegCPnfmXfQzA6b2eH2Mv87Wgixsawp2M2shmGgf8XdvzUaPmdm+0b2fQCSuxrufsjdD7j7geYUr1IihNhYVg12G9a5+TKAo+7+uatMTwJ4dPTzowC+c+PdE0LcKNaS9fZeAB8F8LyZPTca+ySAzwD4hpl9DMBJAL+3lhP2y3T2UmOCy2g1UrduELRPKozLOLWwxhin0UhLZStBq6b2PJdP2oFUUwS9kIqglZATGeqlF1+kc06eOE5t/SCjD8Yz0W4hdf62z87SOSvLPONwpcVlvvkr6RpuAHDxcjpLrd3lWW/9QMprLXM/FgK5txLUrpsktebOnj1L57x+7vXkeDtoXbVqsLv7j8Bj4LdXmy+EuDnQN+iEyAQFuxCZoGAXIhMU7EJkgoJdiEwYa8HJdnsFL//8aNL2jvvvp/MmSBHFkqtrsEBEKwdcWjk3xzOeWgtpiaezwmWcQSDj9IPstTvvup3adu/Zyc9HpM1alWfzbd06Q23NJi9uWQnkzXY7Ldm99LOf0TmtFpciV4I17gVrXHrax6VAJmuvcPmqFRQk7XS4TNms8VCbJ0UsI0lxQC7+ICR0ZxciFxTsQmSCgl2ITFCwC5EJCnYhMkHBLkQmjFV6cy/R66Qlj85yut8VABRlOqvMg0yiSiA1DYIstVde4dLQ4pW0j406X8ZqnWfzVYPeZmWfZ/RZ0PesICrUzh3b6ZxKhQs2rZUgay+Qyk6fThd0JLVDhwRSHoKCk62gN9s8ec2WL3JZi2VZAkAvuHb6JX/N5uZ5vZc+KRIaZXWGGhtBd3YhMkHBLkQmKNiFyAQFuxCZoGAXIhPGuhtfMaBZTb+/dFd4YkKTlYyrBLXkgt3bapCUMDPDK+A2SO26LdNTdE4RtHiaaPKd+n6ftxl65ehL1DZ/6VJy/EpQxnsAvutbb/A1rhpPNmrU08/bKlxJaAWJJOcvpZNFVptXJdv/27byNlTdIOmm1ea78b0uT8jxYGedJ23x9TW79vu07uxCZIKCXYhMULALkQkKdiEyQcEuRCYo2IXIhFWlNzO7DcBfANgLoARwyN2/YGafBvAHAC6MHvpJd//uKkeDWVrKGQTJHWxOOeBzOguB1DTgstZElS+J1dL12KK2Re2516gtqmdWBtJbhdRVA4BaNe1jpRq012ry93yrRXXmuJy0eDmdnNJpB4k1Hb4ekQw1EWTX9FbSUlnX+evcagfSW9CGyoPXJeorNiAymoPX/6uT9mCVQJJbi87eB/DH7v6smW0B8IyZfX9k+7y7/4c1HEMIscmspdfbWQBnRz8vmtlRAPs32jEhxI3lmv5mN7PbAbwLwNOjoY+b2REze9zM+FeShBCbzpqD3cymAXwTwCfcfQHAFwHcBeBBDO/8nyXzDprZYTM73G3zv3eEEBvLmoLdzGoYBvpX3P1bAODu59x94O4lgC8BeCg1190PufsBdz9Qb07eKL+FENfIqsFuZgbgywCOuvvnrhrfd9XDPgTghRvvnhDiRrGW3fj3AvgogOfN7LnR2CcBfMTMHsSwGtZxAH+42oEGgz4Wr8wlbe1FXhPsfONkcjxqtzPoc1s3qCMWZi6R8UqF6yr1Gj9elWQAAkBR8GyzgmTfAYCROm79AX/O7Ra3dQI5bHGeS1TeT6/J9BYuAUaykQfSbGeJ/3nYJzLroMulzUheK4M6c/DA/yDbj+ly1UgGLrn/jLXsxv+IeLOKpi6EuJnQN+iEyAQFuxCZoGAXIhMU7EJkgoJdiEwYa8HJXqeN10+8nDY6lzQGJLstqrlXbfAWPlExyqCGIs0om5riBScj2PMCgF6fS3b9wNZpp20ePDGzQG4Msu8adf4lqV23pNMnlhd4m6/FJZ4R1wukMg9aZVWIrNXqBHJd8JzLILONyZ5ALCuypL0iiIkWab1VlkFLNGoRQvxKoWAXIhMU7EJkgoJdiExQsAuRCQp2ITJhrNKboUS1TEsekQxlpE+WBdlm/QrP5KqU/GlHGWwdkknX7wdZUh74GBbZpKYwG6pRT8uDbnxOHTyLrt9LF44EgOZk4MdEutfblTn+uizPc+mtFjznIlisbid9vn4ga5U0vxGw4FxFIOkikMQatfT6Ly3wTNDlVtoWZeXpzi5EJijYhcgEBbsQmaBgFyITFOxCZIKCXYhMGKv0BneUpABg1CfLiUwS1v7rcqljUOETI2nFSHpSOeDSVYX0hwOAZoPbqkHByUrQ22xQptexYsF6BMU5yy6X3rodbmuTPnbLi1xeK0uefcf6/Q3PFRS+JOsR1IYMiTImo4y4evB6eje9/pfnztE5vW5a7o3WUHd2ITJBwS5EJijYhcgEBbsQmaBgFyITVt2NN7MmgB8CaIwe/5fu/ikzuwPA1wBsB/AsgI+6O89ywHC3cqWbfkiU3MHqp1WKIDmiymvQVcKd7qDtErFFx/MgsSaaFyU0lB7srJOEiy6pTQcARbDj3ltY4OcK1mqqvSM5Xgb18yqBEtJZ4T6yHfehMT08iGrJkbp1QFwLrxYoL9Fzu3TxQvpcHa5cRIlS1Ic1PKYD4P3u/k4M2zM/bGbvAfCnAD7v7ncDuAzgY9d+eiHEuFg12H3IG28xtdE/B/B+AH85Gn8CwAc3xEMhxA1hrf3Zi1EH1/MAvg/gGIAr7v7GZ7LTANK1g4UQNwVrCnZ3H7j7gwBuBfAQgHekHpaaa2YHzeywmR0uSREKIcTGc0278e5+BcD/BPAeALNm/7/8ya0AzpA5h9z9gLsfiDakhBAby6rBbma7zGx29PMEgH8M4CiAHwD4F6OHPQrgOxvlpBBi/awlEWYfgCdsmIlQAfANd/8rM3sRwNfM7N8B+FsAX17tQFapoN5oJm2RjMbqwlWC5Iiw3RG1ABZIMmBJFc7lJAStlfodfq7oT54ekS8BoNtL21baPFlkECSS9FvcNh3U0JuY3Zkc7/W4dMXqxQFx+6QI9lpHr3M/kPI8qE83VeVXVtT2amGe2YJ6d6Sm4AB8DVcNdnc/AuBdifFXMfz7XQjxS4C+QSdEJijYhcgEBbsQmaBgFyITFOxCZIJFtd9u+MnMLgA4Mfp1J4CLYzs5R368GfnxZn7Z/Ph77r4rZRhrsL/pxGaH3f3AppxcfsiPDP3Qx3ghMkHBLkQmbGawH9rEc1+N/Hgz8uPN/Mr4sWl/swshxos+xguRCZsS7Gb2sJn9zMxeMbPHNsOHkR/Hzex5M3vOzA6P8byPm9l5M3vhqrHtZvZ9M3t59P+2TfLj02b22mhNnjOzD4zBj9vM7AdmdtTMfmpm/2o0PtY1CfwY65qYWdPM/sbMfjLy49+Oxu8ws6dH6/F1M+MVLlO4+1j/ASgwLGt1J4A6gJ8AuG/cfox8OQ5g5yac97cAvBvAC1eN/XsAj41+fgzAn26SH58G8K/HvB77ALx79PMWAD8HcN+41yTwY6xrgmFu6/To5xqApzEsGPMNAB8ejf8ZgH95LcfdjDv7QwBecfdXfVh6+msAHtkEPzYNd/8hgEtvGX4Ew8KdwJgKeBI/xo67n3X3Z0c/L2JYHGU/xrwmgR9jxYfc8CKvmxHs+wGcuur3zSxW6QC+Z2bPmNnBTfLhDfa4+1lgeNEB2L2JvnzczI6MPuZv+J8TV2Nmt2NYP+FpbOKavMUPYMxrshFFXjcj2FPlNzZLEnivu78bwD8D8Edm9lub5MfNxBcB3IVhj4CzAD47rhOb2TSAbwL4hLvz7hTj92Psa+LrKPLK2IxgPw3gtqt+p8UqNxp3PzP6/zyAb2NzK++cM7N9ADD6//xmOOHu50YXWgngSxjTmphZDcMA+4q7f2s0PPY1SfmxWWsyOvc1F3llbEaw/xjA3aOdxTqADwN4ctxOmNmUmW1542cAvwPghXjWhvIkhoU7gU0s4PlGcI34EMawJmZmGNYwPOrun7vKNNY1YX6Me002rMjruHYY37Lb+AEMdzqPAfg3m+TDnRgqAT8B8NNx+gHgqxh+HOxh+EnnYwB2AHgKwMuj/7dvkh//GcDzAI5gGGz7xuDHb2L4kfQIgOdG/z4w7jUJ/BjrmgD4+xgWcT2C4RvLn1x1zf4NgFcA/FcAjWs5rr5BJ0Qm6Bt0QmSCgl2ITFCwC5EJCnYhMkHBLkQmKNiFyAQFuxCZoGAXIhP+H+DLh7f3sSxwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAdIklEQVR4nO2dbYxcZ5Xn/+feW9Uv7rYdtx3HcV7sxE7ICyGAJ0IKy7LM7iiDRgpIMyP4gPIBjdFo0C7SrLQRs1pYaT8wqwXEhxUrs0STWbG8DAERrdDOoAwMIO2GdILjJBhCJjHE8Wvit3bb3VV179kPVVmc8PxPd/ql2pPn/5Msdz+nnnvPfbpO3arnX+ccc3cIId78FGvtgBBiOCjYhcgEBbsQmaBgFyITFOxCZIKCXYhMqJYz2czuAfAFACWA/+7un4kePzk54VNTm5K2ohql8wqz5HhZlHROHUiKTV1Tmxmfxw5J3AMAFMZfTw3BxMCPJbHEw/kSJ9JrCy55AWMA95H9bZYqOVvwxw5t4bWR50gwpenNJcePnTiJs2fPJWcuOdjNrATwXwH8KwCHATxmZg+7+8/YnKmpTfiLf//vkrbJzTfRc41X7fScyQk6Z2a+obbzZ1+htrLk82pP21rBi85Ya4TaRstg+aNgD54E7EncNPy6oniumx61NcHEqkpfWxld8xKDPXqBLor0C3vdBC/40bkK/uLdbqefpwDQLvjNzECeIy1+rguvHEyOf+xf30/nLOdt/F0AnnP35929A+BrAO5dxvGEEKvIcoJ9O4AXL/n98GBMCHEZspxgT73b+a33U2a218ymzWx6Zub8Mk4nhFgOywn2wwCuveT3awAcef2D3H2fu+9x9z3RZ2whxOqynGB/DMBuM9tpZm0AHwLw8Mq4JYRYaZa8G+/uPTP7OIC/RV96e8DdnwnnoIBjLGnrVWlJDgDmq3XJ8fEiPQ4ARYvvts7O8Y8TXnPbSDu9T9up+Y51t2hR28WKv9ZWfBMf3W5adgEAK9PKwMULF/m5Kq4mVC3uf6fTpbaySK9JU/M5RbAe7Rbf6e4FUioRUFAYf+ozJQEArti4kdpGxyapLVBg4WVaTbARvvbNLHmXHKgFy9LZ3f27AL67nGMIIYaDvkEnRCYo2IXIBAW7EJmgYBciExTsQmTCsnbj3ygGh3laeukFiQmVpfWTns3TOaMTXE6auu5KaivOcblj8mJalpvvcD/qSZ4A0aznMs76kSCDqgkkLyK9dea4j70gSWZsNErg4LDEGws0qCI4YlHyeb0uX4/a33j2XTtI1hkbS0vHAFBESYzGfWxAknWY7wCcrmOwhtQihHhToWAXIhMU7EJkgoJdiExQsAuRCUPdjXeUqJFOFijBEx0aUiqqA77j3nNumwiyTNaPcz/qJx5Ljs+/zJNntr31LdRmJ7kf8+BJPpNBwsg5ohiMBCWkNjhXIGyKpyUXYSJMeld4fpzv7lcdrgqUXb7LfH6iQ23jZ8+mz3XtbXTOhSDZpenyhKJewdd4pOZrjCZtK0r+XKx65DkQlBjTnV2ITFCwC5EJCnYhMkHBLkQmKNiFyAQFuxCZMFTprU/69cWDziMlSPJMl0sTTPoBgF4gNc2BS01VnZbDbDNPrLlwjstC3ReepbbaggQabsJsi8hXQbLLSJdLgL1fcwkTPX5MQ9o2t447XwZdfCq+jJi7ij+NLxw7lRyftC10TrFhM7XVDde2ekE3oVaQyNOQQnkl6WYDABU5V9iKjJuEEG8mFOxCZIKCXYhMULALkQkKdiEyQcEuRCYsS3ozs0MAZgDUAHruvid6vDtQk1ZJLSLVAEBDXpNIaToAQMe5lFdXfOKG83yeX3lVcnx8y3X8XE066woALGi75FvS5wKAiy2ur7SOvZw+HqlNBwCzo7yuWrOVt+VqN/xeMUekvvHJcTqne55nlM0Ft6XWGJcOq9n0MVtTXC61iktePedy70Qge5XB87suiRQctA5DkNXJWAmd/V+4e/oZJoS4bNDbeCEyYbnB7gD+zsweN7O9K+GQEGJ1WO7b+Lvd/YiZXQnge2b2c3f/4aUPGLwI7AWAKzZNLfN0Qoilsqw7u7sfGfx/AsC3AdyVeMw+d9/j7nsmJtYv53RCiGWw5GA3s3VmNvnqzwB+D8DTK+WYEGJlWc7b+K0Avm39NJsKwP909/+94CySltPrBZJXk5YZvOQZSKz9EAC0C36u9i95Jtr8Ez9Kjvd+57fe0PwG47JQAy5Djcxy+edizQtcjh47kxwv20FBz0l+LgOXf+rgbza5KV20sfVSOgsNAGyWX1drK89GxOFXqKlany5wOnfyAJ1TjqfnAEBz863UNh+ssYGv1QgpHln1+PPbg8KSjCUHu7s/D+BtS50vhBgukt6EyAQFuxCZoGAXIhMU7EJkgoJdiEwYasFJM0NVpk9ZFdyVupeWhty4/lCCpyCtOzXLz3X4JWqbbBFp5cgxOqczsoHayiKoHHnsBDWtu5r3geuuJ4UefY7OGZuZp7b2qXPUNme811vvZHqt2nO8cmTvHM8QbL/C17HLLw0+dmNy/PQLL9I5I2Nc5pvcdj21FTx5EB4Uj2Sr3zUeEx1S+DKS5HRnFyITFOxCZIKCXYhMULALkQkKdiEyYai78YUZRtrpHWiP6m01ZCfT+WtV0fDd+NkWr991/nfeTm2TZbrE3sXzPIGjFyTroB3UEevy3dv2GN/2PV+nd7uD8mioSaIRAMwVfI0vtLitIie8QJQVALgYJMKMj3LlYn6E+9Fel1YupiavoHN6RDECgJmxIGTa/G891uE+9kirsqiVE92N51N0ZxciFxTsQmSCgl2ITFCwC5EJCnYhMkHBLkQmDFd6KwqsG0/LRvUIr9/VqS+kDRa0eAqkt4bIfwAwupUnXMzMpBNoTp7liTWRftKb4/63gvY+nTNkPQB0WdulUb6+57o8oWW8HWR3BHJpr05LbJ0LvMWTBddcXeQ+djtB0lOVlt4mt1/Dz+WB5lXw2oYW3DstUFnp2YKsFiftpCS9CSEU7ELkgoJdiExQsAuRCQp2ITJBwS5EJiwovZnZAwD+AMAJd799MLYJwNcB7ABwCMAfu/vphY8FVCRTamySt0I6f2EmOd4Kste6zrOrWsblkwq8Rlq3SNvKiktoZcV9bJHMJQDozHN5bTxoM1SSumWtdpAZNhJIaF1+bZ2LvHZdTaShSMlrArl0hKXRAWhHkh2R5TpRxiTxHQDGA22rmedrFaUd1sRYtnh4svKLUXbjYu7sfwXgnteN3Q/gEXffDeCRwe9CiMuYBYN90G/99d347gXw4ODnBwF8YIX9EkKsMEv9zL7V3Y8CwOD/K1fOJSHEarDqG3RmttfMps1s+lxQF1wIsbosNdiPm9k2ABj8TzsauPs+d9/j7nvWr+ffOxdCrC5LDfaHAdw3+Pk+AN9ZGXeEEKvFYqS3rwJ4L4DNZnYYwKcAfAbAN8zsowB+DeCPFnMyKwwjI0QaCgr5NZ7OUhtrjdA57aDdztkzaSkPAJpAzhvbOJUcL8cm6RwLWlQhkN5KUoQQAMrgNbogGVsjbS6veZBdFRU99JpLVD1P2+qgAKcHOVtWc1lrNGgd5iTdrFPwLDor+UVXrPgpgK5xW1kGGXGk4Gf0HCiC4zEWDHZ3/zAx/e4bPpsQYs3QN+iEyAQFuxCZoGAXIhMU7EJkgoJdiEwYasFJg6Mq0hJKaTyDarRKy0ZnTvBEu1Pnj1Lby8cPU9uGcd4D7I7b35YcXz/GM/bmA+mtExR6jP4wVVTYkGhlS5FqAIQVDHvBtRnJYCsCuTHK2SpKnunngUTVkAKcVeBHEUh5VnA/xoPed61IwiS+lCWXgWtyzcFS6M4uRC4o2IXIBAW7EJmgYBciExTsQmSCgl2ITBiq9AYYjMgrrUhmIHLSzAzPXnv55WPUdvoUl96ePfATavv5U48mx3fvupXO2bnrFmq7YvPSCvz0elyyKwry+h0VZayCrLEgI64MGphZkbY5kcIWsjVB/7UqKOrJvI+ui8l1QFzQsenxrLdeoDiybL86yMyb66Ql7CaSFLkLQog3Ewp2ITJBwS5EJijYhcgEBbsQmTDk3XhOGewIj42la9DdcvPNdM7uW6+mttlzfN4zP32C2n46/X+T4z/+h1/ROb945ilq23XLndS2++ZgF38TT9ZpkVpzVaB2xCwtcYXN69R8x7oOEoMiL+Zrfs9iG+tBpylYsFMfFeWLVoPV5AN4nTwrg7ZcZNc9Wifd2YXIBAW7EJmgYBciExTsQmSCgl2ITFCwC5EJi2n/9ACAPwBwwt1vH4x9GsCfADg5eNgn3f27C5/OURMthCVOAEBBWjkVQZueMqhZtnHqGmp79z/fQm27du1Mjv/4Bz+gc371wovUNruf1907G3S8veMOLtldd1362iLprdcN5LAoOaXhLZmYemUIJKio/VNQXK1VBTXoyP2sx91AEfW8CgjXii8xQKS3SAGsQ6EvzWLu7H8F4J7E+Ofd/c7Bv0UEuhBiLVkw2N39hwBODcEXIcQqspzP7B83swNm9oCZ8a90CSEuC5Ya7F8EcCOAOwEcBfBZ9kAz22tm02Y2ffYs/xwqhFhdlhTs7n7c3Wt3bwB8CcBdwWP3ufsed9+zYcOGpfophFgmSwp2M9t2ya8fBPD0yrgjhFgtFiO9fRXAewFsNrPDAD4F4L1mdif6STaHAHxscaczFEQCCuuZlWlJoxW0H6pZLTYAhXGJpACX7Hbvfmty3LtcBjl+7JvUdvrkS9TWzPH6eide+gW13bD7LcnxW2+7g87ZunUbtbVaI9TW63Bbp07LcnWkJwXyYFHxv2c3qLvmPdJuLKxbF0h5QS28yP8yeD6yZ2MTtcMqWEZcUKuPWga4+4cTw19eaJ4Q4vJC36ATIhMU7EJkgoJdiExQsAuRCQp2ITJh6AUnWUZRYfx1p0W8HAmyk5oiaGkUZWtRC9DppAsiXnNdOhsOAK67fge1nTzBW1TNd85R28Xjp6nt5VeOJsd//vMn6ZydO3ZR265dN1HblVdup7bJ9ekvUJnxIorz87zgZHeOp421R9MFSQEABWmtFKS9RUlvbkH6WlDF0oOCk0buuVWU1UnmsPZqgO7sQmSDgl2ITFCwC5EJCnYhMkHBLkQmKNiFyIShSm8GgOUFlUEhwoZIXrV16Jw6yGwLVL5QumAVAEdGePbXRJDD70ERxajAYlhq0NOy4rlTJ+iU/S8fp7aD+6ep7YotvEDR1qvShS+3Xc1lypH2emqbmuKZeRPrxqkNZB2boAJkHUqzQaZlcMwimMcSAZseP56T4paRf7qzC5EJCnYhMkHBLkQmKNiFyAQFuxCZMOREGKe7xcF3/oEqvcNYBskFBUmAAIAmyHSwoMZYeyRdn+7C7Hk659gxvtN95Ai3nR3jCSOjLDMIwPhYOilkYnyMzhmp+Lnqmu8IHznCa+g9e+j55Pjc3N/TOb2a13Cb2nw1td3+1lup7aZd1yXHt2zmbb7Wb9hMbSPjXDFwDxJygtZQLCcnqsvIdKgojHRnFyITFOxCZIKCXYhMULALkQkKdiEyQcEuRCYspv3TtQD+GsBV6Heq2efuXzCzTQC+DmAH+i2g/tjdeXE0ADDAirSUw77YDwDeSUteXQ+SGYLWUGWLy1BNIDWVJMlg/+M8WWT2NE9AmZrkCRyHj/B5Gzby5JrC0mtlzUU6B0EeSavFZblWxSduaKdtpc3SOadP8y6/L77wM2o7d/owte2fTstXrTaXya699gZqu3rb9dS2bVs6+QcAtl/F562bSCcU2Ri/F1vB/A/anlHLb+gB+HN3vwXAuwD8mZndCuB+AI+4+24Ajwx+F0JcpiwY7O5+1N2fGPw8A+AggO0A7gXw4OBhDwL4wGo5KYRYPm/oM7uZ7QDwdgCPAtjq7keB/gsCgCtX2jkhxMqx6GA3swkADwH4hLvzoua/PW+vmU2b2fTZM/wzmRBidVlUsFu/sv9DAL7i7t8aDB83s20D+zYAyR0ld9/n7nvcfU+0sSSEWF0WDHYzM/T7sR90989dYnoYwH2Dn+8D8J2Vd08IsVIsJuvtbgAfAfCUme0fjH0SwGcAfMPMPgrg1wD+aKEDuTfodOeStnlSZw4Aijot/xQW1Qrj+T+9ms8LSr/h/Ew6u60zx2vh3bTrLdT2zjveSW2PH3ia2h6d5lLfudm0xHbmHPfxjtt4G6e73/1uaquCtksvHDqUHH/sJ/+HzrntltuobTKo5XfiWLrlFQAcP5lusdXp8vW4aiuvd3fDTi6h9YKWUrPn+EdYJ0XoWiWXNufm0/43rKAdFhHs7v5j8My5311ovhDi8kDfoBMiExTsQmSCgl2ITFCwC5EJCnYhMmHIBScDAsmrIYUlg7qRKAMNrQGX3qJjjpGijf/sPe/jxwteT6uSL/9Nd95Fbbe/k9tYZl60wJs3TVHbjTfeSG3VKG97tfOm25Pj1+y4ic4ZHeXZiBs3bqQ2Jl0BwCunXk6ON4FMtmXLVdS2fj33o6h4gciy4bZenZajewV/7vgSbtO6swuRCQp2ITJBwS5EJijYhcgEBbsQmaBgFyIThiq9NU2DCxfSWVnFWV4PoyJudpp5Oqc2Lq10A9mlCTLimjo9zwO9rtvl2XxlyeWYucCPbdfvpDZWZ9Nq7mPR8Nf853/Na4he7PDinEWRvrbJDdz3puES2qkzfB2rNi+KuW59+nxRT78z53hxziPHX6E2D/wfKblM2W6Teet4eM6fSfsYZd7pzi5EJijYhcgEBbsQmaBgFyITFOxCZMJQd+PPz8zgRz/6ftJ2pvsknTfRWpccr+d4K6FukBzRrfkufq/Hd31Bjtmt+Zxe0E6qClorXZzju/FEFAAAFCThoqr4bvCmK3gizARpTQQA3ZrfK2rSzquwqKUR3yG3YF5Jdv7789JrXLX4ehRB0hBTGYB4N77gJjSkjVk5GVzXxXR7sPkOf27rzi5EJijYhcgEBbsQmaBgFyITFOxCZIKCXYhMWFB6M7NrAfw1gKsANAD2ufsXzOzTAP4EwMnBQz/p7t+Nj1VgpErXGVtXpuU1ACjqtJvtEd4SaLTglxbJYUVU94vUd2uaqKYdl3Ec3I/Gg2NSC4LiZMF1NfyIpXMpB2WUNMTaK0XFBrmtDvTG+ejvSda/DOr/RYUIo+cHLf8HoHt+hk/rpdfx4vEgsaZKJ+T0uhfonMXo7D0Af+7uT5jZJIDHzex7A9vn3f2/LOIYQog1ZjG93o4CODr4ecbMDgLgnQCFEJclb+gzu5ntAPB2AI8Ohj5uZgfM7AEz41+1EkKsOYsOdjObAPAQgE+4+zkAXwRwI4A70b/zf5bM22tm02Y2zQpXCCFWn0UFu/W/YPwQgK+4+7cAwN2Pu3vt7g2ALwFIdi5w933uvsfd94yTJgtCiNVnwWC3/nbylwEcdPfPXTJ+acf6DwJ4euXdE0KsFIvZjb8bwEcAPGVm+wdjnwTwYTO7E33B4RCAjy14JG/gvbSUMzN7ik4br0bThwtUnDp4Het2uGTUrbl00eul2/TAeNabBxJap8tlLa95RlxcQ4/ZIgkwyNYK1tjB/Z+bS39kawKZLEgaC66r7wm3pNc/OFUol0ayZ9SGqghqEdbdtI+zGyfpnG3XTyTHu0Frs8Xsxv8Y6WsMNXUhxOWFvkEnRCYo2IXIBAW7EJmgYBciExTsQmTCUAtOdjpzePHFZ5K2546zLClgjBQHrIJssybKNnN+2XVwTCe2VpufK8qIiwpVBqYwK6uq0q/fFslrZSTLBRlxQeaYk4KTnQ7/O9dR66KS35eK6GlsxBasYR203grFt0B66xmXHHtXjCfHr37rLXTO+on0ucqgiKnu7EJkgoJdiExQsAuRCQp2ITJBwS5EJijYhciEoUpvgKFAOoOtClLYrEm7GfXWsqgwoAUST1AE0qp0762SyTsAPLiuwnkvLwQFMyOY5BVlCHqQUVaSawaAJuh71mnS2qFXbe5Hyf+edZQRF8hoIH3UiijFLpJtibQJAE1gm7h6K7Vde/vNyfHKeD+6M88eSPsQSJu6swuRCQp2ITJBwS5EJijYhcgEBbsQmaBgFyIThiq9uTu6pMhiPR8UeizSch07FgCY8+wfCySeqP+asUwu0qsLAJog2ykqvhgVqmwF8hVTw6JsviKoKhmpWr1A5gFZqyj7rjJ+74mkVAvW2EgBxio4Xh3olN1xvvbrb7qB2rbvuI7a5k8cT44/f3Cazhntnk+O10FhS93ZhcgEBbsQmaBgFyITFOxCZIKCXYhMWHA33sxGAfwQwMjg8d9090+Z2U4AXwOwCcATAD7i7sH2bH9nl+WMVEEdt6KV3sFtl0EiifPXsTA5JVgS1ibJo93gwI/RsbTKAAAbJnkH7CrYIu8RZaAJdvfLYDd+ZITvPvei7BRClLzUC2q/nT8/y48ZuFGT5Z+p+KRyiq/99bvfQm0br5iitpd+8Ry1vfLcC8nxKihEONpKX1gRPrcXZh7A+9z9bei3Z77HzN4F4C8BfN7ddwM4DeCjiziWEGKNWDDYvc+rol5r8M8BvA/ANwfjDwL4wKp4KIRYERbbn70cdHA9AeB7AP4RwBn/zTc/DgPYvjouCiFWgkUFu7vX7n4ngGsA3AUgVdA6+SHIzPaa2bSZTc/NR8XQhRCryRvajXf3MwB+AOBdADaa/f/ttmsAHCFz9rn7HnffMzrCv8IqhFhdFgx2M9tiZhsHP48B+JcADgL4PoA/HDzsPgDfWS0nhRDLZzGJMNsAPGhmJfovDt9w9/9lZj8D8DUz+08AfgrgywseyYGyR15fOrwOWu1zxBIkdwR14SoESTJBex8n9cwA7jss0IV6/GPNxfIsP2TQdon60gQJOfS6gLkulzctSFwBswWyoQWyXA2u6jJJFAC8Sq/VxJUb6Jwtu3dSW1Sj8NnpR6lt7vgr1FaShKgySNZpmAxMZywi2N39AIC3J8afR//zuxDinwD6Bp0QmaBgFyITFOxCZIKCXYhMULALkQnmUcrQSp/M7CSAXw1+3Qzg5aGdnCM/Xov8eC3/1Py43t23pAxDDfbXnNhs2t33rMnJ5Yf8yNAPvY0XIhMU7EJkwloG+741PPelyI/XIj9ey5vGjzX7zC6EGC56Gy9EJqxJsJvZPWb2CzN7zszuXwsfBn4cMrOnzGy/mfFeOyt/3gfM7ISZPX3J2CYz+56Z/XLwP696uLp+fNrMXhqsyX4ze/8Q/LjWzL5vZgfN7Bkz+zeD8aGuSeDHUNfEzEbN7Cdm9uTAj/84GN9pZo8O1uPrZsargaZw96H+A1CiX9bqBgBtAE8CuHXYfgx8OQRg8xqc9z0A3gHg6UvG/jOA+wc/3w/gL9fIj08D+LdDXo9tAN4x+HkSwLMAbh32mgR+DHVNABiAicHPLQCPol8w5hsAPjQY/28A/vSNHHct7ux3AXjO3Z/3funprwG4dw38WDPc/YcATr1u+F70C3cCQyrgSfwYOu5+1N2fGPw8g35xlO0Y8poEfgwV77PiRV7XIti3A3jxkt/XslilA/g7M3vczPaukQ+vstXdjwL9Jx2AK9fQl4+b2YHB2/xV/zhxKWa2A/36CY9iDdfkdX4AQ16T1SjyuhbBnipVslaSwN3u/g4Avw/gz8zsPWvkx+XEFwHciH6PgKMAPjusE5vZBICHAHzC3c8N67yL8GPoa+LLKPLKWItgPwzg2kt+p8UqVxt3PzL4/wSAb2NtK+8cN7NtADD4/8RaOOHuxwdPtAbAlzCkNTGzFvoB9hV3/9ZgeOhrkvJjrdZkcO43XOSVsRbB/hiA3YOdxTaADwF4eNhOmNk6M5t89WcAvwfg6XjWqvIw+oU7gTUs4PlqcA34IIawJmZm6NcwPOjun7vENNQ1YX4Me01WrcjrsHYYX7fb+H70dzr/EcBfrJEPN6CvBDwJ4Jlh+gHgq+i/Heyi/07nowCmADwC4JeD/zetkR//A8BTAA6gH2zbhuDHu9F/S3oAwP7Bv/cPe00CP4a6JgDuQL+I6wH0X1j+wyXP2Z8AeA7A3wAYeSPH1TfohMgEfYNOiExQsAuRCQp2ITJBwS5EJijYhcgEBbsQmaBgFyITFOxCZML/A2dij9MdVUcdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAf9ElEQVR4nO2daYxc15Xf/6f2ruq9m72Q3eImkqIkU6JMy8rI9njsxFCMAWQjmYH9wdAHYzgIxkAMTD4IDhA7QD54gtiGPwQO6FgYTeB4ydiGhcDJWNDMWHYQLdRibZRIcW+y2c3el+rq2k4+VDGg5ft/3WJ3V8t+/x9AdPGeuu/duvXOe6/u/51zzN0hhPj9J7HdAxBCtAY5uxAxQc4uREyQswsRE+TsQsQEObsQMSG1kc5m9hCAbwJIAviv7v7VqPdnMinP5bLhbSX5eadSrQXba7Uq7VOvcUkxlU5SG5LcZmbBdg83N2wR46gUS3xfSb7RdC5NbQnWLWKQUeprMsW/l0w24vAhcxW1Mza/AFCr8n7VSsRxQNrbCznaB3W+r4WFJWpLZ6O+l4jju1whnXifZCJ8nJaWSyivloMTecvObmZJAP8ZwD8DMAbgeTN7wt3fYH1yuSyOffCOoC3d0Un3NTkzE2yfmZmlfUqL3JF6h7uoLdXTS22WDk++R52oFsgXCeDyS3SqkO7KUNvogZ3U1ka+0XqFb6/KfQU9/W3UNryvj9pSqfBAajU+H6l0+EIAAAszZWqbGp+ktnI9/N38wf3h4xAAfJVPyM+f/AW17dqzi9pyaX5yuXp5PNiezHXQPp2FsO25J5+nfTZyG38/gLfd/Zy7lwF8H8DDG9ieEGIL2Yiz7wJw+ab/jzXbhBDvQTbymz30u+C3fuyY2XEAxwEgm+O3kkKIrWUjV/YxAKM3/X8EwNV3vsndT7j7MXc/lklvaD1QCLEBNuLszwM4YGZ7zSwD4DMAnticYQkhNptbvtS6e9XMvgDg79CQ3h5z99cjOyUMSSJ55Pp7aLf20mqwfXp2mvbpG+IrmUP7h6ltthiW+QDAmBRCVp4BYKWyTG018FXfrg6uGOzY0U5tKQ//VFqY55+rnuByUnvEajyTRAGgvBL+bLUKX1XPFrj0Zs4l0coqX+FPZ/LB9r5urv4sL85RW3GhSG2TV/nx2JbhP2FTHj5+8p38GFgthec+Skbd0H21u/8MwM82sg0hRGvQE3RCxAQ5uxAxQc4uREyQswsRE+TsQsSElj7lYqkkUl1hySOT5dJEe1dYRmuf5YETgyM8oCXXXqA2Ky1SWyYTjmryiOikSpHLWgnjOkmhg0te5Yhov4SH+5WW+ecqVbjNqzuobXWBBxtNj4eDlJJkDgFgYDf/zMkMl+VWi1zOy+XDMmUuww/9aorLfCtFPvf5FWrCUB8PGsp2hI/HKvg4xi/+1vNrAIBahcuhurILERPk7ELEBDm7EDFBzi5ETJCzCxETWroan0yl0L0jvLq7OM9TTGXbw6u0HT08IKRriK/GL61G5HdLclUgmwkH8VTrLNMZUC9HrBRHxfdX+DZnr/EV/jZy+i4tRay4G1/Bzad4OqX2Ap//ejk8kApNkgckkvxwrEfkzkpEpAVLkyClZILvK5/jqsDQKE8JNjKym9qGd3FVo5wNj3/s/OVgOwAUV8L+Uq9HzBO1CCF+r5CzCxET5OxCxAQ5uxAxQc4uREyQswsRE1oqvSXMkCFBBh0RwQcDQ+F09AulKdrHUjzgorzA5bBMgsthmXp4jFElnlZKPD9apc5lqLmpeWrLt4fzqgFAKRuW7Lr7ummfjk4ury06l+WKVR75UcuTeYwI1Cgu8O1lMxHXpRSf/zwJesokubzWOcDn99A9vJKMRRzD4JsEqeSEfJ4fw0cfPBJsvz4Wrp4E6MouRGyQswsRE+TsQsQEObsQMUHOLkRMkLMLERM2JL2Z2QUAiwBqAKrufizq/bVaDYsLYUnJ6lySuXzpYrC9Pc1z0K3MLPBxVLjUlHQud6zMh7dZz91atJYFC+E2yGb5NvtGeVmg9u6wxJYnec4AABFRYzWuUqK8EC7LBQAJ8tmWprg0NDt5ndruOnaY2vojIhwZ2TQ/Bro7+Vx19PGyUSs1LrOWIyTM3kL4O+seiZBESU7BqAjAzdDZ/8jdueAthHhPoNt4IWLCRp3dAfzczF4ws+ObMSAhxNaw0dv4B939qpkNAHjSzN5096dvfkPzJHAcANpIxhkhxNazoSu7u19t/p0E8BMA9wfec8Ldj7n7sWwbX1ATQmwtt+zsZlYws44brwF8AsBrmzUwIcTmspHb+EEAPzGzG9v57+7+v6M61Op1LC2HI5sqEUkPL770arB9126e/K89osRTZyEiBIkkSgSAq1eJ3NHO71gsIgFgZwcfx757Rqltx35eSiiZCo/fIkpUXbvEZcrLb4xRW287j6S7633vC7Y/90ZYRgWA+ekitbV3cskrmeBRb6vl8HGV7+bJMtsy3FbI8+865xFlxSIiI/u6+4Ptr73xEu3z5uung+3FRT6Ht+zs7n4OwD232l8I0VokvQkRE+TsQsQEObsQMUHOLkRMkLMLERNamnCyXq9jeYVIbzUuUZVIRFx+J5egcuASSaXCo7Wsxs9/faTW28TENO1TnONSyP4HeG2wvUfDSTYBYNV5dJURGWpxnCewfOv/8McjluaXqa1wiEcI1hCe487BQdqnjciGAJCxiO8z4lmtjpHwU5vXSzz6rjPfQW3tOS6XJj1iIDUuLdcq4c997vQl2mfy3ESwvbKqWm9CxB45uxAxQc4uREyQswsRE+TsQsSE1pZ/SiTQRoI/lq7zFe2h0XBQyJ4D+2mfrrYeahs7e57arp25QG3DPeFV2rYKL1u0OszzxY3cMURtlubn4cRqRJkhkjPu3As8oKU4zVfc7ziyj9s+yPPCXbsc3l9Xmufdu+P9t1NbqpOv/Oe7uCqTLoTnsVTh6sTEXESWtQhVICLWCHXnrra4GD5+Jie5YlALV/mKRFd2IWKCnF2ImCBnFyImyNmFiAlydiFigpxdiJjQWuktlURbb1i+ysxwKSRJzkmFHA9YyHfxgIW9hw9R2/Xz16htbDwsDw52830d3Xknte0a5Dn06s7Pw9UEr8l05vWzwfbJi7y00tDeAWo7/EE+/o4+nnOttFIKtnd3cekqP8DlUqS49FY1Hvxx7e3JYPvIIR6Qs1KNyOOW5JqXJXi5pnKNS45T168G22enuATYlojIo0jQlV2ImCBnFyImyNmFiAlydiFigpxdiJggZxciJqwpvZnZYwD+GMCku9/dbOsF8AMAewBcAPCn7j671rYSZmgjEkqGV8dBtRKWVjyipI5xpQPZiGqyu49wqem5J58Jtl8b53LdPbfz7ZVSfPyZBS7x9NW4xLPo4ZJMdx04SPv0H+QyVLrA97VcXKK2HbvD40hHSG/LPLUeegv8UD370ji1Xbkclt4+dDhcngoA6haWDYHoaDNPcSmyUucltupE6nOSexEAPMHkRn5MrefK/tcAHnpH26MAnnL3AwCeav5fCPEeZk1nb9Zbf2dg7cMAHm++fhzApzZ5XEKITeZWf7MPuvs4ADT/8kewhBDvCbZ8gc7MjpvZSTM7uVrkv4WEEFvLrTr7hJkNA0Dzb3gVBIC7n3D3Y+5+LJvniz1CiK3lVp39CQCPNF8/AuCnmzMcIcRWsR7p7XsAPgqg38zGAHwZwFcB/NDMPg/gEoA/Wd/OEhhMhqN1LixHyB2kdE6lxKO/6hWukSRz3DZy4DZqGzsXLsczMR0hoe3idzPTEXLM4CyXXboqndTWSyIBD3zsbtqnZ2cvtc2vzFHbsvEkoaVaOIliZpzPfbXIbYsFHrWXNp6Ac//RcIRjro9HTM7M8M9cJKWaAKA9w8eRS/LIvBwp2ZWIkI+XlhaD7bU6n8M1nd3dP0tMH1+rrxDivYOeoBMiJsjZhYgJcnYhYoKcXYiYIGcXIia0NOFkvVbD4kxYblpe5PXGmLIyP8ulq6iIoYFRHuWVyPGIuHs+dCTYbqU9fHvJVWpbneZJNnemM9TWEREhaGROxs6d5n0yu6ity3hiw0SZy4o5EpGYmeUSayrD9zV1hQdV3p7nMtoqwmMsLfJxJCOSWy4WeRLISkTiy4Euflx5NTxXqQwfx87B8BPqF86Gk1cCurILERvk7ELEBDm7EDFBzi5ETJCzCxET5OxCxISWSm9IJJAgCQyHRrgcVi6H5atqhWcoXC1xyWt2nEdQ7dgzSm09PV3B9vZ3Ju26idrYFWrLgScoRI2Pv5zmEk//cHuwvR4RQVWbmKC26ysRUWrOpaGeQlgOy+d4xF46x6W8lHEpMp/iH252OpwUc+U8lz3Rz2WyfJYnzEzwIQIpbiwRmXL3wX20z77dYbn02jUuUerKLkRMkLMLERPk7ELEBDm7EDFBzi5ETGjpanwimUC2KxzskO3gq5y5RHiVNpflq8HpJP9oM1d5uaaBncPUVienxto8D6ook9VgAFiq8VV1y/IV4Vwnt/V0hAeZjQjwqS7zXH7lFFc8Ekm+Cj43Hc7jlkjzFfficrgMEgAgyZWLQl8/t3X3Bdu9xj/zmbfGqK17gJdIKGf4tXM5Qh1KEjdsy/JjeLUe3p47V090ZRciJsjZhYgJcnYhYoKcXYiYIGcXIibI2YWICesp//QYgD8GMOnudzfbvgLgzwDciCj5krv/bK1t1ep1LBN5pVLmEg9IeZxKNUIWKnNZLlPgMk5xIVxWBwCyneF+yW4e3PGBj/PCOc++8CK1PfPCy9R2+MABautPhMdYX+Y5/ro7wgE+ADA6ymWtYpHPVQZhWW51hUteiRwvnzR+nQfrFJb4d733wMGwocRzFNbAbRdmaA1TpLp4Lr+lZS69XXz7bLD9/Ok3aZ+dex8MtkfJoeu5sv81gIcC7d9w93ub/9Z0dCHE9rKms7v70wAigjiFEL8LbOQ3+xfM7BUze8zMejZtREKILeFWnf1bAPYDuBfAOICvsTea2XEzO2lmJ1eL/LFSIcTWckvO7u4T7l7zxoO43wZwf8R7T7j7MXc/ls3z56KFEFvLLTm7md0cLfJpAK9tznCEEFvFeqS37wH4KIB+MxsD8GUAHzWzewE4gAsA/nw9O/N6HeWVlaCtECGHVRCW2Go5HuFT6OJyWL4QEQFW4ZFo6Vp4f2PzfP1ydzuXtd5/5P3U9vILb1BbLUKm7GrrDrZnUvy8nk1yyWvsCo8QzOW55HVg395g+8oKl7VyBV7+aWiIRw9eG+ey3Jk3wvN48O57aZ/9vXdR28yzvPzTzAyP2quyGmYApubDEYJdPVz23Lt/f7A9k32B9lnT2d39s4Hm76zVTwjx3kJP0AkRE+TsQsQEObsQMUHOLkRMkLMLERNamnDSAKQsHMHW1s6lt4HecCmhcp1HUGUjkv9NXx6ntnw/f/J3cTwsQ2VzXIJ69vVT1Pbho/RZJHzqX3ya2i6dP09t5VJYluscCCdeBGhQYaNfB5eMEFGG6tylcNLGrgKXRCenwhIUAKTa+ANZQ7t4Esi56wvhfY1fpX1OL4T7AMDw4B5quzzOv5d6Jy//dNsdtwXbL7x+gfa5NhaWACtl/p3oyi5ETJCzCxET5OxCxAQ5uxAxQc4uREyQswsRE1pb6y2RQFtbOOKsWueJ8rp7w9E/6YikgaXlcHQdAEyOXaa2HnAdqlYJR14NDu2gfRaSXAp55kUeofTJj32C2uol/tnOn3072J5r55JXVFKRkUH+2TJtXJbLJMJSU1ue1/TLFPjcj8/MUptHJKosdIYj6YqrPIqussqj155++Qy1XYhIwFno4fJsV2/YJ0YPjdA+/YNhuTGV5vvRlV2ImCBnFyImyNmFiAlydiFigpxdiJjQ2tX4ZBL5rnBOtprz806arOyOXzhN+5RyfHW/HhEkM0ECOABg5LahYHt5ia9m90cEabz+f3mJp8LTv6S2o3fz8k8rxfAqcybH8+4N7uBjLEWsMK8u8ZJG/QNhBaWe4HN/8QrPJVctR0TrVHguwlUP9yut8jx++SwPWrlynZd/SvTwYKPZ6/PUVp0J2+77cLjEEwAMD4S/s3RWq/FCxB45uxAxQc4uREyQswsRE+TsQsQEObsQMWE95Z9GAfwNgCEAdQAn3P2bZtYL4AcA9qBRAupP3Z1HKwCwRIKW+Fle5UEt50+Fgw+WZ6/TPvkRLieVI9KqLRZ5EMTuTDgP2vnLPLBmYXaZ2kbed5Da/tff/4raihWeI+2+w+8LtldWeEBOZHBKhh8is3M8Z1x5NZwfMJXmuQZTaV7+KZ+PKBuV5mNkvRaWeCBMOaK81uie26ltIcHHMZ/gUnAPCTayNv69XFuZDrZX6xvLQVcF8JfufhjAAwD+wszuBPAogKfc/QCAp5r/F0K8R1nT2d193N1fbL5eBHAKwC4ADwN4vPm2xwF8aqsGKYTYOO/qN7uZ7QFwFMCzAAbdfRxonBAA8PtmIcS2s25nN7N2AD8C8EV35z8af7vfcTM7aWYnV5b472EhxNayLmc3szQajv5dd/9xs3nCzIab9mEAwYeG3f2Eux9z92Nt7XwBRgixtazp7GZmaNRjP+XuX7/J9ASAR5qvHwHw080fnhBis1hP1NuDAD4H4FUzuxGm9SUAXwXwQzP7PIBLAP5krQ0lzJBNhSOKJsd4tNnF0+Hotrs/cBftk0pxfW1xiss4Xd08V9vKSji6rbeXl4y6ePEitQ0d2k1t++67k9rOXLxEbXtvC5cS2jfC97WyzOXBWo1HlA0P7qS2q1fC32eV5PEDgHSC76tW43LY3BT/VZkhsqKXeekwr/LjI8PVMCzPz1Dbrt2j1Lbn7rCcd2WOHztFIm3W6nwO13R2d/8VGmXaQnx8rf5CiPcGeoJOiJggZxciJsjZhYgJcnYhYoKcXYiY0NKEk7VaDYvz4QSG81M8YK69LZxEz+pcjslkuATR1xOOXgOA8Sn+lF+xEpY79uwPy10A0D3AZbmzZ8KlmgDg8O791JZK8eSRZQ8ngVwucXmts8ATLBYjkmmuRJShyhXCEub1xXC0FgAU5/kx0N0RTlQKALk0jyjLkiPcIh7wKtb4XOUW+PHRnePz2D08SG3XV8NJLJeqPNmnI7yviLScurILERfk7ELEBDm7EDFBzi5ETJCzCxET5OxCxISWSm/1Wg1Li+G6VoU8ly0efP8fBdsPHd5L+4xNn+O2+XFqW3mby0ksOmyxzJP87Sj0UtuUT1Hbm6fepLaP3HmE2voKHcH25SUekVVIdFObO/9sSxGRYzUPy2ERpd5QyLdTWz7Ja5itEkm0QTj6MUlTUQJLaR5F15bg4ta+nTwKcCrFJcy5+bAcmY6Q8njtOz4+XdmFiAlydiFigpxdiJggZxciJsjZhYgJLV2NT6ZT6B3uC9qGDx6i/Y7eEc6f1t0XXnkGgK5evpKZ4VWjkOrgQRVT18Ir0+48r9rFi9eorTvP892ldvDAiYki39+uQri8klX5Km2txFezy5VwYA0AVJwHFCWT4UMrU+XzW4ooAbZzeIjapmbDgSQAsEjmam6Fr7iXanyMZS7WYLzMy4B5L/8+jZSbyuZ5qSzLhI/FRn7YMLqyCxET5OxCxAQ5uxAxQc4uREyQswsRE+TsQsSENaU3MxsF8DcAhgDUAZxw92+a2VcA/BmAG0LWl9z9Z1Hbcq+jREooXV2+QvtVqmH56rY9PBBmZLCf2g7uPEhtTDICgFwmHLCwusrz3ZUWeQDE/DyXvI4c4lJkW54HhcxNhsfYn+Qy2ZXrPCDnygzPC2dJnsftwFBYakrn3r1cBwCrEYErAJ+PhaVwHrdanQf49GV3UNsb8zxv4BvXufS2bzeXifPZsExcKfFj5/Kl8L7KpCwUsD6dvQrgL939RTPrAPCCmT3ZtH3D3f/TOrYhhNhm1lPrbRzAePP1opmdArBrqwcmhNhc3tVvdjPbA+AogGebTV8ws1fM7DEz4zmThRDbzrqd3czaAfwIwBfdfQHAtwDsB3AvGlf+r5F+x83spJmdXFmKeNZQCLGlrMvZzSyNhqN/191/DADuPuHuNXevA/g2gPtDfd39hLsfc/djbe28uIEQYmtZ09mt8WT9dwCccvev39Q+fNPbPg3gtc0fnhBis1jPavyDAD4H4FUze7nZ9iUAnzWze9FIenUBwJ+vtaFqpYrp8XDIWaXCpZXX3wzLDHsmuFz3Bw98gNp2dHMZ5LY+vvaYIAnUxuZ41NXoHVzGuX5ljtrePvsctXV38wgwFke3WORRb5cu8Xl86+IYtQ1180iuHemwHDa4g0ui6QKX5S5fu0RthQg5r7M3nF9vcZHfZU4Xudw4vcxLMi2Q0mYAAOPX1ZVq+NifOHuG9snVw9+n1bgfrWc1/lcAQnFzkZq6EOK9hZ6gEyImyNmFiAlydiFigpxdiJggZxciJrS2/FPdsbwSjsrpauPJ9c5cDMt1F89P0D5LC0Vqu/+fHKa2nh5ermmoP5z4spDron0uz12gtvouHjW2nAuXmgKAhWUeXVUlMtRSjZ/XSzu4FJlKjlDbzDxPfIm2cOLDakS0WXGRf+aefi5hLkfIYXOL4cSSjefEwlyNiAJ85ewFaus/wqMwsxHX1cunw99nez5L+2Q87EeJhBJOChF75OxCxAQ5uxAxQc4uREyQswsRE+TsQsSElkpviUQCeRLZ5BE1xSxcCgsTE1wieeqnv6S2zk5+jjt45AC15dNhiWq0Y4D2ySZ4Msp6jUebJbgyhEyZR7DVV8OTVU7zRISDfXz8A7t5hN3yNJe8Zkj9OKtxuc4TEUkli3xCCjkuUc0iPP/nrpyjfd66wJNKWhuPsBvYxWXKV37xLLX94bFjwfYPfPgB2ueXf/9ksD2VTNI+urILERPk7ELEBDm7EDFBzi5ETJCzCxET5OxCxISWSm+WANL58PmlWuXROtnecHTY7h4uC42d4hFxv3ry19SW7+KRaG2FsMTT3sbPmYNd+6ktne+jtktTZ6ltoUi0SACltnBU2QzCkYMAsFjhttIkT4rZtsKTNpY9HD24nONSZDrTTm2ZKpcbr81zCXZsfj7YPpvm26t1cHltiCSwBICp8xepLVnmsuLo7eGIz2QqXLcPALo7wpGWyYSkNyFij5xdiJggZxciJsjZhYgJcnYhYsKaq/FmlgPwNIBs8/1/6+5fNrO9AL4PoBfAiwA+504SY/1/6vB6ODfc3DTPGXf1Sni1+M4P7qF9yst8tXV+mgdj/OPfPU9tVRKoUT7I86rtLHNbfycvhXRo6C5qm13kK+STK+GV6aTz1eB8kisQpQzPr3f65VPUdnUyrIbsHOHqxMxZHoBSXeEVgLmOA+QGw6rA7rsO0T49t41SW3GFB/8k09ydeoZ4Dj20hb+buYicfLPzYX+p1bjasZ4r+yqAj7n7PWiUZ37IzB4A8FcAvuHuBwDMAvj8OrYlhNgm1nR2b3DjUphu/nMAHwPwt832xwF8aktGKITYFNZbnz3ZrOA6CeBJAGcBzLn7jXvUMQC8/KkQYttZl7O7e83d7wUwAuB+AKHE68EfyWZ23MxOmtnJ0jJPUCGE2Fre1Wq8u88B+EcADwDoNrMbKxIjAK6SPifc/Zi7H8uRx02FEFvPms5uZjvMrLv5ug3APwVwCsA/APiXzbc9AuCnWzVIIcTGWU8gzDCAx80sicbJ4Yfu/j/N7A0A3zez/wDgJQDfWWtD1UoVc5OzQdupk2dov9Vi+PY/meUBC/2jPdRWKXKF8MppHnzwDF4Otqfb+DQu7gh/XgAYn+Gy1s7BfdTW3cFlnEw6fP5uM35XNdDGA3L693BZbndXJ7X94pmwhHlhhQcoXV8a4+Po5kFPu24Ll+UCgJGRncH20REur12fmqG2ZS9RmzuXezu7eFmxVQ9LbF7ngUaDI+FjOJXJ0D5rOru7vwLgaKD9HBq/34UQvwPoCTohYoKcXYiYIGcXIibI2YWICXJ2IWKCRckFm74zs+sAbiTq6gfAk4e1Do3jN9E4fpPftXHsdvegNttSZ/+NHZuddPdwkSuNQ+PQODZ9HLqNFyImyNmFiAnb6ewntnHfN6Nx/CYax2/yezOObfvNLoRoLbqNFyImbIuzm9lDZvaWmb1tZo9uxxia47hgZq+a2ctmdrKF+33MzCbN7LWb2nrN7EkzO9P8y8P2tnYcXzGzK805ednMPtmCcYya2T+Y2Skze93M/nWzvaVzEjGOls6JmeXM7Dkz+3VzHP++2b7XzJ5tzscPzIyHuIVw95b+A5BEI63VPgAZAL8GcGerx9EcywUA/duw348AuA/Aaze1/UcAjzZfPwrgr7ZpHF8B8G9aPB/DAO5rvu4AcBrAna2ek4hxtHRO0EiY2958nQbwLBoJY34I4DPN9v8C4F+9m+1ux5X9fgBvu/s5b6Se/j6Ah7dhHNuGuz8N4J1B0w+jkbgTaFECTzKOluPu4+7+YvP1IhrJUXahxXMSMY6W4g02Pcnrdjj7LgCXb/r/diardAA/N7MXzOz4No3hBoPuPg40DjoAA9s4li+Y2SvN2/wt/zlxM2a2B438Cc9iG+fkHeMAWjwnW5HkdTucPZTTf7skgQfd/T4A/xzAX5jZR7ZpHO8lvgVgPxo1AsYBfK1VOzazdgA/AvBFd19o1X7XMY6Wz4lvIMkrYzucfQzAzTmBaLLKrcbdrzb/TgL4CbY3886EmQ0DQPPv5HYMwt0nmgdaHcC30aI5MbM0Gg72XXf/cbO55XMSGsd2zUlz3+86yStjO5z9eQAHmiuLGQCfAfBEqwdhZgUz67jxGsAnALwW3WtLeQKNxJ3ANibwvOFcTT6NFsyJmRkaOQxPufvXbzK1dE7YOFo9J1uW5LVVK4zvWG38JBornWcB/NttGsM+NJSAXwN4vZXjAPA9NG4HK2jc6XweQB+ApwCcaf7t3aZx/DcArwJ4BQ1nG27BOD6Exi3pKwBebv77ZKvnJGIcLZ0TAEfQSOL6Chonln930zH7HIC3AfwPANl3s109QSdETNATdELEBDm7EDFBzi5ETJCzCxET5OxCxAQ5uxAxQc4uREyQswsRE/4fTNsmeN/1lIEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    plt.imshow(x_adv[i].cpu().detach().permute(1, 2, 0).numpy())\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## RotNet\n",
    "\n",
    "Train small network on top of pre-trained rotation network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = {'num_classes': 4, 'num_stages': 4}\n",
    "net = NetworkInNetwork(opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.load_state_dict(torch.load('saved_models/model_net_epoch200')['network'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = net._feature_blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "net_wrap = NINWrapper(net, block=2)\n",
    "net_wrap = net_wrap.cuda().eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# net_wrap.fc = nn.Sequential(\n",
    "#     nn.Linear(12288, 200),\n",
    "#     nn.ReLU(inplace=True),\n",
    "#     nn.Linear(200, 200),\n",
    "#     nn.ReLU(inplace=True),\n",
    "#     nn.Linear(200, 128))\n",
    "net_wrap.fc = nn.Sequential(\n",
    "    nn.Linear(12288, 2000),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(2000, 400),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(400, 128))\n",
    "net_wrap = net_wrap.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_wrap.load_state_dict(torch.load('saved_models/transfer_cifar10_exp11.h5'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = PreActResNet(PreActBlock, [2, 2, 2, 2], num_classes=4)\n",
    "net.load_state_dict(torch.load('saved_models/rot_cifar10_exp0.h5'))\n",
    "net_wrap = ResNetWrapper(net, block=3, dim=16384)\n",
    "for param in net_wrap.parameters():\n",
    "    param.requires_grad = False\n",
    "# net_wrap.fc = nn.Sequential(\n",
    "#     nn.Linear(16384, 2000),\n",
    "#     nn.ReLU(inplace=True),\n",
    "#     nn.Linear(2000, 400),\n",
    "#     nn.ReLU(inplace=True),\n",
    "#     nn.Linear(400, 128),\n",
    "# )\n",
    "net_wrap.fc = nn.Sequential(\n",
    "    nn.BatchNorm1d(16384),\n",
    "    nn.Linear(16384, 2000),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.BatchNorm1d(2000),\n",
    "    nn.Linear(2000, 400),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.BatchNorm1d(400),\n",
    "    nn.Linear(400, 128),\n",
    ")\n",
    "net_wrap.load_state_dict(torch.load('saved_models/transfer_cifar10_exp18.h5'))\n",
    "net_wrap = net_wrap.to('cuda').eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "layers = ['fc']\n",
    "\n",
    "dknn = DKNNL2(net_wrap, x_train, y_train, x_valid, y_valid, layers, \n",
    "              k=1, num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7015\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    y_pred = dknn.classify(x_test)\n",
    "    print((y_pred.argmax(1) == y_test.numpy()).sum() / y_test.size(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0011\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    num = 0\n",
    "    for i in range(x_test.size(0) // 100):\n",
    "        begin = i * 100\n",
    "        end = (i + 1) * 100\n",
    "        y_pred = net_wrap(x_test[begin:end].to('cuda'))\n",
    "        num += (y_pred.argmax(1).cpu() == y_test[begin:end]).sum().numpy()\n",
    "    print(num / y_test.size(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_adv = pickle.load(open('x_adv/x_adv_adv_cifar10_exp0.h5.p', 'rb'))\n",
    "# x_adv = pickle.load(open('x_adv/x_ba_cifar10_adv2_0.2_0.001.p', 'rb'))\n",
    "# x_adv = pickle.load(open('x_adv/x_adv_cifar10_resnet_exp2.h5.p', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "torchvision.utils.save_image(x_adv[:10], 'cifar10_cw_adv.png', nrow=10, padding=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "torchvision.utils.save_image(x_test[:10], 'cifar10_test.png', nrow=10, padding=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff = x_adv[:10].cpu() - x_test[:10]\n",
    "torchvision.utils.save_image(diff, 'diff_cw_adv.png', nrow=10, padding=0, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1601, 0.1639, 0.0439, 0.0925, 0.2344, 0.1875, 0.1263, 0.0629, 0.1313,\n",
       "        0.0308, 0.1699, 0.2601, 0.1186, 0.2549, 0.2104, 0.0949, 0.0168, 0.1908,\n",
       "        0.2373, 0.2276, 0.0749, 0.1144, 0.0599, 0.3487, 0.0113, 0.1286, 0.0399,\n",
       "        0.1052, 0.1184, 0.1812, 0.1033, 0.1054, 0.1015, 0.0491, 0.2473, 0.1405,\n",
       "        0.1351, 0.3051, 0.2656, 0.1246, 0.1826, 0.1177, 0.1306, 0.2810, 0.2485,\n",
       "        0.0251, 0.0446, 0.2726, 0.0799, 0.2484, 0.1870, 0.0150, 0.0376, 0.1555,\n",
       "        0.1845, 0.0784, 0.0302, 0.0403, 0.0080, 0.1847, 0.1957, 0.1100, 0.0935,\n",
       "        0.1443, 0.0339, 0.1736, 0.0629, 0.1216, 0.0863, 0.1177, 0.0650, 0.1191,\n",
       "        0.1149, 0.3842, 0.2538, 0.1573, 0.1572, 0.1589, 0.0625, 0.2314, 0.1450,\n",
       "        0.0575, 0.0426, 0.0426, 0.0458, 0.2055, 0.2434, 0.1804, 0.0808, 0.2458,\n",
       "        0.2491, 0.0758, 0.0949, 0.1004, 0.0661, 0.2742, 0.2168, 0.0666, 0.0765,\n",
       "        0.2380])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(x_adv - x_test[ind])[:100].view(100, -1).norm(2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_adv.cpu() - x_test[ind][:100]).view(100, -1).norm(2, 1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}