06_example_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example Experiment\n",
    "> Experiment using Repeated MNIST and BatchBALD vs BALD vs random sampling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook ties everything together and runs an AL loop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Appended /home/blackhc/PycharmProjects/blackhc.batchbald/src to paths\n",
      "Switched to directory /home/blackhc/PycharmProjects/blackhc.batchbald\n",
      "%load_ext autoreload\n",
      "%autoreload 2\n"
     ]
    }
   ],
   "source": [
    "# experiment\n",
    "\n",
    "import blackhc.project.script\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# experiment\n",
    "\n",
    "import math\n",
    "\n",
    "import torch\n",
    "from torch import nn as nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "from batchbald_redux import (\n",
    "    active_learning,\n",
    "    batchbald,\n",
    "    consistent_mc_dropout,\n",
    "    joint_entropy,\n",
    "    repeated_mnist,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's define our Bayesian CNN model that we will use to train MNIST."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# experiment\n",
    "\n",
    "\n",
    "class BayesianCNN(consistent_mc_dropout.BayesianModule):\n",
    "    def __init__(self, num_classes=10):\n",
    "        super().__init__()\n",
    "\n",
    "        self.conv1 = nn.Conv2d(1, 32, kernel_size=5)\n",
    "        self.conv1_drop = consistent_mc_dropout.ConsistentMCDropout2d()\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=5)\n",
    "        self.conv2_drop = consistent_mc_dropout.ConsistentMCDropout2d()\n",
    "        self.fc1 = nn.Linear(1024, 128)\n",
    "        self.fc1_drop = consistent_mc_dropout.ConsistentMCDropout()\n",
    "        self.fc2 = nn.Linear(128, num_classes)\n",
    "\n",
    "    def mc_forward_impl(self, input: torch.Tensor):\n",
    "        input = F.relu(F.max_pool2d(self.conv1_drop(self.conv1(input)), 2))\n",
    "        input = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(input)), 2))\n",
    "        input = input.view(-1, 1024)\n",
    "        input = F.relu(self.fc1_drop(self.fc1(input)))\n",
    "        input = self.fc2(input)\n",
    "        input = F.log_softmax(input, dim=1)\n",
    "\n",
    "        return input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Grab our dataset, we'll use Repeated-MNIST. We will acquire to samples for each class for our initial training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# experiment\n",
    "\n",
    "train_dataset, test_dataset = repeated_mnist.create_repeated_MNIST_dataset(num_repetitions=1, add_noise=False)\n",
    "\n",
    "num_initial_samples = 20\n",
    "num_classes = 10\n",
    "\n",
    "initial_samples = active_learning.get_balanced_sample_indices(\n",
    "    repeated_mnist.get_targets(train_dataset), num_classes=num_classes, n_per_digit=num_initial_samples / num_classes\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this example, we are going to take two shortcuts that will reduce the performance:\n",
    "* we discard most of the training set and only keep 20k samples; and\n",
    "* we don't implement early stopping or epoch-wise training.\n",
    "\n",
    "Instead, we always train by drawing 24576 many samples from the training set. This will overfit in the beginning and underfit later, but it still is sufficient to achieve 90% accuracy with 105 samples in the training set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# experiment\n",
    "max_training_samples = 150\n",
    "acquisition_batch_size = 5\n",
    "num_inference_samples = 100\n",
    "num_test_inference_samples = 5\n",
    "num_samples = 100000\n",
    "\n",
    "test_batch_size = 512\n",
    "batch_size = 64\n",
    "scoring_batch_size = 128\n",
    "training_iterations = 4096 * 6\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "\n",
    "print(f\"use_cuda: {use_cuda}\")\n",
    "\n",
    "device = \"cuda\" if use_cuda else \"cpu\"\n",
    "\n",
    "kwargs = {\"num_workers\": 1, \"pin_memory\": True} if use_cuda else {}\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size, shuffle=False, **kwargs)\n",
    "\n",
    "active_learning_data = active_learning.ActiveLearningData(train_dataset)\n",
    "\n",
    "# Split off the initial samples first.\n",
    "active_learning_data.acquire(initial_samples)\n",
    "\n",
    "# THIS REMOVES MOST OF THE POOL DATA. UNCOMMENT THIS TO TAKE ALL UNLABELLED DATA INTO ACCOUNT!\n",
    "active_learning_data.extract_dataset_from_pool(40000)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    active_learning_data.training_dataset,\n",
    "    sampler=active_learning.RandomFixedLengthSampler(active_learning_data.training_dataset, training_iterations),\n",
    "    batch_size=batch_size,\n",
    "    **kwargs,\n",
    ")\n",
    "\n",
    "pool_loader = torch.utils.data.DataLoader(\n",
    "    active_learning_data.pool_dataset, batch_size=scoring_batch_size, shuffle=False, **kwargs\n",
    ")\n",
    "\n",
    "# Run experiment\n",
    "test_accs = []\n",
    "test_loss = []\n",
    "added_indices = []\n",
    "\n",
    "pbar = tqdm(initial=len(active_learning_data.training_dataset), total=max_training_samples, desc=\"Training Set Size\")\n",
    "\n",
    "while True:\n",
    "    model = BayesianCNN(num_classes).to(device=device)\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    # Train\n",
    "    for data, target in tqdm(train_loader, desc=\"Training\", leave=False):\n",
    "        data = data.to(device=device)\n",
    "        target = target.to(device=device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        prediction = model(data, 1).squeeze(1)\n",
    "        loss = F.nll_loss(prediction, target)\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    # Test\n",
    "    loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in tqdm(test_loader, desc=\"Testing\", leave=False):\n",
    "            data = data.to(device=device)\n",
    "            target = target.to(device=device)\n",
    "\n",
    "            prediction = torch.logsumexp(model(data, num_test_inference_samples), dim=1) - math.log(\n",
    "                num_test_inference_samples\n",
    "            )\n",
    "            loss += F.nll_loss(prediction, target, reduction=\"sum\")\n",
    "\n",
    "            prediction = prediction.max(1)[1]\n",
    "            correct += prediction.eq(target.view_as(prediction)).sum().item()\n",
    "\n",
    "    loss /= len(test_loader.dataset)\n",
    "    test_loss.append(loss)\n",
    "\n",
    "    percentage_correct = 100.0 * correct / len(test_loader.dataset)\n",
    "    test_accs.append(percentage_correct)\n",
    "\n",
    "    print(\n",
    "        \"Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\".format(\n",
    "            loss, correct, len(test_loader.dataset), percentage_correct\n",
    "        )\n",
    "    )\n",
    "\n",
    "    if len(active_learning_data.training_dataset) >= max_training_samples:\n",
    "        break\n",
    "\n",
    "    # Acquire pool predictions\n",
    "    N = len(active_learning_data.pool_dataset)\n",
    "    logits_N_K_C = torch.empty((N, num_inference_samples, num_classes), dtype=torch.double, pin_memory=use_cuda)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "\n",
    "        for i, (data, _) in enumerate(tqdm(pool_loader, desc=\"Evaluating Acquisition Set\", leave=False)):\n",
    "            data = data.to(device=device)\n",
    "\n",
    "            lower = i * pool_loader.batch_size\n",
    "            upper = min(lower + pool_loader.batch_size, N)\n",
    "            logits_N_K_C[lower:upper].copy_(model(data, num_inference_samples).double(), non_blocking=True)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        candidate_batch = batchbald.get_batchbald_batch(\n",
    "            logits_N_K_C, acquisition_batch_size, num_samples, dtype=torch.double, device=device\n",
    "        )\n",
    "\n",
    "    targets = repeated_mnist.get_targets(active_learning_data.pool_dataset)\n",
    "    dataset_indices = active_learning_data.get_dataset_indices(candidate_batch.indices)\n",
    "\n",
    "    print(\"Dataset indices: \", dataset_indices)\n",
    "    print(\"Scores: \", candidate_batch.scores)\n",
    "    print(\"Labels: \", targets[candidate_batch.indices])\n",
    "\n",
    "    active_learning_data.acquire(candidate_batch.indices)\n",
    "    added_indices.append(dataset_indices)\n",
    "    pbar.update(len(dataset_indices))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[64.75,\n",
       " 68.19,\n",
       " 69.0,\n",
       " 65.95,\n",
       " 73.85,\n",
       " 73.78,\n",
       " 76.59,\n",
       " 79.4,\n",
       " 78.45,\n",
       " 79.99,\n",
       " 83.43,\n",
       " 88.24,\n",
       " 89.86,\n",
       " 87.31,\n",
       " 88.02,\n",
       " 88.99,\n",
       " 89.43,\n",
       " 90.39,\n",
       " 89.94,\n",
       " 88.93,\n",
       " 89.93,\n",
       " 92.08,\n",
       " 91.15,\n",
       " 91.77,\n",
       " 89.94,\n",
       " 91.11,\n",
       " 91.43]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hide\n",
    "# experiment\n",
    "test_accs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor(1.7354, device='cuda:0'),\n",
       " tensor(1.6224, device='cuda:0'),\n",
       " tensor(1.5370, device='cuda:0'),\n",
       " tensor(1.6557, device='cuda:0'),\n",
       " tensor(1.2134, device='cuda:0'),\n",
       " tensor(1.2532, device='cuda:0'),\n",
       " tensor(1.0284, device='cuda:0'),\n",
       " tensor(0.9561, device='cuda:0'),\n",
       " tensor(1.0075, device='cuda:0'),\n",
       " tensor(0.8437, device='cuda:0'),\n",
       " tensor(0.6996, device='cuda:0'),\n",
       " tensor(0.4628, device='cuda:0'),\n",
       " tensor(0.3793, device='cuda:0'),\n",
       " tensor(0.4368, device='cuda:0'),\n",
       " tensor(0.4417, device='cuda:0'),\n",
       " tensor(0.4158, device='cuda:0'),\n",
       " tensor(0.3878, device='cuda:0'),\n",
       " tensor(0.3673, device='cuda:0'),\n",
       " tensor(0.3998, device='cuda:0'),\n",
       " tensor(0.3837, device='cuda:0'),\n",
       " tensor(0.3500, device='cuda:0'),\n",
       " tensor(0.2903, device='cuda:0'),\n",
       " tensor(0.3312, device='cuda:0'),\n",
       " tensor(0.3256, device='cuda:0'),\n",
       " tensor(0.3435, device='cuda:0'),\n",
       " tensor(0.3339, device='cuda:0'),\n",
       " tensor(0.3222, device='cuda:0')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hide\n",
    "# experiment\n",
    "test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([42168, 23071, 56348, 40547, 47690]),\n",
       " array([56464, 47674, 19187, 51367, 19298]),\n",
       " array([45073, 36984, 44267, 52314, 29681]),\n",
       " array([  729, 28305, 28838, 22083,  5175]),\n",
       " array([58829, 37974, 20867, 50764, 34142]),\n",
       " array([24533, 33099, 31781, 28525, 39266]),\n",
       " array([12186,  7917, 36587, 23434, 57419]),\n",
       " array([ 2765, 28599, 13371, 31512, 52035]),\n",
       " array([31981, 43609, 28512, 54038, 28856]),\n",
       " array([57632, 25332, 13096, 56300, 55241]),\n",
       " array([ 1549, 15250, 54946, 10265, 10086]),\n",
       " array([52771, 16756, 19089,  1722, 35232]),\n",
       " array([ 6428, 59294,  3810, 55388,  8447]),\n",
       " array([34520, 27458, 13021, 47936,  3056]),\n",
       " array([53260, 36152, 45069, 41933, 17494]),\n",
       " array([23112, 41572, 42703, 24740, 22662]),\n",
       " array([43716, 57714, 24526, 43796, 43686]),\n",
       " array([42020, 13969, 15848, 55393, 52083]),\n",
       " array([58874, 10481, 26072, 50632, 41495]),\n",
       " array([31063,  9633, 57507, 27429, 24426]),\n",
       " array([37469, 30157, 52708, 26622, 34481]),\n",
       " array([ 7851, 47322, 18003, 39593, 16022]),\n",
       " array([ 7833, 51600, 54848, 24462, 36072]),\n",
       " array([22229, 57728, 29464, 16036,  4153]),\n",
       " array([43998, 20924, 36818, 59339,  6231]),\n",
       " array([55628, 44736,  2845, 23718, 13259])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hide\n",
    "# experiment\n",
    "added_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "test_accs = [\n",
    "    64.75,\n",
    "    68.19,\n",
    "    69.0,\n",
    "    65.95,\n",
    "    73.85,\n",
    "    73.78,\n",
    "    76.59,\n",
    "    79.4,\n",
    "    78.45,\n",
    "    79.99,\n",
    "    83.43,\n",
    "    88.24,\n",
    "    89.86,\n",
    "    87.31,\n",
    "    88.02,\n",
    "    88.99,\n",
    "    89.43,\n",
    "    90.39,\n",
    "    89.94,\n",
    "    88.93,\n",
    "    89.93,\n",
    "    92.08,\n",
    "    91.15,\n",
    "    91.77,\n",
    "    89.94,\n",
    "    91.11,\n",
    "    91.43,\n",
    "]\n",
    "max_training_samples = 150\n",
    "acquisition_batch_size = 5\n",
    "num_inference_samples = 100\n",
    "num_test_inference_samples = 5\n",
    "num_samples = 100000\n",
    "num_initial_samples = 20\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: matplotlib in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (3.3.3)\n",
      "Requirement already satisfied: numpy>=1.15 in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from matplotlib) (1.19.2)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from matplotlib) (0.10.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from matplotlib) (8.0.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from matplotlib) (1.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from matplotlib) (2.8.1)\n",
      "Requirement already satisfied: six in /home/blackhc/anaconda3/envs/active_learning/lib/python3.8/site-packages (from cycler>=0.10->matplotlib) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "# hide\n",
    "# experiment\n",
    "!pip install matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuh0lEQVR4nO3dd3xV9f3H8dcnCSGDDQFCAoQV9o4gqAwRkSWoWKnVaquCqK2jrVpt1aq1to7601osWqtV6wYrIA4QBAdKQEjCDJsMSFgJJGR/fn/cExsghBBy9+f5eNzHvffMz4XkfU++55zvV1QVY4wxwSPE2wUYY4zxLAt+Y4wJMhb8xhgTZCz4jTEmyFjwG2NMkAnzdgG10apVK01ISPB2GcYY41dWr169X1VjTpzuF8GfkJBAcnKyt8swxhi/IiK7qptuTT3GGBNkLPiNMSbIWPAbY0yQseA3xpggY8FvjDFBxoLfGGOCjAW/McYEGQt+Y4xfWbn9AJ+u34t1KV93fnEDlzHGAGzee4Tr//UdRaUVnJPQnAcn96ZPXFNvl+V37IjfGOMXCorLuOWN1TSOaMADk3qxPbeAyX/7krvfW0fOkaJ62UdRaTkbs/PrZVu+zI74jTE+T1W5f14qO/YX8MaN5zKsS0umJcXz3JJ0Xvl6Jx+l7uXW0V35+fkJNAwLPeNtp2Tk8U7yHj5cl8WRojIenNyLn53XyU2fxvss+I0xPu/tVXv4YG0WvxqbyLAuLQFoEtGA+yf24uqhHfnjwg38+eNNvPndbu6b0JNxvdsgIjVu88DRYj5Ym8W7yXvYtPcIDcNCmNA3lkOFJTy8YANtm0Qwvm+sJz6ex4k/nCBJSkpS66TNmOC0MTufqc9/xZBOLXj1Z0MICak+0Fek5/Lw/A2k5xxlWOeWPDC5Fz1jmxy3TFl5BSvS9/NO8h4Wb9xHabnSv30zfpQUz+T+7WgS0YCi0nKufnElaVn5/OfGoSQltPDExwTgWEk5G7LzSMvMJzUzj7TMPJ6Y1p++8XU7jyEiq1U16aTpFvzGGF91tLiMS5/7koKSMhb+8gJaNWpY4/Jl5RX857vdPP3ZFvKPlfLjIR24a2wi+UVlvJu8h/fXZLAvv5gW0eFcPjCOK5Pa071t45O2c7CghCtmf83BghLenzWcrq0b1ftnKyguY0N2PqkZroBPy8pja85RKpxIbhkdTp+4ptw5NpEB7ZvVaR8W/MYYv6Kq3P7WWhakZPHmTecytHPLWq97uLCEZxan89rKXTQIFYpKKwgRGNW9NT9KiufCHm0ID6v52pbdBwq5fPZXNAwLZd6tw2ndOOKsP8/cNZmsSM8lNTOP7fsLqIzfmMYN6RvXlD7tmtAnril945vStknEaZurTseC3xjjV974dhf3z0vjN+O6c+vornXaRvq+I7y0YgcdW0VxxaB42jQ5s/BOyTjM9Dkr6RwTzVszhtGoYd1Oi+4/Wsxv3l3H0s25tGnihHxc0x+ez7Su2rLgN8b4jbTMPC6f/TXDOrfkX9efc8p2fU9YuimHG/+dzHldW/HP65JoEHpmV8Ev35LLXe+sI7+olPsn9OSnwzqe9ZF8bZ0q+O06fmOMTzlSVMpt/1lDi6hwnv5Rf6+GPsDoHq3549Q+LN+Sy31zU2t9x3BJWQWPfbSRn778Hc2jGvDfW8/juuEJHgv9mtjlnMYYn6Gq3Ds3lT2HjvHWjHNpeZqTuZ4yfUgHsvKKeHZJOrHNIrlrbGKNy2/PPcrtb60lNTOPa87twO8m9iKiwZndX+BOFvzGBKHDhSVkHDpGz9gmhHr5iLqq11fuYmFKNvdc0oNzPHgZZW3ceVE3sg8f49kl6bRrGsH0IR1OWkZVeW91Bg9+uJ7wsBD+ce1gxvVu64Vqa+bW4BeR24GbAAFeVNVnRKQF8DaQAOwEfqSqh9xZhzHGpbxC+c+3u3jy0y3kHSulSUQYF3SLYURiK0YkxhDbNNJrtaVm5PHIgo2M7h7DzBGdvVbHqYgIj13el5wjxdz/QRptmkQwukfrH+bnF5Vy/7w05q/LYminFjwzfYBX/z1r4raTuyLSB3gLGAKUAB8Ds3B9ERxU1cdF5F6guareU9O27OSuMWdv1c6DPPjf9WzIzmd4l5ZcNjCO73YcZHl6LvvyiwHo1roRIxJjGJEYw9BOLTzWPJFfVMqkZ7+ktLyChb+8gBbR4R7Zb10UFJdx1Zxv2JZTwNszz6VffDNW7zrE7W99T3ZeEXeNTeTmkV184i+pU53cdecRf09gpaoWOgV8AVwGTAFGOcu8CiwDagz+szJq1MnTfvQjuOUWKCyECRNOnn/99a7H/v0wbdrJ82fNgquugj174NprT57/q1/B5MmweTPMnHny/N/9Di66CNauhTvuOHn+Y4/B8OHw9ddw330nz3/mGRgwABYvhkcfPXn+P/4B3bvD/Pnw1FMnz3/tNWjfHt5+G2bPPnn+e+9Bq1bwyiuux4k++giiouDvf4d33jl5/rJlrucnn4QFC46fFxkJixa5Xj/yCCxZcvz8li3h/fddr3/7W/jmm+Pnx8fD66+7Xt9xh+vfsKrERJgzx/V6xgzYsuX4+QMGuP79AK65BjIyjp8/bBj86U+u11dcAQcOHD9/zBj4/e9dr8ePh2PHjp8/aRL8+teu1z7ys5fTIJo/dRjJvJjexDaE568exITyvcid13IloMCWyFYsb5bA8pixvLZyF//8cgcNK0oZmp/BiMM7GJm3k67HDiBu+NlT4J5ul5LZohvvXN3HFfo+/LMX3TCMl7fN53L68/OnP2VabhovtjuHdhXHePcXYxnUoXn9/uxVfqZ65M6retKAESLSUkSigAlAe6CNqmYDOM+tq1tZRGaISLKIJOfm5rqxTGNcKoCPtCX5RaXeLqVelEoIL8YmcWH/G1jYsju3ZXzDkuHhTOwXe9yVJQJ0P7afm7KTeW1YY9Y9cDGvnNuIn+xbR1Z4Yx5NuJCx/X/OtN5Xs+lw/f7bHA6N4J7O41jUsjt3717O4LiT76L1Ra0p4dVN71EmIbwQN5SJBzaxMH+ZK/T9gFuv4xeRG4BbgaPABuAY8DNVbVZlmUOqWuO/ljX1GHerqFDunZvCO8kZTOoXy9+uHuTtks7Kl+n7eWj+erbmHOXCHq15YFIvElpF12lbmYePsXjDPv5vSTr5x0q5aURnfnlhNyLD694MpKrM+z6TPy7cyOFjpdx4QSfuGdfD65dunqnNe4+w52AhY3q29onLNE/k9Ru4ROQxIAO4HRilqtkiEgssU9XuNa1rwW/cqWro94tvSkpGHi9fn8SFPdp4u7Qzlnn4GH9cuIGPUvfSoUUUD07uxZie9fM5DhWU8KdFG3knOYP2LSJ5ZEofRnWv9g/2Gm3NOcrvPkhl5faDDOzQjD9O7Uuvdk1Ov6I5Y14JfhFprao5ItIB+BQYBtwHHKhycreFqt5d03Ys+I27VA39X47pxm2juzLx2RUUFJfx2V0jia7jLfqetjeviHeT9/D8sq0A3DqqKzeN6OyWk7Mrtx/g/nmpbMstYFK/WB6Y1IvWtehyoKi0nOeXbuWFL7YR2SCUe8f3ZPo57f3uKN+feCv4VwAtgVLgLlVdIiItgXeADsBu4EpVPVjTdiz4jTucGPp3XtQNEWH1roNMe+Ebrh+ewIOTe3u7zGoVlpTx7XbXFTlfpu8nPecoABP6tuX+ib2Ia+beywiLy8qZ88V2nlu6lYZhIdx9SQ9+MqTDKUP8iy25PPDfNHYdKOTygXH8dkJPYhr7xs1ZgczrTT1nw4Lf1LeKCuW3c1N5O3nPcaFf6fcfpPH6t7uYd8t5de4Stz5VVChpWXmsSN/PivRcVu86RGm50jAshKGdW3JB11aM7B5DYhvPnhzdsb+A332QyldbDzCwQzMeu6zvcX3g5+QX8fCCDSxIyaZzq2gendqH4V1bebTGYGbBb4zjuNC/sCt3jk086cRcflEpY5/+guZR4cz/xfln3DFXfdiXX8SyzTmsSN/PV1v3c6jQdUVNr9gmXJDYihHdYhjcsbnXuwJQVT5Ym8kjCzaS55yo/cWF3Xh/dQZPfrKZ4vIKbhvdlZkjO5/xsIjm7FjwG0PtQr/SJ+v3MvO11dxzSQ9mjerisRoLS8p47vOtvLRiO6XlSpsmDTm/q+vu2vO6tjrtYCTecqighMcXbeLt5D2Eh4VQUlbBBd1a8ciUPnW+osicHW/cwGWMTzmT0AcY17st43q34ZnFWxjfp63bw0tV+Sh1L48u3EB2XhHTBsdz0wWdSWzTyCcvFTxR8+hw/jytH1cMjucfX2xj6sA4Jp1wz4DxDXbEb4JCRYVy37xU3lq1h19c2JW7ThP6lfbmFTH26S/o174pr98w1G0htjXnKA99uJ4vt+6nV2wTHpnam8EdfauTMuN/7IjfBK26hj5A26YR3D2+B7//II3312QybXB8vdZWUOxq1vnnl9uJaBDKHy7tzU+GdiDMC+cUTPCw4DcB7WxCv9JPhnTgg+8zeXThBkZ3j6mXPuKra9a5d3wPn22/N4HFDitMQHvso41nFfoAISHC45f3paC4jEcWbDjrmrbmHOWaf37Lrf9ZQ/OocN6fNYwnr+xvoW88xo74TcA6UlTKv1fu4vJBcXUO/Urd2jRm1qiuPLskncsGxTMyMeaMt1FQXMazn6fz8pc7iGgQysNTevOToR19ovteE1ws+E3AWrIxh5KyCq4e0qFeTsreMqoLC1KyuH9eKp/eOYKo8Nr9+mQdPsa/v9nFm9/tJu9YKVcOjucea9YxXmTBbwLWgpRs2jaJqLeuciMahPKny/py1ZyVPLM4nfsm9Kxx+e93H+KfX+5gUdpeVJVxvdsyc2QXn7gT2AQ3C34TkPKLSlm+JZdrh3Ws107AhnZuyY+HtOelFdu5tH87+sQ1PW5+WXkFi9L28vJXO/h+92EaNwzj5+cl8NNhCbRvEVVvdRhzNiz4TUBavGEfJeUVTOwXW+/bvnd8TxZvzOHeuSl8cMt5hIWGkFdYypurdvPvr3eSlVdEx5ZRPDS5F9OS2tPIT3r4NMHDfiJNQFqYkk1cs0gGuqFZpWlkAx6a3Jtb/7OGJz7ZTEFJGe+vzuRYaTnDu7Tk4Sl9GN2jtZ20NT7Lgt8EnLzCUpan53L98AS33Wk7oW9bLurZmn8s3054aAhTBrTjZ+d1sgFFjF+w4DcB59MNeyktVyb2a+e2fYgIf76iHx+l7eWS3m2tb3njVyz4TcBZmJpNfPNI+sc3Pf3CZ6Flo4Zce25Ht+7DGHewO3dNQDlcWMKX6fuZaL1CGnNKFvwmoHy6fh9lFcqkvu5r5jHG31nwm4CyIDWbDi2i6BNnJ1mNORULfhMwDhWU8NVWa+Yx5nQs+E3A+GT9XsorlIl96/+mLWMCiQW/CRgLUrJJaBlFb7uW3pgaWfCbgHDgaDFfb7NmHmNqw4LfBISP1++lQmGSG2/aMiZQWPCbgLAwJZvOMdH0aNvY26UY4/Ms+I3fyz1SzMrtB5jU15p5jKkNC37j9yqbedzZN48xgcStwS8id4rIehFJE5E3RSRCRB4SkUwRWes8JrizBhP4FqZk0bV1IxLbNPJ2Kcb4BbcFv4jEAb8EklS1DxAKTHdm/1VVBziPj9xVgwl8OflFfLvjIBOtmceYWnN3U08YECkiYUAUkOXm/Zkg4xrPFia5YaQtYwKV24JfVTOBJ4HdQDaQp6qfOrNvE5EUEXlZRKodCVtEZohIsogk5+bmuqtM4+cWpmTTvU1jurWxq3mMqS13NvU0B6YAnYB2QLSIXAPMBroAA3B9ITxV3fqqOkdVk1Q1KSYmxl1lGj+2N6+IVbsOumVcXWMCmTubei4CdqhqrqqWAnOB4aq6T1XLVbUCeBEY4sYaTABblJaNKkywvnmMOSPuDP7dwLkiEiWus25jgI0iUvW39DIgzY01mAC2MCWbHm0b07W1Xc1jzJlw29CLqvqtiLwHrAHKgO+BOcBLIjIAUGAnMNNdNZjAlZ13jORdh/j1xYneLsUYv+PWMXdV9UHgwRMmX+vOfZrg8FHqXsBu2jKmLuzOXeOXFqRk0btdEzq1ivZ2Kcb4HQt+43cyDhXy/e7DdjWPMXVkwW/8zqLKZh67mseYOrHgN35nQWo2feOa0rGlNfMYUxcW/Mav7DlYyLo91sxjzNmw4Dd+5cN1ru6erJnHmLqz4Dd+o7isnFe/3sl5XVvSvkWUt8sxxm9Z8Bu/MXdNJjlHirllVFdvl2KMX7PgN36hvEL5xxfb6BfflOFdWnq7HGP8mgW/8QuL0rLZeaCQW0Z1sQFXjDlLFvzG56kqf1+6jc4x0Vzcq623yzHG71nwG5+3PH0/G7LzuXlkF0JC7GjfmLNlwW983t+XbiW2aQRTB8R5uxRjAoIFv/Fpq3cd4tsdB7nxgs6Eh9mPqzH1wX6TjE+bvWwbzaIaMP2c9t4uxZiAYcFvfNaWfUdYvHEf1w1LILqhW4eOMCaoWPAbn/XCF9uIbBDK9cMTvF2KMQHFgt/4pIxDhXy4NosfD+lA8+hwb5djTECx4Dc+6aUVOxCBm0Z08nYpxgQcC37jcw4cLeatVbuZOiCO2KaR3i7HmIBjwW98zitf76S4rIKZI7t4uxRjApIFv/EpR4pKefXrnYzr1ZaurRt5uxxjAtJpg19EJomIfUEYj3jzu93kF5Uxa5Qd7RvjLrUJ9OlAuoj8RUR6ursgE7yKy8p5acUOzuvakv7tm3m7HGMC1mmDX1WvAQYC24B/icg3IjJDRBq7vToTVCoHWpk10gZaMcadatWEo6r5wPvAW0AscBmwRkR+4cbaTBCpOtDKeV1toBVj3Kk2bfyTRWQe8DnQABiiquOB/sCv3VyfCRKVA63MGmkDrRjjbrXpAOVK4K+qurzqRFUtFJGf17SiiNwJ3AgokAr8DIgC3gYSgJ3Aj1T10BlXbgKGqjJ7mWuglXG9baAVY9ytNk09DwLfVb4RkUgRSQBQ1SWnWklE4oBfAkmq2gcIxXWi+F5giap2A5Y4700QW56+n/VZ+dw8wgZaMcYTahP87wIVVd6XO9NqIwyIFJEwXEf6WcAU4FVn/qvA1FpuywSo2cu20rZJBFMH2kArxnhCbYI/TFVLKt84r0/ba5aqZgJPAruBbCBPVT8F2qhqtrNMNtC6LoWbwPDOqj2s3H6QGy/oZAOtGOMhtflNyxWRSyvfiMgUYP/pVhKR5riO7jsB7YBoEbmmtoU5l4wmi0hybm5ubVczfuSVr3Zw9/spXNCtFdec29Hb5RgTNGoT/DcD94nIbhHZA9wDzKzFehcBO1Q1V1VLgbnAcGCfiMQCOM851a2sqnNUNUlVk2JiYmrzWYwfeX7pVh6av4Fxvdvw0nVJRDQI9XZJxgSN017Vo6rbgHNFpBEgqnqkltve7awXBRwDxgDJQAFwHfC48/zfuhRu/JOq8sQnm/n7sm1MHdCOJ6/sT1ioNfEY40m1Gs9ORCYCvYGIymusVfXhmtZR1W9F5D1gDVAGfA/MARoB74jIDbi+HK6sc/XGI95etZuV2w9y50WJdGgZVeftVFQoDy/YwCtf7+THQzrwx6l97CoeY7zgtMEvIi/guiJnNPASMI0ql3fWRFUfxHU5aFXFuI7+jR/IOnyMBz9cT1FpBQtTs5lxQWduGd2FqPAzGwO3vEK59/0U3l2dwY3nd+L+iT3tRi1jvKQ2f2MPV9WfAodU9Q/AMKC9e8syvuLxRZtQhfdnDWNCn7b8belWxjz1BfPXZaGqtdpGaXkFt7/1Pe+uzuD2Md0s9I3xstoEf5HzXCgi7YBSXFfqmAC3etdBPlyXxcwRnRncsQXPTB/IuzcPo0V0OL9483umz1nJxuz8GrdRVFrOrNdXsyAlm/sm9ODOsYkW+sZ4WW2Cf76INAOewNVevxN40401GR9QUaH8Yf4G2jaJ4OYqfeOfk9CCD287n8cu68uWfUeY+OwKHvhvGocLS07aRkFxGTe8uorFG3N4ZGofZoywPvaN8QU1NtQ6A7AsUdXDwPsisgCIUNU8TxRnvOf9NRmkZOTxzFUDTmrPDw0Rrh7agQl92/LXz7bw2spdzF+Xxa/HdWf6OR0IDRHyjpXy81dW8f3uQzx1ZX+uGBzvpU9ijDmRnK6dVkS+UdVhHqqnWklJSZqcnOzNEoLK0eIyRj+5jPjmkcydNfy0TTMbs/N56MP1fLvjIL1im/CrixP56+ItbN57hGenD2R831gPVW6MqUpEVqtq0onTa9PU86mIXCHWMBs0nl+6ldwjxTw4uXet2uN7xjbhrRnn8rerB3KosIQbXk0mfd9R5vw0yULfGB9Um2vy7gKigTIRKQIEUFVt4tbKjFfsPlDIP1fs4PJBcQw4g+EPRYRJ/dpxYY/WvLFyN4M6Nmdwx+buK9QYU2e1uXPXhlgMIo99tJGwUOGeS3rUaf2o8DBuGtG5nqsyxtSn2tzANaK66ScOzGL839fb9vPx+r38Zlx32jSJ8HY5xhg3qU1Tz2+qvI4AhgCrgQvdUpHxirLyCh6ev4H45pHccL7dpmFMIKtNU8/kqu9FpD3wF7dVZLzirVV72LT3CLN/Msh6yjQmwNWlW8QMoE99F2K8J+9YKU9/toWhnVpwSR8b89aYQFebNv7ncA2WDq4vigHAOjfWZDzs2SXpHC4s4YHJvaw7BWOCQG3a+KveOVUGvKmqX7mpHuNhW3OO8urXO7nqnA70btfU2+UYYzygNsH/HlCkquUAIhIqIlGqWuje0own/HHhBiLDQ/n1xYneLsUY4yG1aeNfAkRWeR8JLHZPOcaTlm7OYenmXG4f042WjRp6uxxjjIfUJvgjVPVo5Rvndd2HYTI+obS8gkcXbKBzq2h+OizB2+UYYzyoNsFfICKDKt+IyGBcY+gaP/baN7vYllvA7yb1JDzMxrw1JpjUpo3/DuBdEcly3scCV7mtIuN2BwtKeGbxFkYkxjC6e2tvl2OM8bDa3MC1SkR6AN1xddC2SVVL3V6ZcZunP9tMQUk5v7chEI0JSqf9G19EbgWiVTVNVVOBRiJyi/tLM+6wMTuf/3y7m2vP7Ui3Ntb/njHBqDaNuzc5I3ABoKqHgJvcVpFxG1Xl4fkbaBrZgDsvsss3jQlWtQn+kKqDsIhIKBDuvpKMu3yyfh/fbD/AXWMTaRrVwNvlGGO8pDYndz8B3hGRF3B13XAzsMitVZl6V1RazmMfbaR7m8b8eEgHb5djjPGi2gT/PcAMYBauk7vf47qyx/iRl7/awe6Dhbxx41DCQu3yTWOC2WkTQFUrgJXAdiAJGANsdHNdph7l5Bfx/OdbubhXG87r2srb5RhjvOyUR/wikghMB34MHADeBlDV0Z4pzdSXv3yymdJy5f6JPb1dijHGB9TU1LMJWAFMVtWtACJyp0eqMvVm3Z7DvLc6g5kjO9OxZbS3yzHG+ICamnquAPYCS0XkRREZg6uNv1ZEpLuIrK3yyBeRO0TkIRHJrDJ9wtl+CFM9VeXhBRto1aght43u6u1yjDE+4pTBr6rzVPUqoAewDLgTaCMis0Xk4tNtWFU3q+oAVR0ADAYKgXnO7L9WzlPVj872Q5jqfbgui9W7DnH3uO40jrDLN40xLrU5uVugqm+o6iQgHlgL3HuG+xkDbFPVXWdeoqmLYyXlPL5oE33jmjJtcLy3yzHG+JAzuq5PVQ+q6j9U9cIz3M904M0q728TkRQReVlEmle3gojMEJFkEUnOzc09w92ZF77YRnZeEQ9M7kVIiPXHY4z5H7df0C0i4cClwLvOpNlAF1xj92YDT1W3nqrOUdUkVU2KiYlxd5kBJfPwMf6xfBuT+7fjnIQW3i7HGONjPHEnz3hgjaruA1DVfapa7twf8CIwxAM1BJXHF20C4N7xPbxciTHGF3ki+H9MlWYeEal61+9lQJoHaggaq3YeZP66LGaO6EJcs8jTr2CMCTq16bKhzkQkChgLzKwy+S8iMgBXvz87T5hnzkJFhav3zdimEdw8sou3yzHG+Ci3Br+qFgItT5h2rTv3GczeW5NBamYe/zd9AJHhod4uxxjjo6y3rgBxpKiUv3y8mcEdm3Np/3beLscY48PcesRv6u7DdVks3ZRDZHgo0eGhRIaHERUeSlR4KJENQoly3kc60+auyWT/0WL+eV2SDadojKmRBb+PeurTzeQeKSayQSiFJeUcKy0/7TpXDIqnf/tm7i/OGOPXLPh9UF5hKbsOFPKbcd251eljp6JCKSord30JlLieC0vKfnhdXFbBiETrctkYc3oW/D4oJfMwAP3jm/0wLSREnOYd+y8zxpwdO7nrg1Iy8gDoG9fUy5UYYwKRBb8PSsk4TMeWUTYgujHGLSz4fVBqRh79qjTzGGNMfbLg9zG5R4rJyiuinzXzGGPcxILfx6Q6J3b7xVvwG2Pcw4Lfx6Rk5CECve2I3xjjJhb8PiYlI4+uMY1o1NAu2zTGuIcFvw9RVVIy8uhrzTzGGDey4Pch2XlF7D9afNyNW8YYU98s+H3IDzdu2RG/McaNLPh9SGrmYcJChF6xTbxdijEmgFnw+5CUjDwS2zQmooENomKMcR8Lfh9ReWLXrt83xribBb+P2H2wkLxjpdZVgzHG7Sz4fUTliV074jfGuJsFv49IyThMeFgIiW0ae7sUY0yAs+D3ESkZefSMbUJ4mP2XGGPcy1LGB1RUKGmZefS3Zh5jjAdY8PuA7fuPUlBSbiNuGWM8woLfB/zvxG4z7xZijAkKFvw+ICUjj8gGoXRt3cjbpRhjgoAFvw9IyThMn7gmhIaIt0sxxgQBtwW/iHQXkbVVHvkicoeItBCRz0Qk3Xlu7q4a/EFpeQXrs/KtmccY4zFuC35V3ayqA1R1ADAYKATmAfcCS1S1G7DEeR+00vcdpbiswm7cMsZ4jKeaesYA21R1FzAFeNWZ/iow1UM1+KSUjMOAndg1xniOp4J/OvCm87qNqmYDOM+tq1tBRGaISLKIJOfm5nqoTM9LycyjcUQYHVtEebsUY0yQcHvwi0g4cCnw7pmsp6pzVDVJVZNiYmLcU5wPSHV65AyxE7vGGA/xxBH/eGCNqu5z3u8TkVgA5znHAzX4pOKycjbtzadvXDNvl2KMCSKeCP4f879mHoAPgeuc19cB//VADT5pU/YRSsvVumowxniUW4NfRKKAscDcKpMfB8aKSLoz73F31uDLKk/s2hi7xhhPCnPnxlW1EGh5wrQDuK7yCXopGXm0iA4nrlmkt0sxxgQRu3O3jnbuL+C5JekcLCip8zYqh1oUsRO7xhjPcesRfyDaub+Avy3dyrzvMymvULLyivjT5X3PeDuFJWWk5xxhXO82bqjSGGNOzYK/lqoGfliIcP3wBA4XlvJO8h5uuqATnWPOrIO1DVn5VKjduGWM8TwL/tOoLvBnjuxM68YR5B4pZlFaNk99toXnrx50RttdZ2PsGmO8xIL/FGoK/EoxjRty4/mdePbzrdw8Iu+Mrs5JzThM2yYRtG4ScfqFjTGmHlnwn+DEwL9uWAI3j+x8yoC+cURnXlu5i798sonXbhha6/2kZJzZF4UxxtQXC/4q/vzxJuYs316rwK/UJKIBt47uyqMLN/L11v0M79rqtPvJLypl+/4CLh8UV1+lG2NMrdnlnI6N2fnMXraNCX1jWXH3aB6Y3KvWzTDXnNuR2KYR/PmTzajqaZdPc9r3+9qJXWOMF1jwO2Yv20Z0eCiPTulzxu3uEQ1CufOiRNbtOcynG/addvmUTCf4bXB1Y4wXWPADuw8UsiAli5+c25GmUQ3qtI3LB8XRJSaaJz7ZTHlFzUf9qRl5tG8RSYvo8DrtyxhjzoYFPzBnxTbCQkK44fxOdd5GWGgIv764O1tzjjJ3TUaNy67LOEw/65HTGOMlQR/8OUeKeCc5gysGx9HmLC+tvKRPW/rFN+WZxekUlZZXu8zBghIyDh2z6/eNMV4T9MH/r692UlZewcwRXc56WyLCPZf0IPPwMd74dne1y1iPnMYYbwvq4M8vKuX1b3Yxvm8sCa2i62Wb53VtxfldW/H80q0cKSo9aX5qhp3YNcZ4V1AH/+srd3GkuIxZI8/+aL+q34zrzsGCEl5aseOkeesy8ugcE03jiLqdRDbGmLMVtMFfVFrOy1/uYERiDH3q+ei7f/tmjO/TlpdWbOfA0eLj5qVmHqa/Xb9vjPGioA3+d1dnsP9oSb0f7Vf61cXdOVZazvNLt/0wbV9+Efvyi62ZxxjjVUEZ/GXlFcxZvo2BHZpxbucWbtlH19aNuHJwe15fuYuMQ4WAq38egP7tLfiNMd4TlMG/MDWbPQePMWtkF7eOfnX7Rd1A4JnF6YCrR84QgV6xFvzGGO8JuuBXVWYv20a31o24qKd7R79q1yyS64Z1ZO6aDLbsO8K6jDwS2zQmMjzUrfs1xpiaBF3wL92cw6a9R7h5ZBdCQtw/1u0to7oSHR7GE59sJjUzz27cMsZ4XdAF/+xl24hrFsmlA9p5ZH/No8OZMaIzn23Yx8GCEuuR0xjjdUEV/Kt2HmTVzkPcdEEnGoR67qP//PxOtGrk6pCtvx3xG2O8LKiCf/aybbSIDueqczp4dL/RDcO4d3xPurVuRPe2jT26b2OMOVHQjMC1MTufzzfl8KuxiV45uTptcDzTBsd7fL/GGHOioDnif+EL10ArPx2W4O1SjDHGq4Ii+HcfKGT+urMbaMUYYwKFW4NfRJqJyHsisklENorIMBF5SEQyRWSt85jgzhoAXlyx/awHWjHGmEDh7jb+/wM+VtVpIhIORAHjgL+q6pNu3jcAuUeKeSd5T70MtGKMMYHAbcEvIk2AEcD1AKpaApS4s4uE6vzrqx2Ullcwox4GWjHGmEDgzqaezkAu8C8R+V5EXhKRytFObhORFBF5WUSaV7eyiMwQkWQRSc7Nza1TAflFpbzmDLTSqZ4GWjHGGH/nzuAPAwYBs1V1IFAA3AvMBroAA4Bs4KnqVlbVOaqapKpJMTExdSrgjZW73TLQijHG+DN3Bn8GkKGq3zrv3wMGqeo+VS1X1QrgRWCIuwpo1SicHyXF1/tAK8YY48/c1savqntFZI+IdFfVzcAYYIOIxKpqtrPYZUCau2q4Mqk9Vya1d9fmjTHGL7n7qp5fAG84V/RsB34GPCsiAwAFdgIz3VyDMcaYKtwa/Kq6Fkg6YfK17tynMcaYmgXFnbvGGGP+x4LfGGOCjAW/McYEGQt+Y4wJMhb8xhgTZCz4jTEmyIiqeruG0xKRXGBXHVdvBeyvx3I8yWr3Dn+t3V/rBqvdXTqq6kl93vhF8J8NEUlW1RPvJfALVrt3+Gvt/lo3WO2eZk09xhgTZCz4jTEmyARD8M/xdgFnwWr3Dn+t3V/rBqvdowK+jd8YY8zxguGI3xhjTBUW/MYYE2QCKvhFpL2ILBWRjSKyXkRud6a3EJHPRCTdea52nF9vE5FQZ3ziBc57f6m7mYi8JyKbnH/7YX5U+53Oz0qaiLwpIhG+WrszRnWOiKRVmXbKWkXktyKyVUQ2i8g471T9Qy3V1f6E8zOTIiLzRKRZlXk+XXuVeb8WERWRVlWm+UztpxJQwQ+UAb9S1Z7AucCtItIL11i/S1S1G7DEee+Lbgc2VnnvL3X/H/CxqvYA+uP6DD5fu4jEAb8EklS1DxAKTMd3a38FuOSEadXW6vzcTwd6O+v8XURCPVfqSV7h5No/A/qoaj9gC/Bb8JvaEZH2wFhgd5VpvlZ7tQIq+FU1W1XXOK+P4AqgOGAK8Kqz2KvAVK8UWAMRiQcmAi9VmewPdTcBRgD/BFDVElU9jB/U7ggDIkUkDIgCsvDR2lV1OXDwhMmnqnUK8JaqFqvqDmArbhzf+nSqq11VP1XVMuftSiDeee3ztTv+CtyNazTBSj5V+6kEVPBXJSIJwEDgW6BN5Ti/znNrL5Z2Ks/g+iGqqDLNH+ruDOQC/3KaqV4SkWj8oHZVzQSexHXElg3kqeqn+EHtVZyq1jhgT5XlMpxpvurnwCLntc/XLiKXApmquu6EWT5fOwRo8ItII+B94A5Vzfd2PacjIpOAHFVd7e1a6iAMGATMVtWBQAG+0zRSI6c9fArQCWgHRIvINd6tqt5INdN88tptEbkfVzPtG5WTqlnMZ2oXkSjgfuCB6mZXM81naq8UcMEvIg1whf4bqjrXmbxPRGKd+bFAjrfqO4XzgEtFZCfwFnChiLyO79cNriOaDFX91nn/Hq4vAn+o/SJgh6rmqmopMBcYjn/UXulUtWYA7assF4+rGcuniMh1wCTgJ/q/m4p8vfYuuA4W1jm/s/HAGhFpi+/XDgRY8IuI4Gpr3qiqT1eZ9SFwnfP6OuC/nq6tJqr6W1WNV9UEXCeGPlfVa/DxugFUdS+wR0S6O5PGABvwg9pxNfGcKyJRzs/OGFznhfyh9kqnqvVDYLqINBSRTkA34Dsv1HdKInIJcA9wqaoWVpnl07WraqqqtlbVBOd3NgMY5Pwu+HTtP1DVgHkA5+P6syoFWOs8JgAtcV3xkO48t/B2rTV8hlHAAue1X9QNDACSnX/3D4DmflT7H4BNQBrwGtDQV2sH3sR1LqIUV9jcUFOtuJojtgGbgfE+WPtWXO3hlb+rL/hL7SfM3wm08sXaT/WwLhuMMSbIBFRTjzHGmNOz4DfGmCBjwW+MMUHGgt8YY4KMBb8xxgQZC37jU0TkTyIySkSmiki1dwA783rVYduXnmqbVZZpJyLvnem2vU1EEqrrPdKY6ljwG18zFFf/SiOBFadYZipQbfA7na1VS1U/VNXHa9q5qmap6rTalWqMf7LgNz7B6Zs9BTgH+Aa4EZgtIg+csNxw4FLgCRFZKyJdRGSZiDwmIl8At4vIZBH51uk0brGItHHWvV5E/ua8fkVEnhWRr0Vku4hMc6b/cOTsLD9XRD52+rv/S5U6bhCRLc6+X6zc7gm1jnRqXOvU0lhEGonIEhFZIyKpIjKlyn43OZ3cpYnIGyJykYh85ex7iLPcQyLymoh87ky/qZr9hjr/nqvE1df9TGd6rIgsd+pJE5ELzvo/zvgnb99BZg97VD5wdV/7HNAA+KqG5V4BplV5vwz4e5X3zfnfeNI3Ak85r68H/lZlG+/iOvjpBWx1picAaVWW3w40BSKAXbj6YWmH627NFk6tKyq3e0Kd84HznNeNcHVoFwY0caa1wnX3qjj7LQP6OjWtBl525k0BPnDWeQhYB0Q66+9x6qla9wzgd87rhrjuqu4E/Aq435keCjT29v+5PbzzOOWfxcZ4wUBct+73wNXfz5l4u8rreOBtp9OycGDHKdb5QFUrgA2VfxVUY4mq5gGIyAagI67A/UJVDzrT3wUSq1n3K+BpEXkDmKuqGU4ngo+JyAhcXXDHAZX73qGqqc421zv7VhFJxRXslf6rqseAYyKyFNcX5toq8y8G+lX+FYPri6sbsAp42anhA1Wtuo4JIhb8xutEZACuI/B4YD+uAVFERNYCw5yQO52CKq+fA55W1Q9FZBSuo+TqFFctoxbLlOP6nTnVssdR1cdFZCGu/qJWishFuEaGiwEGq2qp07tjRDX7qqjyvoLjf1dP7GflxPcC/EJVPzmxJucLZyLwmog8oar/rs1nMYHF2viN16nqWlUdgGv4vV7A58A4VR1witA/AjSuYZNNgUzn9XU1LFdX3wEjRaS5czL5iuoWEpEu6urJ8c+4mlt6OLXlOKE/GtdfEGdqirjGBm6Jq1O/VSfM/wSY5RzZIyKJIhItIh2dfb+IqxfbQXXYtwkAdsRvfIKIxACHVLVCRHqoak1NPW8BL4rIL4HqrsB5CHhXRDJxDenXqT5rVdVMEXkM19VHWbiapfKqWfQOJ9zLnWUW4frCmi8iybiaZzbVoYTvgIVAB+ARVc0S14hzlV7C1TS0RkQE1whpU3F9SfxGREqBo8BP67BvEwCsd05j6kBEGqnqUeeIfx7wsqrO88B+HwKOquqT7t6XCVzW1GNM3TzknINIw3Xy+AOvVmPMGbAjfmOMCTJ2xG+MMUHGgt8YY4KMBb8xxgQZC35jjAkyFvzGGBNk/h+YPuCfExToawAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# experiment\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.plot(np.arange(num_initial_samples, max_training_samples + 1, acquisition_batch_size), test_accs)\n",
    "plt.xlabel(\"# training samples\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.hlines(90, 20, 150, linestyles=\"dashed\", color=\"r\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:active_learning]",
   "language": "python",
   "name": "conda-env-active_learning-py"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}