notebooks/validation_exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pickle\n",
    "import dicto as do\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from python_path import PythonPath\n",
    "with PythonPath(\"/code/traffic_sign_classifier/\"):\n",
    "    from german_traffic_densenet import model_fn\n",
    "    from german_traffic_dataset import input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = do.load(\"/code/traffic_sign_classifier/configs.yml\")\n",
    "with open(\"/data/valid.p\", \"rb\") as fd:\n",
    "    validation = pickle.load(fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'coords': (4410, 4), 'labels': (4410,), 'features': (4410, 32, 32, 3), 'sizes': (4410, 2)}\n"
     ]
    }
   ],
   "source": [
    "print({key: value.shape for key, value in validation.items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAECCAYAAAAxcwG5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAGwJJREFUeJztnW2wpGV5539Xd58zCDMKBIEBXwgytUGtFdlZYsClTOKmWJISzVvpB8MHl0lZcUtrs7tFmdrgpvYlZqMmlWyZwoISiKvi20KlrKwuqwtWZYkjIi/CBlAwyMhgAEGZmXO6+9oP3WMOQ1//06fPOX2G3P9f1anT57n7ue/ruZ/n6uf0/X+u64rMxBjTHp2tNsAYszXY+Y1plN60bzwr4veOh5dtpjHT8CR85/7M391qO4x5vjO18x8PL9sLD26iLVOxG87YahuM+YeA/+03plHs/MY0ip3fmEaZ+jt/xf3wgj+Fs78Mux6GU34AO7owOA32XwJf/69wew9+/DDBX8Hx58O7q/52w91fhU+v1y5jjGbdzv9H8Kr/Br+4A374Svj2KfCDx2H7Xjj7j+BNN8Our8L1R/6LcRo8egHce2R/r4b967XJGLM663b+V8HfvR8+/q/hvpV3+LvgptfDZbfB2f8Jzv73cM/K/c6A710PX17v+MaY2Vj3d/53wrf/HfzNSscHeDX88BdhL8DNlueMOepY951fsQCD8SDDI9sehx3/Cv7J43DsifDMP4eH3wSPbqY9xpi/Z9Oc/yB0/ie8BuAiuP/I9nvhzHvhzMN//ymwCx78c/gf58EPNssuY8yITXP+X4E3fg9OfhXc92544PD242H5V+Dmt8K958ETAF+AU/4A3nAfnHEx/MY34c9OhuXNss0Ys0k6/x746c/Dz5wM378RPrey7Wz40afhS78K+14GB18GB/8lPHQHXPdy+O7fwYm/A+duhl3GmL9nXXf+/wtn3QIXJXTOgtt+Gb7yTvinH4GLToHH/hdceyYcOPz+34vOFb3e5CGD5NzhkIcGA/4q4qL/0utddLht5Urik8MhO7bvuPTwXrNxdOQwODqsmJ1ZZ3/jmd9MypFE48ZbOLnHQ4cO0e/3pzo1Mzv/AOL/wMVvg+tOhaf+BC77Ddh2Hbz+VNj/v+Has+FHzxqs1+O0nadP7K+TQx498Ax8/zE627Zx+otP+XHbcMVxnnToIC/Z9VNjG+p/XEIcfnfYr4+r0613rE7hsD61yo5BPmcddJouZZ90Js9JiKQtQ3lp1oN1RZ9RGNkJcc5yULYNh+KgxTxW50ztoVxVjiTmY6A+GNTJrneauPnue++euouZnf8uOH07PP6y8ff2W+DpG+D1p8H3vgzX7YJn1trn/YcOAfDi7qaKEMYY1vGd/0l44bHwFMBb4cIb4KzT4Jlb4Frl+A8cOkh/wifkNw8e4AtPPwXA+ccdN6tZxpgpmfkWe9h9/y285pPwswH5Cnjqcvjple/7Eew8Fk7+GTgwHA741JNP8N3lJX5q2zGcOP7+/7dLS9xz6CAAv/zCF7Fr2zGzmmWMmZKZnf8EeOpueOFDcAJAQtwCpzL6eRZnwUOfgo/+5073ivOP285tz/yIby8tcefBAwwyeWG3y3kvOJY37tjBP7LjGzMXZnb+V8Ejfwk/8YfwF38Ot/wJXPYm+MwueEztd+H2HVy4fcdztnfkoo0xZqOZ2fl7MPxn8PlPwNsT4iz4+mqOTyb9/uRnd+QK9ooV52EOWR70n7P9uXuIFdRh/UHTp1YChtVqrljl7YjjWhYfeDKj+gxKgJpfOZRsFB/Yxap+V9ih5qqceyBmkdjEhOhs9uJcS5WgXl6rVAKl0JTmr0E4WNey+vlw3/lw33r6MMZsDc7kY0yj2PmNaRQ7vzGNYuc3plHs/MY0ytSr/U/Cd9ZbLeeJ4ZAXHZj85G9HSC8rWw4uLtJZXgJgKAJBpPwmbBwI+bCSZJQ0lDMGiSjlswqaGTH56Dqd2g5tv7BDtRUy4FBMfio5TIylZLtOeXD1PvK8iCCcoZIPRZ9V27Q+MStTO/9G1Mc79tjjLn3Ry8+Y2Nbp1tF0K69bP/VvzMbgf/uNaRQ7vzGNYuc3plHs/MY0ypxT5mSZn2ooojM6RWoqRGontcren3l1e+1rrLOulqs9tRIwebtaOVbpp2RAjTBklsATdc7UUac6tg0IgJnSDFIGmq0dNfcbge/8xjSKnd+YRrHzG9Modn5jGsXOb0yj2PmNaZSjpjqGyrk3i+ChVJKBkq9mGmvGMIuoQ0hmkRWhlvSU1DeMOkBK5cdTk1zKXkoelNKWkNHELSyKcyOrFMmmGc+Laiy6VFJfmU5yaot85zemWez8xjSKnd+YRrHzG9Modn5jGsXOb0yjrEvqi4gHgaeBAdDPzN2r7FHKYkrlqWSqgdChBgNRCqseSkpi9Z5KshPIsVQUnopmrOZ3luPSEuxQ2VjsJ0trqYtASYQiMWA1H+quJyU2sZ86OJH6T3Q6k8g99Ts3Quf/2cz8/gb0Y4yZI/6335hGWa/zJ/CFiPhaROzZCIOMMfNhvf/2X5CZj0TEycAXI+LezLx55RvGHwp7ABYWFtc5nDFmo1jXnT8zHxn/3g98DjhvwnuuzMzdmbm71ztqQgmMaZ6ZnT8ijouIHYdfA78A3LVRhhljNpf13IpPAT43Lh3VA/57Zv6l3iWppIjMumLPsEzUufayVbCKwjZzZseqNxGNJiU7lThT6V7VWHXk3kKnnvuBKPO1JOTU8ozJuReSnQjdUzLmsCgb1tHhiqJN2CHqfEkZsxqpM8N5XkPU4czOn5nfAl4z6/7GmK3FUp8xjWLnN6ZR7PzGNIqd35hGsfMb0yhb8NRNoYfImnCVPFiPoiQlFbWlEjuWfYrBYsbPVxVNp8Sc6oT2RH9CzYNOfYl0ol+2ZSEtKjms262fAO0uqEtVRHDmZBuHQqZkWMuiCMl09lp9a8/GWV8f00uKvvMb0yh2fmMaxc5vTKPY+Y1pFDu/MY1y1MTYpsp/VuWDU3EPs6WsI2faccZyXVLhqHdTbb1i6b7TV/nl6rauKCm2TdhxqArGEkFE3W31an+vW++XwyXRNvn+poKqVP2voVrtHyiVoG6qrgMV3LVKhNRU+M5vTKPY+Y1pFDu/MY1i5zemUez8xjSKnd+YRpmv1JfUsoaqZ1Q0RdTyT4hgj86MJahmkfRUgI7WHOsmGYhTyk2zjZXDOnhnobdQtlXy4UER2bO89EzZNpDnWpXXmjyeynXXFWOpu2Wggn5mkO2UnFfm95v+GvWd35hGsfMb0yh2fmMaxc5vTKPY+Y1pFDu/MY2yqtQXEVcDvwTsz8xXj7edCHwSOAN4EPj1zHxi1dECooiYUurVsJA8tIw2m2QXMyQGlAFiM5pYzRPAQleUIiukuWoOQefHGyzXUt9QRLH1Chu7/Xof1V8RnAfoM13mEhTnOUNE54l57IhzNhDRkaUdKuqz7G5jc/h9FLjoiG2XAzdl5i7gpvHfxpjnEas6f2beDDx+xOZLgGvGr68B3rzBdhljNplZv/Ofkpn7AMa/T944k4wx82DTH++NiD3AHoCFhfpxUGPMfJn1zv9oROwEGP/eX70xM6/MzN2ZubvXO2qyhhnTPLM6/43ApePXlwI3bIw5xph5MY3U93HgDcBJEfEwcAXw+8D1EfEO4DvAr009YinbCQrJQ0ZzqUi1jpLK1h6ZpSLEFFIFVCXAhqI8VdHUOaY+1QvbttVjCSsHy0ISY3Jk3IIo/7UsIv5StA1VRGhhv4qyG4gknR2RgLQjzpkq5SVrmNUdrptVnT8z31Y0/fz6hzfGbBV+ws+YRrHzG9Modn5jGsXOb0yj2PmNaZS5PnUT1EFRMoKpkEmiTGII0VGykRhLyICl5KgixOreZE24BdHWUfXiCvu7C0LOE3Xwuov1U5kpJcfJR74ga+7Vs7U8qMfS0Z2TUfUJpcwqFLuuuHb6wv5hERaqxqqsX8tM+M5vTKPY+Y1pFDu/MY1i5zemUez8xjSKnd+YRpmr1JeEjm4q95vMQIbu1U0ySecMspE6JilDqVJsotagsrHTnXxKOyLpp6pP2FlYLNsWhOS4fGB5coOIYNtWq4BSnh0IibBqUnKeChLsyui8ukldB1Fm41RRguvHd35jGsXOb0yj2PmNaRQ7vzGNYuc3plHmnE43y+AYteJcrW2qnGkh6mR1xVqpCvgombEkV1cE76hOlbrQ7U4OxEmVX07mziubWBDBQv0iv99AlOTqZL3c31OlsKj7rAPJyl1WyScp5krsNtN1pczYgD585zemUez8xjSKnd+YRrHzG9Modn5jGsXOb0yjTFOu62rgl4D9mfnq8bb3AZcBj43f9t7M/PxUIxZSSYjPoSoQR5bJktKWahNBM9VQSjoUx6WCRDoiP16IIJ3eYhGII6bqwNJS2aby6nVFma9OIQMOh4fqsYQMGELqU3kXK2lRKMEMZdCPOGd1l1IirGTuWXITroVp7vwfBS6asP1DmXnO+Gc6xzfGHDWs6vyZeTPw+BxsMcbMkfV8539XRNwREVdHxAkbZpExZi7M6vwfBl4BnAPsAz5QvTEi9kTE3ojY2+/3ZxzOGLPRzOT8mfloZg5y9FD4R4DzxHuvzMzdmbm715tzKIExpmQm54+InSv+fAtw18aYY4yZF9NIfR8H3gCcFBEPA1cAb4iIcxgFFz0I/OY0gwVCvhDaS7WP+uQqMsgBq5XQWnveNNlfV0hDZe42HZUYHVXyavJXq/6heqyDh2qpb1slHQJC9WJhoSjz1a/H6gupT0ViLgods+pRRuAJebOjSsQJOVJFVYqd6rHW3ttzWNX5M/NtEzZftQFjG2O2ED/hZ0yj2PmNaRQ7vzGNYuc3plHs/MY0ytyfulFyyFpJVQJJlXfSWt+ax1PH1FMSjyh3pSLVur1a6ju0NDlq7lC/HkuWp+qKGlpqrgqJU0UCDgf1E6DDQS1V9oT0uVjM45KYDx2dJxKryipwa09Qq/orbVxDIKDv/MY0ip3fmEax8xvTKHZ+YxrFzm9Mo9j5jWmU+QfYF9KXSsbZnSUiSu6jIgjXjqoZuChkqBCaY4TYT0SWdUuJcLZEKiqBp1S2iu0hpMOOiCAcHKwTf3aEJlZJfcsqolL0txlJNctrX0R2qkSz0+I7vzGNYuc3plHs/MY0ip3fmEax8xvTKHNe7Q+qzxu5OF+0yVgJsW6vVnOHqlxXtcqugj0GdTbBjlI4FkTuPLHQu1zkwVOr9osiUEhUDSM7a19xrkpTAURR4gugKwJxcrluq66DxW6RYxA4JAKMUGXDxEWsyoNVTcMNVqWOxHd+YxrFzm9Mo9j5jWkUO78xjWLnN6ZR7PzGNMo05bpeClwLnAoMgSsz848j4kTgk8AZjEp2/XpmPqH6ykwGlYyiZJJC2FhQQTNChlKBOFo+nMyiKMkVSisTslF3oT41Iq6Hg0sHJ24fCH2wK4KI+sL+oZisXmWksF3lLexsq6XP/vBA2TYsypctiLGU3NsX8ltHSnP1PGYxKR11b660wzVcv9Pc+fvAb2fm2cDrgN+KiFcClwM3ZeYu4Kbx38aY5wmrOn9m7svM28avnwbuAU4HLgGuGb/tGuDNm2WkMWbjWdN3/og4A3gtcCtwSmbug9EHBHDyRhtnjNk8pn68NyK2A58B3pOZT6lHGY/Ybw+wB0TZZmPM3Jnqzh8RC4wc/2OZ+dnx5kcjYue4fSewf9K+mXllZu7OzN297vwTBxljJrOq88foFn8VcE9mfnBF043ApePXlwI3bLx5xpjNYppb8QXA24E7I+L28bb3Ar8PXB8R7wC+A/zadEOuvTRRGYWn6yPVLSLESkVfVVF9PdWfkoYWxfQrPU/Ib9WUdLr153xfzWPWUYmDZRENuDC5Tc5v3QQiCq+3WEf1HSpy/6nIzm29+rwMRQDhUErIa4/4U+XoyqY1BFqu6vyZ+RVq9fDnpx/KGHM04Sf8jGkUO78xjWLnN6ZR7PzGNIqd35hGmetTNxF1RJ2IfdOKXjWWiNrqioeNdPRb0SaSY8qEj0K+UvYPRTLLar/jRCmsA4dEklGV3HN5qWzr5+Rj6woJc7lf25Gd+pwt9Opj6y5MvrIGS/VYUB/zoig3pq5hRanayUjXtTY8F9/5jWkUO78xjWLnN6ZR7PzGNIqd35hGsfMb0yhzlfoSyFLCWnvCzSqx52r0RE24RZH4s1vU3UshG4VIYNIVkXZkLecdEJJYFBFpC716LKH0oXIwbIta3Fo6NDma7hkhU/ZFyFwI+e2YFxxTti32ivkvahoCDEWtvp6Q+vrychT1IQs5WHVXJQtdi0f4zm9Mo9j5jWkUO78xjWLnN6ZR7PzGNMqc0+lGWZpIL1NOXlVWOc46okOVdryHWAXuV6XGRJkpEXQSPZHzTax8D4b18vzC4uSVb5WzriyttUqbCrgaFOdMBU6FmkeR/G9YqDBAmfuvJwKF+oM6YCnF3KsycLOt3K+9rNxaYuB85zemUez8xjSKnd+YRrHzG9Modn5jGsXOb0yjrCr1RcRLgWuBUxlpbldm5h9HxPuAy4DHxm99b2Z+XveWRBEM0hUBE1F8RqkSVHVvECJoJlQprCJXn8rF11VSn8j6piQbFVyyMGX15Gf1JyTHjkpMJ3IXLnYmz8m2OgZHypHkjBnylibLszmo+1MycQo7lDS3IHIyLlfzqGqblcFu0zONzt8Hfjszb4uIHcDXIuKL47YPZeYfrmE8Y8xRwjS1+vYB+8avn46Ie4DTN9swY8zmsqbv/BFxBvBa4NbxpndFxB0RcXVEnLDBthljNpGpnT8itgOfAd6TmU8BHwZeAZzD6D+DDxT77YmIvRGxt189HmuMmTtTOX9ELDBy/I9l5mcBMvPRzBzkaAXkI8B5k/bNzCszc3dm7u6JuufGmPmyqvPHKArmKuCezPzgiu07V7ztLcBdG2+eMWazmOZWfAHwduDOiLh9vO29wNsi4hxGqtSDwG+ux5BKzgPoFlKaUrWGQsoJEak2EF9NKgu7Sl8RJa2UjUMhOfaExJZF7rwlJWEKiW0oohxDRB5WotNQyJtRhaoxigetUNGd5Nrz40mhVVx0Q9HWFW1LWUWLzhYlOC3TrPZ/pRhrFU3fGHM04yf8jGkUO78xjWLnN6ZR7PzGNIqd35hGmetTN0HQqZI0KrmpStAopKGuSAaZKqpPJZgs2oZKzhNtUqJSCUhF23CW6DehG5UJV6nSqo6oE26K4xISrDpmlZC1Sq6q5E11XobifqmEz75I/FnPlLiGi7hVdb6OxHd+YxrFzm9Mo9j5jWkUO78xjWLnN6ZR7PzGNMp8pb4IFnpFskshzQ0GSkSZjBLROiLCTQlYVZ8pJCptiUicGTIFadnSLT7PpXwlZK++iCDsiOPuFElGUySlVDJVlTwV6qhPGCWgnLhd1PcbiGjLvrh0xCXMQF5zk49bXsPFPK4l2s93fmMaxc5vTKPY+Y1pFDu/MY1i5zemUez8xjTK3HNpZyVhqcCsQvQQ6s/MEXNUUqTYS0WVicBDQkYeiv3qpvKoh0Iu1fXnhB2i/lxUdRTF7SaHKnKv3m8o7KjmsSMM6YuoT4WSI9WFsPb4R33tTIvv/MY0ip3fmEax8xvTKHZ+YxrFzm9Mo6y62h8RxwA3A9vG7/90Zl4RET8JfAI4EbgNeHtm1gnrDlMsLMsV885khSDEKrVa6x+I1WFZjqkIBumoskoiQKcjJY7ZAmCqPHgdEfxSxOAc7nGGFqCY466obZZFaS3QuRVTnc/iEhlMrmo2ahNBODI+R55PlTey2i6OWV3gUzLNnf8Q8HOZ+RpG5bgviojXAe8HPpSZu4AngHes3xxjzLxY1flzxA/Hfy6MfxL4OeDT4+3XAG/eFAuNMZvCVN/5I6I7rtC7H/gi8ADwZOaPy4s+DJy+OSYaYzaDqZw/MweZeQ7wEuA84OxJb5u0b0TsiYi9EbF3uV8nUDDGzJc1rfZn5pPAl4HXAcdHxOEFw5cAjxT7XJmZuzNzd5nFxxgzd1Z1/oh4cUQcP379AuCNwD3Al4BfHb/tUuCGzTLSGLPxTBPYsxO4JkaaVQe4PjP/IiK+CXwiIv4j8HXgqlV7ig7dxW0Tm8oyXkDEZH1lOKiVxf6gyt62igwogjPKwJNOPY3bhMS20BMy4FqSsa2gDGiSgSCiRJksk6V0r0KeFXuoxsGwlsoGfSGjFTJxlWMQoCsmf6Bid1SAlIzUmjxeCBsXFiZfV7J02RGs6vyZeQfw2gnbv8Xo+78x5nmIn/AzplHs/MY0ip3fmEax8xvTKHZ+YxolciPCg6YdLOIx4KHxnycB35/b4DW249nYjmfzfLPj5Zn54mk6nKvzP2vgiL2ZuXtLBrcdtsN2+N9+Y1rFzm9Mo2yl81+5hWOvxHY8G9vxbP7B2rFl3/mNMVuL/+03plG2xPkj4qKI+H8RcX9EXL4VNozteDAi7oyI2yNi7xzHvToi9kfEXSu2nRgRX4yI+8a/T9giO94XEd8dz8ntEXHxHOx4aUR8KSLuiYi7I+Ld4+1znRNhx1znJCKOiYi/johvjO34D+PtPxkRt47n45MRsbiugTJzrj+MYj0fAM4EFoFvAK+ctx1jWx4ETtqCcS8EzgXuWrHtD4DLx68vB96/RXa8D/g3c56PncC549c7gL8BXjnvORF2zHVOGAU3bx+/XgBuZZRA53rgrePtfwa8cz3jbMWd/zzg/sz8Vo5SfX8CuGQL7NgyMvNm4PEjNl/CKBEqzCkhamHH3MnMfZl52/j104ySxZzOnOdE2DFXcsSmJ83dCuc/HfjbFX9vZfLPBL4QEV+LiD1bZMNhTsnMfTC6CIGTt9CWd0XEHeOvBZv+9WMlEXEGo/wRt7KFc3KEHTDnOZlH0tytcP5JqUa2SnK4IDPPBf4F8FsRceEW2XE08WHgFYxqNOwDPjCvgSNiO/AZ4D2Z+dS8xp3CjrnPSa4jae60bIXzPwy8dMXfZfLPzSYzHxn/3g98jq3NTPRoROwEGP/evxVGZOaj4wtvCHyEOc1JRCwwcriPZeZnx5vnPieT7NiqORmPveakudOyFc7/VWDXeOVyEXgrcOO8jYiI4yJix+HXwC8Ad+m9NpUbGSVChS1MiHrY2ca8hTnMSYwSz10F3JOZH1zRNNc5qeyY95zMLWnuvFYwj1jNvJjRSuoDwO9skQ1nMlIavgHcPU87gI8z+vdxmdF/Qu8AfgK4Cbhv/PvELbLjOuBO4A5GzrdzDna8ntG/sHcAt49/Lp73nAg75jonwD9mlBT3DkYfNL+74pr9a+B+4FPAtvWM4yf8jGkUP+FnTKPY+Y1pFDu/MY1i5zemUez8xjSKnd+YRrHzG9Modn5jGuX/A0/nRyHULljIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "im = ax.imshow(validation['features'][4400])\n",
    "ax.text(0, 1, validation['labels'][4400], fontsize=20, horizontalalignment='center', verticalalignment='center', transform=ax.transAxes, bbox=dict(facecolor='red', alpha=0.5))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class_id\n",
       "0      30\n",
       "1     240\n",
       "2     240\n",
       "3     150\n",
       "4     210\n",
       "5     210\n",
       "6      60\n",
       "7     150\n",
       "8     150\n",
       "9     150\n",
       "10    210\n",
       "11    150\n",
       "12    210\n",
       "13    240\n",
       "14     90\n",
       "15     90\n",
       "16     60\n",
       "17    120\n",
       "18    120\n",
       "19     30\n",
       "20     60\n",
       "21     60\n",
       "22     60\n",
       "23     60\n",
       "24     30\n",
       "25    150\n",
       "26     60\n",
       "27     30\n",
       "28     60\n",
       "29     30\n",
       "30     60\n",
       "31     90\n",
       "32     30\n",
       "33     90\n",
       "34     60\n",
       "35    120\n",
       "36     60\n",
       "37     30\n",
       "38    210\n",
       "39     30\n",
       "40     60\n",
       "41     30\n",
       "42     30\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_df = pd.DataFrame(validation['labels'], columns=['class_id'])\n",
    "labels_df.groupby(['class_id']).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "def make_indexes(tuple_index):\n",
    "    rows, columns = tuple_index\n",
    "    for index in product(range(rows), range(columns)):\n",
    "        yield index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_list(img_indexes, shape):\n",
    "    try:\n",
    "        grid = np.array(img_indexes).reshape(shape)\n",
    "    except ValueError:\n",
    "        grid = np.array(img_indexes + [-1]).reshape(shape)\n",
    "    fig, axes = plt.subplots(*shape,)\n",
    "    for index in make_indexes(shape):\n",
    "        img_index  = grid[index]\n",
    "        img = validation['features'][img_index]\n",
    "        axes[index].set_title(validation['labels'][img_index])\n",
    "        axes[index].imshow(img)\n",
    "        axes[index].set_axis_off()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_list = labels_df.groupby(['class_id']).apply(lambda x: x.sample()).index.tolist()\n",
    "sample_index = [x[1] for x in sample_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAEICAYAAADGG5iAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztvVmQXNd55/mde3Nfai8UqlAACguxEAABLqK4EyZFa5c8tlrtluW2O0Ztt92O6ZnoefBDd8TEhCMm5mEeZmJm3GFZY7dlm7ZkW5ZkUhQlUdx3kSBB7FsVUKgq1J6Ve+Zd5kHq8//fNEAUqWIhqPv9IhjxMZF5782b5546//NtJgxDURRFiRPOjb4ARVGU9UYnPkVRYodOfIqixA6d+BRFiR068SmKEjt04lMUJXboxKcoSuzQiW+VGGP6jDHfMsZUjTETxpgv3ehrUpSfF2PMHxhjXjfGNI0xf36jr2e9SNzoC/gQ8f+ISEtEhkTkkIg8Zox5KwzDYzf2shTl52JKRP5IRD4uItkbfC3rhtHMjetjjMmLyJKI7A/D8PTPXvu6iFwOw/APb+jFKcoaYIz5IxEZDcPwt2/0tawHKnVXxy4R8f/bpPcz3hKRfTfoehRF+TnQiW91FESk1PFaSUSKN+BaFEX5OdGJb3VURKSr47UuESnfgGtRFOXnRCe+1XFaRBLGmJvotYMioo4NRfkQohPfKgjDsCoi/yAi/6sxJm+MuVdEPi8iX7+xV6YoPx/GmIQxJiMiroi4xpiMMeYXPtpDJ77V8/vyU3f/rIg8KiK/p6Esyi8A/0lE6iLyhyLy5Z/Z/+mGXtE6oOEsiqLEDl3xKYoSO3TiUxQldujEpyhK7NCJT1GU2LGubusv/w9/aD0pYRDY1wNysBgHc3EmlbF2aDqcMH7bmqN7xqz9H3/rt6x95shRa3/zOz+09kxp2dqOa2C3G9aeO3vC2ufHz1u74fmRy8hnkdedH9hg7ds/8Qlr/89f+ry1X/r616z99ilkwAX0vZst3JtWu2Xt0I3+XJsP3m/toZ1D1u5LI6Fk04ZN1u7t6rb2Tfv34IsrPxf1dmAH5+SFGfv6speydm7hgrXPP/pfrR1UK5FjDTzwsLU3/vLnrO3TM9JuYUy889rL1j5+/qy1t2/dau1kGtexbQyhqI99+1vW3nVof+Q6btl7yNqLswvWnl+atfbo2DZrF3MYc5XxCbz/1eet3Vyctvbgrcj2PPQZPB8iIo1U3tq1WhOf9/BclGo1XPumAWv35lKrGte64lMUJXboxKcoSuxYV6k71J2zdspNwk7ALuTwnnwme9X3iIgUkljRDu0YtnbWda29eROk5+c+ftjabYPPZrK4BW7gWfvIkz+w9t+XsdRvJNOR6+juwrI81z1o7Y9/9F5rm0rd2rOlqrVbLr6rR/ejHkBOey3URjC0vBcRaXn4u3Xgrs9Y++AoJG3C4D2hUXX7QZBJ4B730phLLSGV++yTr1o7oN/R7cZvJSLSdwAS09BYNrTFYmh7ptjTY+0NwxutvWXHDhyUtpVEYBd7kH4+2DcojJPAuYfoO/UP9eE9tD1jHEjx7AYcqziG62guQibP/OSItU9twDaNiMi2uw9buyeN59OjKaAuOF+SrnW16IpPUZTYoROfoiixY12l7n/8/X9rbdfB8jSRwGVkyAPFr/OyWkTECHljXYdsHHfj6Bay4eUSgWwoVyBHzrz2irUvXLxo7RxJmWzIskHEraxYO02et9f+GvUL3nDggZ5anMRnM73WDgO8h7MIXfJsO/QeERGXJEyrAo/0qWPwni2W4cEuN/Cezz78MVHWhoaP3yHjYLtk+vQ71q6fpRq29LvlN/G4FAngIJblCXwmQYPCa+Ec+Qa2TrZloAWblxCJUKXfXWi7Y0sPtmnq0xjvIiIXLo9bO0fPl9A2jMPPIF1f4NNWTQnPV+jBG91aXrL25HPPRs7dvxGS3clABg+M9Fu7HeIe4An8aeHM1aArPkVRYodOfIqixI51lbojw8PXfQ/LPJ+WzH5H4LDvI7AxoOV3zoGnypCkWFmEZ/bkUQRVPvO971v7yBsIeF65CLkYNLFENx1S1yev6RJJgtrJU/gMvW4o4DnZDZmS7sYi3c0jGJQ9eE6HUzZsw1scepAOr7yIwNTXXsZ3KpUhClTqrh2VOrYgGgv4TRfffNPa3gq2HDh4vzyBwGYRkdKjf4bjNuD9TVLwepK9qRlEBgQUBbFCsrLUhNSl4SSGHraWH32+HN5uCVhm47gpei6yPuwkPRMJisbwaUvAb0OqliYuR8595PHHrT1w7ydxrF5I8wG6vplLV6w9smNEVoOu+BRFiR068SmKEjvWVeo++ig8nX6LpKoHqUCmNOotsiEhRESqdUjXgTEEWP7rL/0BPjMDD+q3/vxPrf3KC09be+78lLXbDZw84NRgXvY7HTnDDm4hSxjvGgVeTZ1yD0uQnrUsBUJvgPcq1QXp7nTEadbbuN5aC/byDO7N7AyCRqttnFtZOwaLCGpfXFq0duUKxpYXGVDQm16ZfZIcbyBSJ1lp+jAm+keRI+tTzvrS5Li1a1ewVdMkqZugKAaWnk2SzCIiKdpXSWcQWeB2YzyGWWzPtBy8x6f1VO8AvLJdRQRrJ+m5CSS6h2P6EO1gUjjH0YvYLsi0YQ+PIld3teiKT1GU2KETn6IosWNdpe7f/gVKMnGZKSNsk8eKvKGd2XgBfWazf6u1L44j6POlv0T5n6efQFmq0uwcjkPaIqJOyTPlpuCZSmYzwqQykDkJXrFTCR2vCYnpkcRnaRxSeaLaJLy1XheW/dkhSHoRkTpJ1zqVr3LS8O6FOUgk8eBJU9aOFknGriHIroNf/JfWfvFF5OoWs5CVfseOyJ4DN1ubx+nCKZRJmzmCY5UunLO2RznAHh3XoaBlQ95bHq4JE5XcAY3NGgcnU16sm8f3yAxB0vYfOGjtmz4Fr+yO/ShF5dJx2CsuInKFtpwG6Hk7cwXfb2wTzrchF82fXw264lMUJXboxKcoSuzQiU9RlNixrnt8DXbP0/5dMo99rGIPankZykxoN/BZEZGQkpSFsjq++dWvWvvkU09ZuzSPuna8r8dJ28kC9iy6R5AoPTA6au1sD65VRCSRwS1McYYG7ZE0VrB/tzKNMIPyNMqUN5awx9KiMJX2MkJT/CCaNZLuQ8T77EWU+6418b5qHddU8bH3p6wd3/nRC9b+1AN3WXvTbait11/GfmyN9n+vXMEYEBE5OIg9wrnvf9fa5578nrX9MpWr57YNtGuXoD00Q6FSSQpbMbR3HQ23EfHqePZatP/cbtL+8xKeqTKFZrUWENKTokwnQ/v6Xbuwl1lIoTCJiEgfFRoZ6saYnV9BKM4c1bVcmETI1qG90aIP10JXfIqixA6d+BRFiR3rKnWFEpYzvZCPyT7UzUuQ7A0oSbtejc7RPbSUz9QRxX32lWesvcLylpfyVLa+eyOuY2TfbmvnBlDSm8NZEonospwrB3AtQO7eVqCo+65RdD1rUuGEhbPokDV/AbK1TnX2fOoOJyJSOn3c2id+iOtqUviM08Y9zHd2qlPWBEP3+8g7b1v7jlshdffsHrP233znMWvv7kUpdxGRV//P/8PalynDSCgzh+vghSQTi9t2Wnvo1o/i9Z0Y19lePF8OfdZrR0NKaksYaxWqTTl/6pi1F84gxKa2gEIB7bl5a5997J+svXQJ4/qB3/331s7vQfiLiEgujfCUNsX7jPSieMe5ywj1mV2MZnWtBl3xKYoSO3TiUxQldpjwGsn0HwRf+OKv25P5OchbhyRwq43rKThwv9ZNNIl61zCWw9WjyMq49DaW314Lnw9JevZsG7P2lv0HrJ2lUtycWRIailbvqBSQSFKGB8lvh5KwXfIch9xBjZbxXhXL9eWzqOU3cwKR+fVydEkfkszOUuR8OwMJs9yCVG768IQ/8/Lb2nJtjbgwvWB/yDPvoNx8k7J6Dh1Ew+7Jd7BFMf3db0eONfkEPLlCNes4zSJLXcl2fOYL1t7yEJrYpwbxHp88q/y8h5T91DkPBDQ2uaBIYwl1HxdP4HtcIlk++/breH8J7w/pWem/Bfdjz6/8euTcN911v7WLJM0LaXz+6El4crO9yO64ebRXG4oriqJcDZ34FEWJHevq1eWThQ14ihIUUOwYkpuCZfLAwObIsVLLCKS8OIG6e36bkrBZCg4iIHlkL5bZiQKWyUslBF7OzMGrFSQhHfsGoo2Xh6gpc4a7vZEXeW4eHqjLV+D9atLfnb5BHHdgC5owD5A378rpaJnyZhX3oD6Pa3e4nllARRG8aBC4sjY886MfW7tchRc9oCISWQoc7rqMIPYrz2KbRkQkJHnLmi09guiDQ7/zP1l7030P4P00Tn0afyFJVS5TwfHwTscayFC3OIeaeqcG4IUeuPMj1s6N4vnMUaTExacReF2dQ7D2wtvYEjjh/F3k3KlueG+33HqPtRMuntUSbQ0FRf5W0QSDa6ErPkVRYodOfIqixI51lbqZADIvrCD/z6EOUlwHz6PFfspEWwXPX4S8ba5g2cszuclBXmzYvQvHolLh0xfHrf36W1h+L69AHiTIO9c3jGW8iMj+g6gxtmUQsndhElL+hZfg5VqhgOQkBZB2bUCtvT0377H26GZ4vwsLkP4iIi3KX47U/KvgfoR0a4OOIFVlbdhGja7LM/gdnnsTwb5CWw4b3sB48Cj/VETEoWgCh2ox7v/X/87am+4/bO1F2p6ZnEGpey7n3t+H/N88BQc71K0tmYzWmeQtowtU4/IkRRy0A0jMUWqMPnwY1ycU6XD+CXiwm4vwyi4eh3dYRGTylRdx7ZsRfF2rYxusVcWzMD1H9fjGolti10JXfIqixA6d+BRFiR3r69VN4XROBS6lNjUHl5BK62QhHXNBNHB4fhZL/IDKUoUklYuDkI+FQciGOnVsO3XqjLUbHq7vwC0IbE76kCOXp+CRExEZH0f+YSaE5/jkEcjmlkBGfPSe26yd8vC9z51FoPLFC5Dxvfvg4S1sijZLriwiF9kvw8MrJHuTWeQcDwysrtmy8t544KFfsjYHBd95GK9f+PGz1n7zLKTjP0sgoHzv4Xvx+c0kb4+8/oq1/+TP/8Tal8lbnClga2jrTTdZ+xMf/7S1b6cxHnR4/F97CTnvf/ynf2ztZSpFVcjhHMUBPF/3P/KItT96x53WHlpADu/k009Y26foBBGRS88/Z+2NB/G89O6/w9orDTzDO8a0y5qiKMp10YlPUZTYsa5SN58mz1EKdmSxT56mdBeqr6a9aJ5qm5qNszx2Evh8YRDetmQa+cAN8nr29uM922+Gx3bzVtg+NWGuUsNoEZGlRXiXSj243nodHq/BrZCrW3dSCS7qjLY8C5kyXcb56hRkunFDNHg6W7xk7VaVyldRTm5XDwJOH/6NfyvK2hNSsHCZqiv3DECC+fOIaPAookE6mmm7BTTd3vrgwzhuFWPia1+D9LwwhaDgz3/ic9bOUn74Cy9DZv/jP37T2r192AYZykWjJr7zj/9g7So9a7/92/Au9+cRaPz0j39g7R9///vWHvlNbK9svetea5eopNXyhZORc9eoSvnZp1FFfTiDexNQx8BiLprHvxp0xacoSuzQiU9RlNixrlL3i5//DZyYclBzJIFTOW6GAtnw2o+wfBYRmfCv3hzbSSCYMUUNj3mKL3RhiX/gdjQjT1OlZVLcstCAjFwiSSkiEvbiHGnK1U3QAQzJby4llUjh9VQK3rywhu8WcJBpISpH8kX8f5W83Fz6KqzBC/fy4/+ftb/yb6KlgJT3z7kp5GIvUw55Po/xtEDNoAKfu11FyW4ctnb3tu3WXqphe2bvHuSaf/Zf/Ka1H3jgQWv7TcjTTBLX8ZffQV7seaqsXBjFFoyISIm2cHYfQCXpBx+B/M7SsxbS8/jVv/hTa09cGrf2vrsPW3vgZniUVy6dj5y73aTn7TRkcM8h3MNxqq6+UIbc3/XlL8tq0BWfoiixQyc+RVFix7pK3ZF9aCpSp7IyPnnClivwkE2dRw7f8UvRwOEmeXV8coy5CXwlk6S8RPJycU/fMOJVw2frS/C8vU05vAutqMS+Y9s2a5NTl3sQSUjSxlDAKldzdpI4d0A9g0Oqnut2NDpK0LaAQ99JWEo1cazlaUgyZe145mkE+87OY9zs2I78VYd74cq1q55n+hF0n8ghN3W4Hx79r/zef7C2yz1pKc/Xo/631SZKZdVo2yZBOfLpFOW7ikiRzt0OqM8zbaMUuaLyEDzYKTruwiJFQWSxpdW7C/noieeipbnai7j2xjyCnt0yrn2O5oNqiO+3WnTFpyhK7NCJT1GU2LGuUvev/+z/tvb4KcjYSg1L2CDEXMyVjzMdgZ7BNVqKcO6jF1y9yUqk9Cw1EuKKxkfeOmLt87MoobPrALxRIiLbtyA/NyyhT64xrHW5cRG9zj15SfZG0zfJO2w6/k7R5yOSnQK6g4Dfo3/nPgjOHUcv3a4+9E0e6kH+6iwFOUfaG3eMY5ckZ2Ss0DZHwuUmQZCeZ44dtfaj33jU2i+9SLmvm8esvYf68KZS0amgqwvBwpMleFCbDTwjYQG59IUuyqvPQNKuLOOzLXructQwKVVAILSISINKVnETroVLqEBeKOA+jWzEPV8t+iQoihI7dOJTFCV2rKvUrVMjlSrlyy6UkIfIlWMT1KAlzEbz8YKIzKOlvwcPlF+HhGapyx7eFnm5Th5DxdwTFNy5eTvK+hzYhbxbEZEcSYQ6yVWWugEt8Q15aR2SriYidSkAmXuiSjTw1ScPM5+Ddgsith9cO3BWef/cfg/6wGYNgoXDBI3lPLykkV2JjrJUbQpU9rm8GH1crjE+atUVa6doPGwZRFB0i3T26YsIHL5tNyqJi4j0dCPI/8w5SM9KDVLX0JZKliIMivRdyyU07Wq08awVsgiBcJNRjzLtCkSaLyXofm4fQ0X1N49A4gtyJN4VXfEpihI7dOJTFCV26MSnKErsWNc9Pg6nCHwI+YB0fGhwSV4dexztRHQfgCPWjaHCAT72+Jq8d9iG+5yTHCbOoQz4sbPj1h7ZClf/nYeQcVLMdtyykENjsAfh0n6ER9/P8/C9fYPPNqgBWpuyUjzqZNWiPR8RkcYK9nQ8ytbgnTyqZC7td8kYUN4/FWqUfewSEuk3bkGHsMFeanRN2UVhR+e72hzq9tUo6yHTh7qRvC3IYUy7Dt1u7e27UMhg/Bgyj776tf9i7Se//xjePxrtTtZL19umvfIVqk3pC2UVUZe2LgpPuVBGiFezif3LPH3W96LZUGH0C+I6aIzv3I+wsmKXlp5XFEW5LjrxKYoSO9ZZ6lIICi9naanrUgPiFK1zHS8aipFIQ+p69BlD76vPIym/XkYJ7GYTx700gWRnJ4Xo8/5BZGTML0NyzME7/7PrxS0sUH297h7UyptcxHL/5AmEEKQMvveJC5BIqQJCCbrpe7ZKkLYiIhWKivc5G4XvByWSeybaqU5ZG+5++BPWHnrnJ9a+NIME+1oe44GLD7SXo03im1SifvYEwjS6tqLgAYeCXab39/RjO2egG5kXY/sQqjI6hNCW1y8gE6JWi7Z26KWy+dyIfuYKZVXsplATl7d5MM7qFP7SaqBGYLOM87Uq0XFtWN9S2JqbRsjMmQt4jiItLVaJrvgURYkdOvEpihI71lXqTlzB0rgSYHk6PIAl+pY+eJMy1InNC6IeySsLkI9zJdQ680nqtleoA9o0luiZQXidOJshqOI4J19/1dqGot07E/2z5G27/Q54mjZvR9nwegMdpSaOw8NWbcBL6+YQyX7gZvIGUoT77KlTkXPXqQx+QPIgkYYH3KfS9e16tHGzsja0ybM/Pg/ZdvooihcMdGMbZWiYMik6pG5IXtOZl5639saDaKZ96hzGwZ/97V9ae/ft91j74QfQjLxBWz5HKYohVcRzUMjj+kRExnZg/G7owb8999ST1k5S/ECLChn8hIo29JNEL2Qh90tv4T3NcvQeMG4GY3kTNUbvPcAtI977Fo6u+BRFiR068SmKEjvWVeqevAB52peE57K/C57LQhGeGz+AFPRa1EBcROpcq6xANd+pu1RAnylfHLe2SWIZz0v6sWtct3G4/ln0b0WaPLBDPbCT9JmdN+Na83OQHS0fEmloGF7nLVSrrDmNpuHzE7BFRAKS9S7dj1QX7mEjTUGmokUKPgi++TfoXneCtiOyCUjJFCXuD5L8M+PwrIqIeDWUUV96501rX34J5e1HD0H23rIVrQ+ee+I71n79KZRzL5UgJZvUxP43f/WL1t7QFw0C9qjowCc/8WlrP/7Ed639F/8Fjc2rdVx3gYKfP/PJz1i7SyBJL7zxirX9ZkfnQrKTVHK/sBGd4DZuwPVu6qfnf5Xoik9RlNihE5+iKLFjXaXu/bdg2VqeQwDiBWoofOoS5e2+S25pi+qe5VNYlrsZLLlNlT288DqtTKDW3uDevdbuHaFy2DnIb+5g5tB5RUQSlDPskHevQbXRGjV46pbLsHM9uB9dFODamEVQ9UUqgd+uRINMOdAzkaflPsmqqovPONF+5MoaceXtt6ydFYyPTaMYT8MkJR2D3724czJyrKWjCFpuk0S98Pi3rb2nF5EEv/pFFKDbRU26T58+Y22Pljf7DyKf9+Ctt1k7mYh6Rl2Sunc9hCbim7aNWXv8/Li1uRXENvK+bhrCFs7kDx+39hx5frlGpYhISLUpB/fdYu3RnThuSDn5vmDsR5/Oa6MrPkVRYodOfIqixI51lbpOAd4bfxnepQZ5mrw2Nd/mMk9udCnO1aECmr6TvX04hw8vckB5gg3Knb3yDsrN+1XI5MExeJASaZx7qQmpKiKSp25UxRwuivMPx8/BczdNAa7ZJdyPfIW6V03j/e1lBFWbDuXvUhPyJHnSFgMca6WOrYPsAK5VWTuGbr/L2qaGH2nbGNoUhNQkfmER+bX53dGufalZ/FtrhkpUTZyz9ulv/Fdr7/61L1v7nvsOW/vujyF/2HC+K+Vr+1TyzA+jHv+QohJS1Ah8jErUb7npZvoAjmWoDNv0yy/gur/9DWtz97TO8vvJIWwFdO9DSbhTl1D+a+eOUWu/n9WbrvgURYkdOvEpihI71lXqNnzUdCoMQJI+MoLl80AfPGFBAW7IdD5aeiaboQrM9DVqlxDk+9IPEcQ5SZ7cNuXIclmgueNYftep/E5xI3IrM/0IUhYRESrHQ/3IpXwFcrqxBHnscBe4RQQzL5Bd8KlMF50qdKM/V2pgg7WbebyzQRWph0cQLHvXx35VlLWnP4+x/OZFeCvH9iKf1LQxBrhbXrknGji8+TAk6pUnESzsUTXmylnk2x7/+p9YuzSJ0mZbHnjI2oURNNw2PIRYYQYdnc25zFmk7RltRXF1ZOqmduk5PHenvgV5y88Ud4eTdLS6+tDd6FpXpe6K3QG2q3I0H8wsYvtoyyq3c3TFpyhK7NCJT1GU2LGuUndsL4IRH/zoL1t7+yikLjcqSZEnNzTRpTj/H6/Y56YQELrYgHRd8bBMXp5CgHBIDY1Cai5enoJkrs5OWbuz+XGSA5ipBFSTjlWkclDcFt2QbMjQN+KvGroIyUxugLdLRKTWjZ9vwYf3t00etnmSFz95Ch42+Y2viLI2TNH42NCFrZABypOerEMKLtax3eF2KMzW3jutvSWFsTb5xLfwnjlUdq5fxjg9//d/Ze05yoUdpCZE/XvhRU4PYlvJyXQ09Sa7TWO5voRzL5PkvvzSs9aePYYgbJ8qMPODGqYwdkfuezBy7tF7INOFKkY/eBc8vJkEnrWJSYxxEZW6iqIoV0UnPkVRYse6St2FCpak3332e9Ye2wUZsGsUQZ9JkqGtarR0TUCBzm0KmCytwHtW8pHDV01AMja7EOybSEAOG+ofarjXJ53LbyPIWUQkoEBMboxCbYMlRTaXj+Jc5JDygU2RZEcWHsOFVDSncYkq1zY5GLUJO0mSYLYGaaKsHYtTkLqpHuTRzszDs18iz2OWgogbFXhrRUQuVyElP/8rv2LtVgh5PP8iZGXzEqIVfGoYtEy5sCtnUSprgraSHJLlJhst7cThzA0a441FXB9Xj/a55zMFJPOINRl4YkfuP2ztB37/f4yce3AjJPiLlxG07NPzMr2A+/mDl16z9u9sRhmsd0NXfIqixA6d+BRFiR3rKnUnz6FUTtNDKapz1IDniRUssr1lSM9EuyNRlQIuHVoCN6maa6UJj1Kd5HCD7FQGS/wCLfez1EvUkFcroM+KiDjcrIj+jvgU9Ok6WPCHSQq8zsIO6NwrJJmXW1jS19rRfEpDf7YCH15rh3rxZqmidTLz3ivVKtdnZGyXtYcGESx8+gQqKFNQgSwsYstnrox8XBGR1ttocjVx8g1rD+YgUTfci0ZCfZTDu3IKTa3alI/utShgfwG2z9XAvei49nzaSqJg44BkLDukI85pisbIbERZqq0f+5S1933u89a+6QCaa4mI+A2qTN4PmxtnZakR2WgvqlCvFl3xKYoSO3TiUxQldujEpyhK7FjfIgUUkhLSnl25Bre4G2B/IE1ZEYlENLI8Qe9zad/BoS5olRptrCSwZ5dO4/11iiyvtfBZ18H5shQCkHaifyuStMdnyHcf0qZHi8LzwwRueZP2CxdLCMNptCl0h2ppJ9PRnyvh4P9D2mWJBL3QfqHxtcvaB8HvfOW3rF2hNgOPGezP3r4PZd6PvIq9u5mZaOe8Xqr9ePRthKS8TVlIXUMoTjFAISJD9yC5fwsNggo1IK9OI2uptYLrK7fwfIiICIen0PrIy6GtQZJCsLr6EcYzcAu+600PoejC5gPI3MoNIkwrk4wWjK838ExuKmJfOkPP3iI98yMjOPdq0RWfoiixQyc+RVFix/pKXUp6cKjeVyJDJdSzWPb6Hl73VqLu9gxJ2pBq3Hkk+TzqHFXsQ1hHi0qzNyhspb6C11s+5GYyATlhgmhYTZs+H0RKaOP7OSRvuY5gi5b09Ra+n5PEd0tTyEvYkdHOZ8tQCAtfRr2E4gXVMFo2X1kburoh/xpVZGKUl/D7lmlcfvReSFL+3UVEsg5tyaQgB2frCE/l9LWgAAAgAElEQVQ5/vbL1j49ddnaKcrEuOP2+6y96dBHcMx92P7JUBZGQJ39RESKtI2SJCnqFbHt090Hyd29EaXgE1Qncv9991i7v4tqapJs7ajTIItL6IjYoLGcFJw7n8bnh7ZHi3esBl3xKYoSO3TiUxQldqyr1G1SsnMih+Vz4EKm1atYiidpOZvORy/VVPFv+QyW+MUUvEAp6oBWC+GlqsxTx7UaFtotnzyjPmdb4FxBGJUmLfLMBlyum5t906uNJt7TJpnjk3fYpYbKDguB6KnF57r0SfxPks5drWF/odVZXlxZE+YpeiBMQp7u3opOfS3yku48uMfaJ6klgohIUKHIhzxtf5Tx+lAvpF2NOgMuUkbIc0+hefeefSiB302NwmeodmWlFm1Wn6AOftkCpPyGAj5/ahxZI7cUkKGxdyfK6Z8+j+9Xpu2jO25GtkbSRJ/tY+PwdPdSwYJjU9hGGNuAZ7tN90ayuNZ3Q1d8iqLEDp34FEWJHesqdZNUbpo9oO02SUQKsm1SfbwgiAYwZx2SyhzISx7eFsm8lSqCNStl8t7SuYOAr4mCfRMkQztqJXDTc27czDj8OpffpioD3C89JMnMAdlu0BGAHKntR9eRhJ2moOdEsK4/d2xI0+8w3YA83bsHgbyvv/W8tb8xjW5ot38UpeZFRHaPoQ1DlrYmnn7uOWtPkSfXo/GeooB/z4UHdWDDRhyTWqPVyAs8SCXeRUQcel+LnsnNW3B9zTPHrO0LtVegIOfeDah9WT1x0trfe/Jpa2/btDly7neOo9jCAerAxgHd5T178T2uoJ7nlodRZv/d0BWfoiixQyc+RVFix7pqH+4h7JGUDB1qoJ3A8r5RxTK+2eFNTZCXt9WCpE1RzqGhEzpUmt0jD1uzRrXK6LPsoHU9XKvT+afiGu3eDP2D75GEpu9BFcgjndUc8upGPutF8ymNi2OFVGKeE4U9rh/YcQ+VteGti/Cm7hkoWLtCWzB334E81QvnL1j74iQ8qyIih7YgEHjvfpJzLiRfkMLve/ES6lryA8Yd/Bzq1kapwDI6hjp2wyPwQIuIZGh7ZuIypHmhG/LYoVzdsa1oGTFJXecKI5C6+3aibuHJ78DrXFpEwLKIyNnXX7f2mePHrV1tQdJyQvoK1R58RKWuoijK1dGJT1GU2GHCMLz+uxRFUX6B0BWfoiixQyc+RVFih058iqLEDp34FEWJHTrxKYoSO3TiWwXGmLQx5mvGmAljTNkY86Yx5pM3+roUZS0wxvylMWbaGLNijDltjPnKjb6mDxqd+FZHQkQuiciDItItIv9ZRL5hjBm7gdekKGvF/yYiY2EYdonI50Tkj4wxq0uB+JCiE98qCMOwGobh/xKG4XgYhkEYhv8kIhdE5Bd6cCjxIAzDY2FoK/WGP/tvx7t85EOPTnzvA2PMkIjsEpFj13uvonwYMMb8v8aYmoicFJFpEXn8Oh/5UKOZG+8RY0xSRL4nIufCMPzdG309irJWGGNcEblbRA6LyP8ehmH73T/x4UVXfO8BY4wjIl8XkZaI/MENvhxFWVPCMPTDMHxeREZF5Pdu9PV8kGhJ3lViflpq+WsiMiQin/pF/muoxJ6E6B6f8jP+WET2ishnwzCsX+/NivJhwBizwRjz68aYgjHGNcZ8XET+lYg8daOv7YNE9/hWgTFmq4iMi0hTok0efzcMw7+6IRelKGuAMWZQRP5ORA7KTxdCEyLyf4Vh+NUbemEfMDrxKYoSO1TqKooSO3TiUxQldujEpyhK7NCJT1GU2LGucXwP/cuD1pPSaqF9HXV+lCT1jdvWn7N2d60VOdaJmbK1a6NojVcvrVg7wecgJ86AU7X2fX3d1t697yPWPvxv/j3eP4q2eD+NXQaXT79l7ef/Fg5eZyPa8H3nxR9aewpd/+Thhz5n7Y/sQfvB7Mgma4/PX8Znj70ROffLT/2DtZeXrlh7eOOItTdv2oDXdwxb+z986avcGFP5Ofh3//2/soOruxu/Xa2yaO00PWnVJgb85Rr1SRSRcgM/S9Ykrd3fhzaNg8P4fbt6+q2dTBfx2SxsJ0EtSn20U11eQVvHlk8DU0QS1OI0R71P09SqMkxi3ZSh5/bs8aPWPn/htLWHNvVZu2cE4/Lll49Ezl2fxX3rymSsXUilcO6Aw2jx/R7/0ZOrGte64lMUJXboxKcoSuxYV6mbNFi2ttt5a48M4fW5hXFrn5y6aG2nEV2K9/aj4/yujYPWPjM/j/M5WWvXE1gBF2jJPW0w95dm8dnbW1i6DwjHOs4LEwanrL28+KK1Z5YgCS5OTOM9ddzyvz79dWv/PcmUkY34bv29+A7zC5ci5+53cY4v7L3D2u0k5PgrC5ANlZUKPvwlUdaIdh3bLleqE/iHEGPIZPFbZZKQsBsysEVEEm2SpaSCSW2KQ8dN+PitFy9BVhayGDeZNLaM2h6eI+ORXGxDAouI1JqQwYuVMv0Lri+VwlhO0fOVDGB39xSsnSvgHvQk8Z6tvdhuEhFZ8ugcDp5Pl+5Bs351qbtadMWnKErs0IlPUZTYsa5St7oMr4znw7NVyOMyjAvv1dF3yOvkYekuIlKexdL8ysw71k7TVJ4qYDk8NILjjt2Epf+VKUjB6fmGtRtt/pvAjqJoURY/hNS4slSDbUgGJOGFK7aWrN3VDdu4kDwHRuF93T6M79ncju8gItLnQiIc7sW/VZK4jraHbQDxonJGWRtqVYwb4+B3b7fwO7CUTKextSMh5J+ISMLDuPPIsxpZo7AEDuh/msvWnJ07a22/CSno+Xh/SOEUxnQ4Q1lWkiT2yc4l8V0bJHXrbdyPFslhcTBeM+yxzkEOi4j49Nw6BtLcb+IeNkK87rWjkRarQVd8iqLEDp34FEWJHesqdSl2UlokGSen4fX0q5CeUsayvN3kalAiC20s6xMu5u9iAZK4pw+e4z0kp7eGOFalBjlSqpKMjTqRiWjAaTsFOTLpQsKcnZizdssbsHYyj+t7+L6brb25MGTtfbcftnZvHwKy20vnI+eePgEP8/EANzfX32XtrRl4sMM6JIiydpRquK9+CMkXBpBgHsnKRJPkmxsdT80AY4jUoAQB5KpPwfge2csNeO0Xl+iZYqnbpueIaummTXQNFNDJWySD8ym8nqKxH5L0rNUwZhttvGf2yqy1exLYbqrUouUtV8rwIjsO3TeS9QHdT942Wy264lMUJXboxKcoSuxYV6nrJJAjKz6W36VlClhsYwmcJNdSyo96bloUtBgaLKedJKSCR/mHE2cv4LNLeM9iBcepN3COIGCtywGSUWnC3i9fIAMqy5DiyTSW8ltuQive3v7D1v7kx/dbeyE8Z+2pGu7Zzq17I6dutCatffrsGWtvyuEe7hhD64REhwdRWRuaJB89knysHh3aqklS8G7bj3pTfVqLJBKUm3qNgsEeBTNzDvBKFbbj0ZYRByr72OapeVG56CbhaU3kYXv0vloN0tol722CEgecEM9mJoGtpzJd63KH1K3UcV0u3U+XvmtA8jYMNIBZURTluujEpyhK7Fhfr24Sy96kwM5ncRmlGry6bSo9k3WjkqC3D4G5PpeuyUEelMoo1TS/AvvULAU/puEBzeaQL8uBydEA5qIw+QTKEO3bsNvaG2+HN7UewptVSSBo+c1Tr1m7bzMkbWonzrechTxI03cTEWkOURmiCuwLE5C9hryM23chn1dZOziol+KXhVRax9YMBRGbDo+kQ1KZpB3n/UZfxnMUeBTw7MN2SSb7tP3TpIDgf1aWisZ/zsX7auThpapvEQ+vYXc0jT9y0Eq2H2Wp3CVsC4mI+NOX6X9wXW2WurQVpVJXURRlFejEpyhK7NCJT1GU2LGue3y79h60dq6N/aqUj32v+SL2B+auIPuhuhRNsN/Uiz2+viLCNy4uI2J9vAQ3eeDhq3Z1YT9iZy+u4/a9KGE/3Mt10mgPIegXZjCNkvFf/jWUq/cKiOb36q9b+/QZ2O0C9u9G92OvsZTAuZs1FGqYnz4RObdD4TfpAXyPRgmbKU8/+yNrv/AKSoL/5z/8tChrQ5uS8t0Q+80+7T05FNsS0l4Vl4UXETG8/xfyPhZtkFF4Sn0FY8CnMvYeh7MEeA68BkJFvBaeKT/sCNNqIlSl3uDCHHiO0lRjsNXEXp4xOK5L5eID2gtNUr3AZI6KNkj0ewsXYaC99pDXbKbj2leBrvgURYkdOvEpihI71lXq3vnRPdb2JyBvi5TFMdeP5ex4GsvhmRQSn0VEbhlCuMjQCkJgFsuQx2kqud03hJp49+5BCMrHBtENbRNlfeR/8ANrnzzxp9be2hWtiVe5iHLwhTyW/u0dKExQuHszrnX3L1n7xCzkxNkz+H61ypS1Q4pir3fUHauTS99s22rtRgaR9o0AUmPhEmXOKGuG72P94FOxiJBKsBsqoR5QiIbjR6VuUriTGMetwKyuIPzDr89YuzyNsvetRTwHngcpHrQhe03AWRGdISEkgx3aMuIsKcqy8ilzw6fvmiGJz2E/KQpty2VwHJHofYtGqtDWQeQT771hoK74FEWJHTrxKYoSO9ZV6r7+Bhprb0lAqk7Noc5cNzXT3r4Tjbz7ctHuZldmEN09XYdMdLogVw/kUcL9oZvgfd3JFb3fOonjLOMcMwZZFQFJ4IV8tAR+ljxQzhwliV+ALHVewlJ8w9A2azeS6KZWCeCNDnvhyc0OwHudduAFFhGZuIhiBt09uG8buhAVf3kJJciDJu65spZAqgX+1YtbhPw61YP0gmjGROhA6rqk8+pVjKdmCdtE5TnI28XLKFrhtckTSzI2QTKUMyw8lx+KjqIIVBAgCCCb2xQ14ZD31qUtqoRDhTGowEHaocbp6ei49qmwQci19gyuib3f5n2s33TFpyhK7NCJT1GU2LGuUndlBUvVKyE8muV5lJrObMGS+d774AWuj6PIgIjItx7H/49TZ7ahbnRyenAHJN+uOjyl3ilIxGwFUmOwD57Rgf2ofZfaBkmaHoOHVkQinZ5bS/Aue2UUJmhOvmnt4DKuozKDhukjSXwHswcl6fPbSBZ50dLzbfJgv/ECpPnYFsjp4T5I4IlIY2hlzQgpOJkDjdkVS+OEA5tDE/WmJugzHBi9wgUFljH2K7MYQw51IHdpe6bYiy2OLHU0c6nbW5CNbuFwEQCvgme1UaJWCFVI7iYFPLNEb1JgdLOMIP1WFds5XjNaj4/rawaGPb7cGoI/o6XnFUVRrotOfIqixI51lboz5+C5XKR8Q+NiSd9NS/0fH3nG2tVXkWcqIhLUKM91EJL48BbIxE01SNrLp96yds5D7brCntusPfDwZ61d/PRDOFlPtOHxtUhFbAQe5y6+Yu25f3rM2ulXUTdveAEetnQLwc8bB+EVe/ni6cj56gnIk8lzkNBn3obHuzeJ+7SJgriVtSO8hh3JLQ3YI0lvcTvWHtTwW6hrWo2ClmsL2EZhr2cqh2D83pExa3cPooOfk8Q4M+TJ9TvqXUbyifuonl8dcrNVxVbL0uy4tRfncH2JOiT64gw80LPjp6zdbkfrTDokdRMRjy+du0G5+6FKXUVRlOuiE5+iKLFjXaVuaQZezzStbqmhkxgq01PsgZTbtHcscqylOVz65pvhge2bxjmuvAV56zVwki37f83a2z/5cZzvgZtwgm4KSqXzdhbAudYN9AUS1Wz5iLUHPwcPcTb1hrVbLz9v7YXpl6zdeAXl7OtNeJ1FRCaPjVu7MgcvmaRw7VVyis3PQ34ra0cQUn4uvc5BwIZt9vB2NMMOPBzLI1lZX0RwfYICgd0cpODGrfDmp/sRvN90eFuJbC6V1TGwfZLcnOvrUhn6kDzBxY1INmgFkNPVeYpiWMSzefqtIzhmHls7IiJpqgjXNwBPcJNKas1xkIeWnlcURbk+OvEpihI71lXqGio3k+RmyU3kIc69gyDdxCwub9sAlu4iIgfugDe2x0Eg5luvv2htZxbyr5s6jG24+0FrlwaRCzs1gRJTAXW/aoaQHH3D0esY60dwaNiGl2uxhe+0PI3Pb2xiHV98ECWqlpsL1jYvH7d2dgZB2LduhJQREZmjcj6vUKe6cABevBJ13pqfhdRQ1g4TCaClPFOStyGL4IgZ1ZjspW1RM3nOvXWolVv/CKqG54exjVKq4zhLJchk18HJe3swtoQ7o4lIqYRnZ5HkqoS4pkQaY7+YRGXy7gFch1dF0LzXgGd6YQrltNxcNIC5QdPSSgnPhUt5xrxF4BgtS6UoinJddOJTFCV2rKvUFYPTNdqwPQ/z79R5LKWrC1TmaTga5Lh7M7xZCyfHrV2eQf7gSBEyYOf98N4m98Lj+viP/sraF8nr2TOIIOdiP8698+YDkesY60EDpcbyy9auVJDTOH8WgcPzJ/D6vkfut3b3px+xdu3iE9YOF+C+6tq0P3Luw/s+Zu3mM/jM8VnIp+ocgmDbTfXqfhCEPmQiN8oJwmuENhs2OwKHKbA/aJGEptzZfC9kZWEAY6sWYpyWqpCPc3OQiz3d0WrH/41qvRL5/xkKNnY8kuwucuGr5IEOM5DDxT5cU66AraFyDdfUrmAryBhuZiTipDAfcNmuQK7uGQ/eu9LVFZ+iKPFDJz5FUWLHukpdL8SatFbHUndgAN6lm3ag7NOevRutnSwiD1FEJGjBe1YdRw6w4+EcPaM7rH3zvYetPZNELuE0RUIeuu0ea99+163WTqVxLi7rIyJSp/6l549hKd+VQ+Dl/v3IH27k4dmqLkOC5LZDEnTtPmTt2ivIaZxs4bMiIultkBSXX8DfsHMLkPsLFcq59KKSQlkb/JAa7QS4x9cMqw04iLhD6rap2nGkuRSOli5ibEkS2zZN8vyWF+DJTdLrfXnKfeUKz61o3+oGBS2P9qO0WTaHcTo5d8Ha1QqewRTlCRe7EZxcWcB7uJKzhJ15wiStI5HVfEfNNezVoSs+RVFih058iqLEjvX16ro4XS4DT9iWLQi4vevBO629cw8FQjpRiekfp2Dhy7B9+kqmB8HJiRy8UWXqRbqQQuXjV09CWg8PwN60HUvprr7oLZs/Cfl55ex2aw89iOZGxQHKaQwQbN0ykMYuBXSnd91r7cln3rF2rRntLVxMQHb4bcjxZgVyKUOevn7Ke1TWEMp5Da6Rk2siwcz00U6VRiWWTMBByxT8n8MY4qVLbQXytlXHFk6GgpZDysFtkQT2O/Jd+X8NVYkuFHDCbAVja75ETZPoO2VyeM7dBOUoN3Edkf7BEvV0h1d3hkduXKi5uoqiKNdHJz5FUWLHukrdVBZL4xzVnqnV4N18iyolz1CF10YzOkfvbiMQ029Rr08Hkq+VhcerQX046zTfO5vgRT59FPLgu//wqLVLTZSPuu0RqswsIl+49wFrJ1s4R6NJC/McXTutyt95Fj1v99y3z9rZ3ZDJXRsguRdXokGmxSqkSsFQL9MWSZMknbuF4HBlLcFvzfmkAUmwiAR+lyNFvJj0GYcqNUcqJ9PbSyXkYldr2ILxXTwr3iy2SwoDeB7ZGy0iYiLykYKqDSStSVJToIClJ3ut6QLpi4fXkLM/PVZ41X8Lo2/iq5X3iq74FEWJHTrxKYoSO3TiUxQldqzrHl+e9gQcKjFf87H3dOEKauLVqabdlnQ0FKOvC3tfK7TvwLHu3MGK3fjDRUST37UFITN3DqO0+6ceQu27E28hQ+LvvvndyHUEh7Hnt5LB7ayVqCBAi8IP8tiLq9M+YEj7gD7V/ystIcuklYw2fZ6+iGbSM/PU3Jn2SLpS2EsttaN1z5S1IaDMg4D26KJ7VbRfR+EhoRPd4HJCXovwPhiFi/DeIRU1KC2j9l02i+elvxv72PMryNCoX0H4Sz4fnQqStKOWoO9kaG8tQfuZkaIBkXuA8Bze+7v6DuJP8el9PhUpiIT+XGvvb5Xoik9RlNihE5+iKLFjXaVujsqjL5Qhbym4WzbmkcVx282oP/eRzejiJCIiEwiBOePgAE0PYndxAedoVCGzcwXI5EfupwTsDL0njXX16OhhayfC1yOXkUwjZCZByeNVKnsvVbzeroxbu3fgpLXdJuSw20B4Tl8BcsQZRh02EREvzUULcL4cLkkqywhfKIfvo3CZcl08krHcnYwlqYmka5DU7cha4LLyQrbvY/vDq2IbJZHHeA8oJKW7F8UIenpQW7IVYK1TKmF7xCSiayCXtkV8KoHvUI3AgOrrOQJJyuX0uWR+6JNcJzqlqk8vcEYJ917nDnGms0XcKtAVn6IosUMnPkVRYse6St10nrybK1g+dyXgrbxrA7ysYwUs0afcaBeoBSqt3aTGxq0FyL+lWRQjGJ9AKW03NWbt04vIhhg0sG+j050+Ae1Y6YnKzSQt65MrKL89GcB7tiNALcH5Ig7sHYDMNoOIol/6xj/hBCuQ9IV+anguIkt5FG4YSOK4fpG6VJFs2FBE0QZl7QivkWnAIs6QzW/xOuRf0qXugzTm/TbG08oKti/cbmyLpFzIynYbz0G5gsiAGsnkdApjt68nWgQkk4akXaZtqcULHHWBc/QW8Ww75MktL2MLhjvIJUlaO5Rt1YlDnuNr3Np/Vr5/NeiKT1GU2KETn6IosWN9vboFeDd9HxKuWcPyfvIi5OblWTQHP+/DFhHpaVPns37ymk5hKV5aQhGAi+dfsfbGIdTguzQ7bu2TE09be26BmigPoVvbgYMIBhURyXXhOnYcwHU89dST1n78MZTobm1CkPSmLtg7huDNvnwSTdUHDAKpEzmUxhcRKdch9xfnKeA0iaX/QD9kupt4P6GeyvXhwgIUpMtBx5EAX5K9nQX5SPU5FGXgt7h7W4Neh2e2kIUnd2UFwcylZUjdJNV97B9EIH+mQCXpRSRIQbqWfDyTy0vUiTCP8T7Yj3qXlUVcU2kB203s+TUGx08ko53fEiRd2x4X5rhGYYL3Max1xacoSuzQiU9RlNixvrm6RaoL1sZyfWoOS9iVEMv7rVvhhVyZw5JZREQcSLgr+T5r1xPkXSKP6PE3X7D2xu3oevaxPnhcF/0xa5cceNG27EQXuBGqlSciIkncwpGD8Lr+Ug++R7mB5XqiAJk90g+pm3rnVWuvXIFU2HEIjdBzu1GSXkTk2Lf/xtoLDXgAB0ZxPwpUyrw+i3qDytrBebgB56NyjiuVjmcvpOnQaZ5g3LkURJ/K4/etlvA7Oi4CmDcO77b2ht4xXIeLcyQpX95NUOiCG/Ws5pKQsZuzeA7HhiFXkwHGfpWiI8qzk9ZutSG5nQTOMTCK6I10F2wRkcoUurGFbRzXsNQ17O1Vr66iKMp10YlPUZTYsa5St1ZDMGPKpQDJFXgnK5TrWKFgy2QzWja9QvmozW4soYe7sWyuzqJsU+XMcWu/9iMECH/iv/uCte++41PW9rO/bO1sF+RzMhUtDcXepXQWnq3tew7J1UHwaf2Nx6x99BsodZ+i3ONsz15rnz4SLT3/+LPowNakMl2tLAW+luGdHtsa9Z4pa0OkAbZcPR81uJa8DaLvb/sYy4kMAvgdj7rwtShAfR7PlCMY78VBjIdkFsdxXTw4nO8adMhFl7zNnMabIClfXcLYqs6iTYRXQwl8drkm8vgOqf4ivV4UxszCCx3dFrhWE3HtsqYoinJddOJTFCV2rKvUbbcgaXuLkIwuNcNeaSLfsEGBkO1ER/ke6h7WJNlbpNJNThIesvnLWJYfe+1Za1dCXNNDn/6ktffshcTM9cATG5qOW8ZliGjJHdQgS8MleKmqR1+y9ukf/K21k0uQyV0DCFReasAz/fxpeKZFRI67uPZWGjK2l+7NwZsPWvvQPjQ8V9YOrizF1YMNrStM5PVrS7NII+8UgorzSQQbN0gOu3TclQV4e9t1SOBsEVEJ6S4cJ5XD8YOONZDhcU3PbaVMOcBcHbyJbamQvp+bxTZRuoBnfnAQz5RJRqXuFHVgrNLuTqS0F1e0dtSrqyiKcl104lMUJXasr9SlQN4Web/alEOaCDk/EcttriIrInLvXQhC/sgtd+C4i1iWz6Rx3HIFgZTVMuyjr/zY2rPnf2Ltm8cgdXfvQeDw3r04r4iI4+B71MuQGlPn37b2lUvwKJfnIbm3k9TYkocXeGnoE9Z+Nfg+jh9CJouIbKLSQfUQUvngHlzjvns+Yu1SoLm6HwTsEeXG2lEFdnWPZNgheyPB0FSSKZVBQHGWguYrS3heGivwrLbreNbaVQoIXiIvcBpe1s41kBuRukg2CCjxgJsKCZWWSuaw7eJTubRWiOvm+5RLRkvOJSjQmfOdQyp3xUnNnQ3JV4Ou+BRFiR068SmKEjvWVeqWl+FlzXEwYxZL43KJltIlWtqaSMdc2bMT1YsTLSzfL1JO71vjkJgbeuFFyhh4uRI+jhuWkWM48ZNnrD135Ki130xSFWkRyVLf2qQDj3TWoWtPYbnevfNWa+ezaKDkkjf7bO0Ja//ZEQRbJ2eORc5dCJFnnM7he5w6cgrHmoP8mW7DS/7ooc+LsjZwILBjWIJdowmOYakbJbzG+wzl2HZ1o4RZoQ/jemoKZdjaVIrKo2rl7SbssIGkANNxIdwnN8EqnfR7SIH2QmWsWkmEWXAebYJrbpEdvEsTLM59Zvc5V2buvPbVoCs+RVFih058iqLEjnWVuj29yB+88ybIzcVpyNM3ymgK1CRvr0t9RUVEXnjueWu3K5DQk1NU/bWMZf18L3J4Ew6qGucoH7JrcMTaHgVq1hrcGCW6LA9JgvRQsGbOJXk7DDkSbN5j7dcuIKDz7cY5a1+YQYmqIkVwpvp3RM9NZXpK1BBm4ey4td3x0/hOxeh2gfJBQLorvEYZpUhub3Q8cU9Z9pqGDv0DeXUL/ShB1kVjbqZOvXB9nMOlPrecJxx2ePxZvkca2vL4p9c98up6dCiX3t5q4x9qNVxHKtvxTIV0r6gPsIQslc1VrNWjKz5FUWKHTnyKoqrw4IkAAAUxSURBVMQOnfgURYkd67rHx879XTsQ1nGugg5quwYQorGxgHl5Yh57biIip0/NW9uhfYBGm+qNpRAmM1dHeEk6jQyL0EEj5YqLTIp+SqLOCl4vpnF9IiJpF/t6GwexZ5ejhs4hFVE4v4jMjYttivJ3qNFzN7q6baeS47MUiiAistLCfmjFQzZKSrBvOdSNiH9T0MyND4KAi1Nw+AVnYfjcYYz2sDpiMULad/MD7qxG+32c2ZDB+DNFZO+stPH+BO3fRfqUcbNuN7pT5jiYGlwOYaFrihYN4OwV2juksBiHMjfaPt+zyKkjIUEcVsO2E9kzvUbY0LugKz5FUWKHTnyKosSOdZW6Qgn9z59FmEV/N2TlvxhGyMvEBcjCv1mEFBQRqSQhP3uL6ALl0OssIxy2KcMiSS75RgtL7IWAkr+z0KrNjsyNNP1/mITUzVIzaOO0yMYt30u1A/tIkvYW8R2yJK2rHdnYpSYalb/yJrYLKlWcb9tmRPkX+0hzK2uGYc3Hfa6vkZEQvlsoBh3KuXqpRwk51ITCSIyLscUhJYHPyf1Xl4VBh95MU8gMh6343tWlbhiR01x7kN7DYTxUBCHR0VQ9Iq2vER4UXP2Wrxpd8SmKEjt04lMUJXasq9T97MN3WvuNY+gQdrEC6Xnffnh7TRrdmqrNaDeq/AAk8aYtaPI9tu2AtYeGIB/DAFLZ96kQAtX44lLhvNxOULPlVCq6sO7twTn6eiAre3rxeiaPzySpiXOSkrlTVJOMZUYigWIOEYkjIo36PmsP9qE5+dQ8skDaCXzXvg3aZe2DwKHakpzZE0Yk3NUbYHfKNJd+40i5epaJ7OElO5XAuEnSuGEJHJHldHa/Hc2MSlNrNZfGo0MyNiDPrJvE+epNjDneneHv4JJWTbvRcZ1w+V5dPRKBpblr3rvY1RWfoiixQyc+RVFix/oGMM+gRli+gaX1iXHUjPvz+desXaqg4ECjw5u6cRBBvtkeeHUP3ooS7gO9+Izj4nweBZO2PEhdr41iB/wXgaWuYxrCFLL4t6EhXNNAH64plaPk6sRqluVXlyPSUWPALeN9mzMIxK46CHR+4TxqEmYXIL8funsVl6GsCkPRCkmSfBIpRkCeUS4x16HkfJZtLpdkJwnt87YPeVnp/Z5DUjdBWyokVbnugW+iW0kuyU2XZG9A0Q4cuJ2kmpoVD88ae3tbfD72/DrRm+Cw1A05gJmfSvYiq9RVFEW5LjrxKYoSO9ZV6h47fdHa51bgZR2vYPk8T3XlUhT42z+AenoiIuk08hI5wLLSgid46jSkdUDRoDdtRwe1rhSVyab3NCJ1y7i0eOQyxLRwjc1lSPYSLeUTNUiQBElScRDwnMni/T94Bp3fJi5Dsnzsngci5+7NQNI+f+R1ax89/bi1j5xHGfptu24XZe3xKQfVayFnmlcVkU5s9H7p6DDGUrdFtkfSrqP1N0w6bCsB6ZotInfbq1BOuMHYchNc607E4ZmBSsy3ApLNFNifyGMsU1lAaTawNeTm8B6PpbuJ3gMO8ucc4EijcpbibvTaV4Ou+BRFiR068SmKEjtM+H668SqKonyI0RWfoiixQyc+RVFih058iqLEDp34FEWJHTrxKYoSO3TiUxQldujEpyhK7NCJT1GU2KETn6IosUMnPkVRYodOfIqixA6d+BRFiR068SmKEjt04lMUJXboxKcoSuzQiU9RlNihE5+iKLFDJz5FUWKHTnyKosQOnfgURYkdOvEpihI7dOJTFCV26MSnKErs+P8BpJww2CzfiRUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    " plot_list(sample_index[0:4], shape=(2,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = do.load(\"/code/traffic_sign_classifier/configs.yml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "validate_input_fn = lambda : input_fn(validation['features'].astype(np.float32), validation['labels'].astype(np.int32), params.train_params, training=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/models/paper_densenet_normalized_augmented', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f7f88f590f0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "classifier = tf.estimator.Estimator(model_fn, model_dir=params.model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = classifier.predict(input_fn=validate_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /models/paper_densenet_normalized_augmented/model.ckpt-55937\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "1 32 41\n",
      "77 21 31\n",
      "78 19 31\n",
      "79 21 31\n",
      "204 18 26\n",
      "271 20 23\n",
      "273 20 23\n",
      "301 4 1\n",
      "302 4 1\n",
      "308 4 1\n",
      "330 0 1\n",
      "331 0 1\n",
      "355 0 1\n",
      "390 4 1\n",
      "391 4 1\n",
      "392 15 1\n",
      "393 0 1\n",
      "394 4 1\n",
      "401 0 1\n",
      "405 0 1\n",
      "420 0 1\n",
      "426 0 1\n",
      "427 0 1\n",
      "428 0 1\n",
      "429 0 1\n",
      "430 0 1\n",
      "431 0 1\n",
      "450 13 1\n",
      "540 13 40\n",
      "541 42 40\n",
      "542 32 40\n",
      "548 32 40\n",
      "690 42 16\n",
      "691 42 16\n",
      "692 42 16\n",
      "693 42 16\n",
      "694 42 16\n",
      "695 42 16\n",
      "696 42 16\n",
      "697 42 16\n",
      "698 42 16\n",
      "699 42 16\n",
      "700 42 16\n",
      "701 42 16\n",
      "702 42 16\n",
      "703 42 16\n",
      "704 42 16\n",
      "705 42 16\n",
      "706 42 16\n",
      "707 42 16\n",
      "708 42 16\n",
      "709 42 16\n",
      "710 42 16\n",
      "711 42 16\n",
      "712 42 16\n",
      "713 42 16\n",
      "714 42 16\n",
      "715 42 16\n",
      "716 42 16\n",
      "717 42 16\n",
      "718 42 16\n",
      "719 42 16\n",
      "750 9 3\n",
      "753 16 3\n",
      "755 16 3\n",
      "773 5 3\n",
      "774 5 3\n",
      "775 5 3\n",
      "776 5 3\n",
      "778 5 3\n",
      "779 5 3\n",
      "1287 28 11\n",
      "1407 11 27\n",
      "1408 11 27\n",
      "1790 1 5\n",
      "1795 6 5\n",
      "1796 32 5\n",
      "1920 42 38\n",
      "1921 42 38\n",
      "1923 36 38\n",
      "1924 42 38\n",
      "1926 42 38\n",
      "1927 42 38\n",
      "1928 42 38\n",
      "1929 42 38\n",
      "1930 42 38\n",
      "1931 42 38\n",
      "1932 36 38\n",
      "1933 42 38\n",
      "1935 42 38\n",
      "1936 42 38\n",
      "1937 42 38\n",
      "1938 40 38\n",
      "1939 40 38\n",
      "1940 42 38\n",
      "1941 42 38\n",
      "1943 42 38\n",
      "1944 40 38\n",
      "1945 42 38\n",
      "1946 42 38\n",
      "1947 40 38\n",
      "1949 42 38\n",
      "1953 34 38\n",
      "1975 40 38\n",
      "2070 4 8\n",
      "2071 4 8\n",
      "2072 4 8\n",
      "2073 4 8\n",
      "2074 4 8\n",
      "2075 4 8\n",
      "2076 4 8\n",
      "2077 7 8\n",
      "2100 1 8\n",
      "2101 1 8\n",
      "2124 41 8\n",
      "2125 0 8\n",
      "2157 1 8\n",
      "2402 33 35\n",
      "2407 33 35\n",
      "2492 34 35\n",
      "2610 26 18\n",
      "2615 24 18\n",
      "2638 24 18\n",
      "2641 24 18\n",
      "2660 24 18\n",
      "2670 24 18\n",
      "2693 24 18\n",
      "2780 6 13\n",
      "2851 34 13\n",
      "2858 40 13\n",
      "3091 8 7\n",
      "3240 26 21\n",
      "3241 20 21\n",
      "3242 20 21\n",
      "3243 23 21\n",
      "3244 23 21\n",
      "3245 23 21\n",
      "3246 23 21\n",
      "3247 31 21\n",
      "3248 31 21\n",
      "3249 31 21\n",
      "3251 31 21\n",
      "3330 25 20\n",
      "3331 25 20\n",
      "3332 24 20\n",
      "3333 11 20\n",
      "3334 24 20\n",
      "3335 25 20\n",
      "3336 11 20\n",
      "3337 25 20\n",
      "3339 29 20\n",
      "3417 35 33\n",
      "3452 20 28\n",
      "3874 8 15\n",
      "4008 14 17\n",
      "4068 1 2\n",
      "4071 1 2\n",
      "4072 1 2\n",
      "4073 1 2\n",
      "4082 7 2\n",
      "4085 5 2\n",
      "4111 13 2\n",
      "4290 26 25\n",
      "4291 26 25\n",
      "4292 26 25\n",
      "4293 0 25\n",
      "4294 22 25\n",
      "4297 15 25\n",
      "4303 29 25\n"
     ]
    }
   ],
   "source": [
    "missclassified = []\n",
    "for i, prediction in enumerate(predictions):\n",
    "    if prediction['class_ids'] != labels_df.class_id.iloc[i]:\n",
    "        print(i, prediction['class_ids'], labels_df.class_id.iloc[i])\n",
    "        missclassified.append((prediction['class_ids'], labels_df.class_id.iloc[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(32, 41),\n",
       " (21, 31),\n",
       " (19, 31),\n",
       " (21, 31),\n",
       " (18, 26),\n",
       " (20, 23),\n",
       " (20, 23),\n",
       " (4, 1),\n",
       " (4, 1),\n",
       " (4, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (4, 1),\n",
       " (4, 1),\n",
       " (15, 1),\n",
       " (0, 1),\n",
       " (4, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (0, 1),\n",
       " (13, 1),\n",
       " (13, 40),\n",
       " (42, 40),\n",
       " (32, 40),\n",
       " (32, 40),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (42, 16),\n",
       " (9, 3),\n",
       " (16, 3),\n",
       " (16, 3),\n",
       " (5, 3),\n",
       " (5, 3),\n",
       " (5, 3),\n",
       " (5, 3),\n",
       " (5, 3),\n",
       " (5, 3),\n",
       " (28, 11),\n",
       " (11, 27),\n",
       " (11, 27),\n",
       " (1, 5),\n",
       " (6, 5),\n",
       " (32, 5),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (36, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (36, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (40, 38),\n",
       " (40, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (40, 38),\n",
       " (42, 38),\n",
       " (42, 38),\n",
       " (40, 38),\n",
       " (42, 38),\n",
       " (34, 38),\n",
       " (40, 38),\n",
       " (4, 8),\n",
       " (4, 8),\n",
       " (4, 8),\n",
       " (4, 8),\n",
       " (4, 8),\n",
       " (4, 8),\n",
       " (4, 8),\n",
       " (7, 8),\n",
       " (1, 8),\n",
       " (1, 8),\n",
       " (41, 8),\n",
       " (0, 8),\n",
       " (1, 8),\n",
       " (33, 35),\n",
       " (33, 35),\n",
       " (34, 35),\n",
       " (26, 18),\n",
       " (24, 18),\n",
       " (24, 18),\n",
       " (24, 18),\n",
       " (24, 18),\n",
       " (24, 18),\n",
       " (24, 18),\n",
       " (6, 13),\n",
       " (34, 13),\n",
       " (40, 13),\n",
       " (8, 7),\n",
       " (26, 21),\n",
       " (20, 21),\n",
       " (20, 21),\n",
       " (23, 21),\n",
       " (23, 21),\n",
       " (23, 21),\n",
       " (23, 21),\n",
       " (31, 21),\n",
       " (31, 21),\n",
       " (31, 21),\n",
       " (31, 21),\n",
       " (25, 20),\n",
       " (25, 20),\n",
       " (24, 20),\n",
       " (11, 20),\n",
       " (24, 20),\n",
       " (25, 20),\n",
       " (11, 20),\n",
       " (25, 20),\n",
       " (29, 20),\n",
       " (35, 33),\n",
       " (20, 28),\n",
       " (8, 15),\n",
       " (14, 17),\n",
       " (1, 2),\n",
       " (1, 2),\n",
       " (1, 2),\n",
       " (1, 2),\n",
       " (7, 2),\n",
       " (5, 2),\n",
       " (13, 2),\n",
       " (26, 25),\n",
       " (26, 25),\n",
       " (26, 25),\n",
       " (0, 25),\n",
       " (22, 25),\n",
       " (15, 25),\n",
       " (29, 25)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missclassified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4410"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels_df.class_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.8321995464852607"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(len(missclassified)*100)/len(labels_df.class_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}