TutorialLSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Long Short Term Memory\n",
    "## A simple guide using keras library "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this tutorial we will use supernova simulations to train a Recurren Neural Network using LSTM units."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-31 14:37:11.234365: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2022-07-31 14:37:11.234383: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import h5py\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = './data/supernovae.hdf5'\n",
    "f = h5py.File(filename, 'r')\n",
    "light_curves = f['curves'][()]\n",
    "labels       = f['labels'][()]\n",
    "ids          = f['sne_id'][()]\n",
    "\n",
    "I = [1,3,32,33]\n",
    "labels_new = np.array([0 if l in I else 1 for l in labels ])\n",
    "multi_class = {1: 0, 2:1, 3:2, 21:3, 22:4, 23:5, 32:6, 33:7}\n",
    "labels     = np.array([multi_class[l] for l in labels])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"./img/sne_types.png\" alt=\"Girl in a jacket\" width=\"350\" height=\"400\"> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Supernova classes are very unbalanced. A faster way is to group by parent class; It means that we are going to transform a multiclass problem into a binary one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUwAAADSCAYAAAAhdfnYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUG0lEQVR4nO3df5BdZX3H8fcHsBFrQX4slCaUZUpqBRyhpJH+sNrGllgckzqgUZFY06Yi/mrtj2B/aW3a0NpSbYsMFSWgEiLqkBERNYrFaYQu/kJAxmgiWRPIYhBRS9rAp3+cZ6d3Nze7z929d+8m+bxm7uy533uec55n9+xnn3NO7o1sExERkzuk3x2IiNhfJDAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwZ5CkqyX9TVl+nqThCda1pFMqtjlY1j1sCv2ZctuIg1ECswck3SrpYUlz+t2X6A9JP2h5PCHpv1uev2IG9v8qSZ/v4fZvbhnP/0r6n5bnV/Rqvy37H/PHvnUy0kuZWXSZpEHgOcAjwIuAD/W1Q9EXtp86uixpK/C7tj/dvx51l+0XjC5LuhoYtv3n/evRzMgMs/suBL4AXA0s78YGJZ0r6UuSvi9pm6S3tlnt1ZK2S9oh6c0tbQ+RtErSNyV9V9J6SUfvYz+vkvQtSY9K2jITM6GDiaQ5knZJemZL7bgy+xwYvUwj6S2SHpK0tfVnUNq/Q9L9kh6UdIWkw6fQjyskvWNc7UZJf1iW/1TSd8pxcJ+kRR1s+yZJrx9X+6qkpWXZkt5QjrOHJP2DpENa1n21pHvLGdotkk7qdHy9lMDsvguBD5THOZKO78I2f1i2+zTgXOCi0QOwxa8B84HfBFZJen6pvwFYCjwX+CngYeDfxu9A0o8D7wJeYPsngF8CvtyFvkdhezewDrigpfwy4NO2R8rznwSOBebS/MG9UtLTy2uXAj8LnAGcUtb5yyl05YPASyUJQNJRNMfNurKv1wG/UI6Dc4CtHWx7LS3jk/Ss0s+Pt6zz28AC4OeBJcCry7pLgbcALwYGgNuA6zoeXQ8lMLtI0q8AJwHrbd8JfBN4+XS3a/tW23fZfsL2V2kOoueOW+1ttn9o+y7gfTS/iAC/D/yZ7eHyC/tW4Lx93Oh5Ajhd0uG2d9i+e7p9j72sBV7eMqt6JXDtuHX+wvZu258DbgJeUsLt94A/sL3L9qPA3wLLptCH2wDTXDoCOA/YZHs78DgwBzhV0pNsb7X9zQ62fSMwX9L8lvFdb/t/Wta5tIzhfuCfGXus/p3te23vKeM7YzbNMhOY3bUc+KTth8rzD9KF03JJz5b0WUkjkh4BXkMzC2m1rWX52zSzSWgC/KOSvifpe8C9NL8UY2a+tn8IvLRse0c5tfq56fY9xrJ9O80Zw3PL9/cUYEPLKg+Xn8Wo0Z/lAPAU4M6Wn+UnSr3TPphmpjsaVC+nOSPC9mbgTTR/WHdKWifpp9psZl/b3g2sBy4ofxRext5/ECY6Vt/ZMr5dgGhmqLNCArNLyrWkl9D8Ijwg6QHgD4BnldOS6fggzS/VibaPBK6gOZBandiy/NPA9rK8jeY0+2ktjyfb/s74ndi+xfZvACcAXwf+fZr9jvZGT1tfCdxg+7GW144ql0dGjf4sHwL+Gzit5ed4ZOvNpQ5dR3OmcRLwbODDoy/Y/qDt0bMl01wK6MRa4BXAIuBHtjeNe32iY/X3xx2rh9v+zw733zMJzO5ZSjNzO5XmGtMZwDNoTn8unOa2fwLYZfsxSQtpf5r/F5KeIuk04HeA60v9CmD16GlNubmwZHxjScdLelH5Zd0N/KCMJ7rvWprreBcA17R5/W2SfkzSc4AXAh+y/QTNH7DLJB0HIGmupHOm0gHbXwJGgPcAt9j+Xtnm0yX9upp/EvcYTUh3dByUgHwC+Ef2nl0C/LGkoySdCLyRscfqJeUYRtKRks7veHA9lMDsnuXA+2zfb/uB0Qfwr8Ar9nHNsNZrgb+W9CjNRf71bdb5HLAZ2Ai8w/YnS/2dNLPTT5b2X6CZUYx3CPBmmr/2u2iukb52Gn2OfbA9DHyRZvZ227iXH6C5Mbed5jT5Nba/Xl77U5qf8RckfR/4NPB0pu464Pk0ZzCj5gBraGa0DwDH0dyI6dQ1wDOB97d57UbgTpqbijcBVwHY/ijNbHZdGd/XgBe0ad83ygcIR8w8Se8Ftrf+20VJzwPeb3tev/rVLZIuBFaWU/vWuoH55Vrpfif/cD1ihql5c8OLgTP73JWekPQUmrOTy/vdl27LKXnEDJL0dppTzX+wvaXf/em2ck11BHiQsaf6B4SckkdEVMoMMyKiUgIzIqLSfnvT59hjj/Xg4GC/uxF9cOeddz5ku+N3uHQix9fBa6Lja78NzMHBQYaGhvrdjegDSd/u9T5yfB28Jjq+ckoeEVEpgRkRUSmBGRFRKYEZEVEpgRkRUWm/vUseETNvcNVN/e5C12xdc27HbTLDjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqFT9eZiSDgWGgO/YfqGko4HrgUFgK/AS2w+XdS8BVgCPA2+wfUupnwVcDRwOfBx4o21LmgNcA5wFfBd4qe2tXRhfRzr5rL+pfJZeROzfOplhvhG4t+X5KmCj7fnAxvIcSacCy4DTgMXA5SVsAd4NrATml8fiUl8BPGz7FOAy4NIpjSYiooeqAlPSPOBc4D0t5SXA2rK8FljaUl9ne7ftLcBmYKGkE4AjbG+ybZoZ5dI227oBWCRJUxpRRESP1M4w/xn4E+CJltrxtncAlK/HlfpcYFvLesOlNrcsj6+PaWN7D/AIcMz4TkhaKWlI0tDIyEhl1yPq5PiKyUwamJJeCOy0fWflNtvNDD1BfaI2Ywv2lbYX2F4wMDBQ2Z2IOjm+YjI1N31+GXiRpN8CngwcIen9wIOSTrC9o5xu7yzrDwMntrSfB2wv9Xlt6q1thiUdBhwJ7JrimCIiemLSGabtS2zPsz1IczPnM7YvADYAy8tqy4Eby/IGYJmkOZJOprm5c0c5bX9U0tnl+uSF49qMbuu8so+9ZpgREf00nf9mdw2wXtIK4H7gfADbd0taD9wD7AEutv14aXMR///Pim4uD4CrgGslbaaZWS6bRr8iInqio8C0fStwa1n+LrBoH+utBla3qQ8Bp7epP0YJ3IiI2Srv9ImIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqDRpYEp6sqQ7JH1F0t2S3lbqR0v6lKRvlK9HtbS5RNJmSfdJOqelfpaku8pr75KkUp8j6fpSv13SYA/GGhExLTUzzN3Ar9t+FnAGsFjS2cAqYKPt+cDG8hxJpwLLgNOAxcDlkg4t23o3sBKYXx6LS30F8LDtU4DLgEunP7SIiO6aNDDd+EF5+qTyMLAEWFvqa4GlZXkJsM72bttbgM3AQkknAEfY3mTbwDXj2oxu6wZg0ejsMyJitqi6hinpUElfBnYCn7J9O3C87R0A5etxZfW5wLaW5sOlNrcsj6+PaWN7D/AIcEybfqyUNCRpaGRkpGqAEbVyfMVkqgLT9uO2zwDm0cwWT59g9XYzQ09Qn6jN+H5caXuB7QUDAwOT9DqiMzm+YjId3SW3/T3gVpprjw+W02zK151ltWHgxJZm84DtpT6vTX1MG0mHAUcCuzrpW0REr9XcJR+Q9LSyfDjwfODrwAZgeVltOXBjWd4ALCt3vk+mublzRzltf1TS2eX65IXj2oxu6zzgM+U6Z0TErHFYxTonAGvLne5DgPW2PyZpE7Be0grgfuB8ANt3S1oP3APsAS62/XjZ1kXA1cDhwM3lAXAVcK2kzTQzy2XdGFxERDdNGpi2vwqc2ab+XWDRPtqsBla3qQ8Be13/tP0YJXAjImarvNMnIqJSAjMiolICMyKiUgIzIqJSAjMiolICMyKiUgIzIqJSAjMiolLNO30iohhcdVO/u9A1W9ec2+8u7Hcyw4yIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqJTAjIiolMCMiKiUwIyIqDRpYEo6UdJnJd0r6W5Jbyz1oyV9StI3ytejWtpcImmzpPskndNSP0vSXeW1d0lSqc+RdH2p3y5psAdjjYiYlpoZ5h7gzbafAZwNXCzpVGAVsNH2fGBjeU55bRlwGrAYuFzSoWVb7wZWAvPLY3GprwAetn0KcBlwaRfGFhHRVZMGpu0dtr9Ylh8F7gXmAkuAtWW1tcDSsrwEWGd7t+0twGZgoaQTgCNsb7Jt4JpxbUa3dQOwaHT2GRExW3R0DbOcKp8J3A4cb3sHNKEKHFdWmwtsa2k2XGpzy/L4+pg2tvcAjwDHdNK3iIheqw5MSU8FPgy8yfb3J1q1Tc0T1CdqM74PKyUNSRoaGRmZrMsRHcnxFZOpCkxJT6IJyw/Y/kgpP1hOsylfd5b6MHBiS/N5wPZSn9emPqaNpMOAI4Fd4/th+0rbC2wvGBgYqOl6RLUcXzGZmrvkAq4C7rX9Ty0vbQCWl+XlwI0t9WXlzvfJNDd37iin7Y9KOrts88JxbUa3dR7wmXKdMyJi1qj5f8l/GXglcJekL5faW4A1wHpJK4D7gfMBbN8taT1wD80d9ottP17aXQRcDRwO3Fwe0ATytZI208wsl01vWBER3TdpYNr+PO2vMQIs2keb1cDqNvUh4PQ29ccogRsRMVvlnT4REZUSmBERlRKYERGVEpgREZVq7pLvVwZX3dTR+lvXnNujnkTEgSYzzIiISgnMiIhKCcyIiEoJzIiISgnMiIhKCcyIiEoJzIiISgnMiIhKCcyIiEoJzIiISgnMiIhKCcyIiEoJzIiISgnMiIhKCcyIiEoJzIiISgnMiIhKCcyIiEoJzIiISgnMiIhKkwampPdK2inpay21oyV9StI3ytejWl67RNJmSfdJOqelfpaku8pr75KkUp8j6fpSv13SYJfHGBHRFTUzzKuBxeNqq4CNtucDG8tzJJ0KLANOK20ul3RoafNuYCUwvzxGt7kCeNj2KcBlwKVTHUxERC9NGpi2/wPYNa68BFhbltcCS1vq62zvtr0F2AwslHQCcITtTbYNXDOuzei2bgAWjc4+IyJmk6lewzze9g6A8vW4Up8LbGtZb7jU5pbl8fUxbWzvAR4BjplivyIieqbbN33azQw9QX2iNntvXFopaUjS0MjIyBS7GNFejq+YzFQD88Fymk35urPUh4ETW9abB2wv9Xlt6mPaSDoMOJK9LwEAYPtK2wtsLxgYGJhi1yPay/EVk5lqYG4Alpfl5cCNLfVl5c73yTQ3d+4op+2PSjq7XJ+8cFyb0W2dB3ymXOeMiJhVDptsBUnXAc8DjpU0DPwVsAZYL2kFcD9wPoDtuyWtB+4B9gAX2368bOoimjvuhwM3lwfAVcC1kjbTzCyXdWVkERFdNmlg2n7ZPl5atI/1VwOr29SHgNPb1B+jBG5ExGyWd/pERFRKYEZEVEpgRkRUSmBGRFRKYEZEVEpgRkRUSmBGRFRKYEZEVEpgRkRUSmBGRFRKYEZEVEpgRkRUSmBGRFSa9NOKIsYbXHVT9bpb15zbw55EzKzMMCMiKiUwIyIqJTAjIirlGmbMmE6ufUKuf8bskxlmRESlzDD7LLOuiP1HZpgREZUSmBERlRKYERGVEpgREZUSmBERlWbNXXJJi4F3AocC77G9ps9dOqDl7nxE52ZFYEo6FPg34DeAYeC/JG2wfU9/e1Yn4RNxcJgVgQksBDbb/haApHXAEmC/CMx+SVBHzKzZcg1zLrCt5flwqUVEzBqy3e8+IOl84Bzbv1uevxJYaPv149ZbCawsT58O3NfBbo4FHppiF6fath/77FfbmdznSbYHprivfZrm8dVt0/l+Hgj6Of59Hl+zJTB/EXir7XPK80sAbP9dF/cxZHvBTLbtxz771bZf/T1QHezfk9k6/tlySv5fwHxJJ0v6MWAZsKHPfYqIGGNW3PSxvUfS64BbaP5Z0Xtt393nbkVEjDErAhPA9seBj/dwF1f2oW0/9tmvtv3q74HqYP+ezMrxz4prmBER+4PZcg0zImLWO2ADU9IPurUNSYOSvlax3vMkfaxX+6ltW/rxiKQvSbpX0l912LZqDNMZdzd+Pgeqg/l7M53fhZlwwAZmcJvtM4EFwAWSzup3hyL2dwd8YEp6qqSNkr4o6S5JS3q8yyMkfVTSPZKukHRI6cfi0oevSNpY0e9XSfqIpE9I+oakv59KZ2z/ELgT+JmptO9A23HXkPQn5WfzFUn50JWYtWbNXfIeegz4bdvfl3Qs8IXywR69utu1EDgV+DbwCeDFkj4H/Dvwq7a3SDq6cltnAGcCu4H7JP2L7W0TNxlL0jHA2cDbO2k3BXuNG7hhskaSXgAsBZ5t+0cdfG8iZtzBEJgC/lbSrwJP0LxH/XjggR7t746WDxG5DvgVmsD7D9tbAGzvqtzWRtuPlG3dA5zE2PfcT+Q5kr5EM+Y1M/DvWtuNe9LABJ4PvM/2j6Cj703EjDsYAvMVwABwlu3/lbQVeHIP9zd+5mqa0J7KjHZ3y/LjdPbzus32C6ewz6lqN+4aU/3eRMy4A/4aJnAksLOE5a/RzNJ6aWF5i+chwEuBzwObgOdKOhngAD3tbDfuGp8EXi3pKXDAfm/iAHEwBOYHgAWShmhmm1/v8f42AWuArwFbgI/aHqH5FJyPSPoKcH2P+zBdiyQNtzx+saLNXuOu2ZHtT9B8bsCQpC8DfzTFPkf0XN7pExFR6WCYYUZEdEUCMyKiUgIzIqJSAjMiolICMyKiUgIzIqJSAjMiolICMyKi0v8Bp3CZ4e6+X9QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(5,3), sharey=True)\n",
    "\n",
    "# (labels, class names, figure title)\n",
    "multi_tuple = (labels, ['Ia', 'II', 'Ibc', 'IIn', 'IIP', 'IIL', 'Ib', 'Ic'], 'All labels')\n",
    "binar_tuple = (labels_new, ['I', 'II'], 'Type I vs Type II')\n",
    "\n",
    "for index, (y_vector, cls_names, title) in enumerate([multi_tuple, binar_tuple]):\n",
    "    values, counts = np.unique(y_vector, return_counts=True)\n",
    "    x_range = np.arange(len(values))\n",
    "    axes[index].bar(x_range,counts)\n",
    "    axes[index].set_title(title)\n",
    "    axes[index].set_xticks(x_range)\n",
    "    axes[index].set_xticklabels(cls_names)\n",
    "    \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA30AAAFNCAYAAAC0bwY9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABEmElEQVR4nO3de5ycZXn4/8+1pyTLBgg5EXJaoshZd0lIQytWPGKpUOsJq4Khgrb2W1urter35xlrq631UKtADSCg0K+itFREaS1RiSExqxxTNCZsDuRESLLktIf798c8C0vYJHuYmWdm9vN+vea18zwz88y1987OzPXc933dkVJCkiRJklSb6vIOQJIkSZJUOiZ9kiRJklTDTPokSZIkqYaZ9EmSJElSDTPpkyRJkqQaZtInSZIkSTXMpE+SJEmSaphJnySprCLihRHx04jYGRGPR8RPIuLsvOOqRBHx4ohYn9NzfzQibsjjuSVJxdWQdwCSpLEjIo4G/gP4E+AWoAk4F9hf5jgCiJRSXzmfV5KkPNjTJ0kqp+cBpJS+kVLqTSntTSndmVL6JTy7dykiWiMiRURDtv2jiPjbiFie9RR+NyKOG3D/RVkv4hMR8YuIePGA234UEVdGxE+APcC87NjvjIhHImJHRPxzlhASEXUR8X8jYl1EbImI6yPimOy2OyLizwb+Ytnz/WF2/fMR0RkRuyJiZUScO+B+dRHxNxHx64jYHhG3DPwdBtzvKOB7wAkR0ZVdToiIPRExecD95kfE1ohojIi3ZT2nX8za5+GIeOmA+x4TEf8aEZsiYkNEfDIi6kfyh5QkVQ+TPklSOf0v0BsR10XEqyJi0giOcQlwGXAC0AN8ASAiZgK3A58EjgPeC3wrIqYOeOxbgSuAicC6bN/vA2cDLwDeALwy2/+27HIeMA9oAb6U3XYT8Kb+g0bEacDc7PkB7gXasjhuAv4tIsZnt/058AfA72a/ww7gnw/+JVNKTwKvAjamlFqyy0bgR1mc/d4CfDOl1J1t/xawBpgCfAT49oCk8rqszZ4LtAOvAN5+8HNLkmqLSZ8kqWxSSruAFwIJuBrYGhG3RcT0YRzm6yml+7Ok6P8D3pD1Vr0F+M+U0n+mlPpSSj8AVgC/N+Cx16aUHkgp9QxIkj6dUnoipfQo8N8UkjWANwP/mFJak1LqAj4AXJz1Ot4KtEXE3AH3/XZKaX/2e96QUtqePc8/AOOAk7P7vgP4UEppfXb/jwKv6+/NHILrst+V7Pd+E/D1AbdvAf4ppdSdUroZWA1ckLXxq4C/SCk9mVLaAnwOuHiIzytJqlImfZKkskopPZRSeltKaRZwBoXern8axiE6B1xfBzRS6NWaC7w+G9r5REQ8QSHBnHGIx/Z7bMD1PRR69MjiWjfgtnUU5sJPTyntptCr158wXQzc2H/HiPiriHgoG2L5BHBMFiNZnLcOiPEhoBcYauL7XeC0iJgHvBzYmVJaPuD2DSmldFDcJ2TP2whsGvDcXwWmDfF5JUlVykIukqTcpJQejohrKfR+ATwJNA+4y/GDPGz2gOtzgG5gG4WE7usppcsP95TDCG8jhURp4HP1AJuz7W8AH4mIu4EJFHoJyebvvR94KfBASqkvInYAkT2uE7gspfSTIcTwrHhTSvsi4hYKvYun8MxePoCZEREDEr85wG3Z8+4HpqSUeobw3JKkGmFPnySpbCLilKwXbFa2PZvC8MRl2V06gBdFxJysaMoHBjnMWyLitIhoBj4O/L+UUi9wA/DqiHhlRNRHxPgoLHkwa4ThfgP4y4g4MSJagE8BNw9ImP6TQlL48Wx/fyXQiRSSw61AQ0R8GDh6wHG/AlzZPzQ0IqZGxEWHiGEzMLm/gMwA11OYb3hh9nsPNA3486ywy+uBUykMe90E3An8Q0QcnRWUeU5E/O6QW0SSVJVM+iRJ5bSbQqGRn0XEkxSSvfuBvwLI5uHdDPwSWElheYeDfR24lsKwzPEUCqOQUuoELgI+SCHh6gTex8g/676WPdfdwG+AfcD/6b8xm4/3beBlFIq19Ps+haqb/0thaOU+njms9PMUet7ujIjdFNrgtwYLIKX0MIXkc002JPOEbP9PgD7g5ymltQc97GfASRR6P68EXpdS2p7ddgmFZTIepFBA5v/xzOGvkqQaFM8c9i9JUuWKiB8BN6SUrsk7lrxFxH8BNw1si4h4G/D2lNILcwtMklRxnNMnSVKViYizgbMo9GxKknRYDu+UJKmKRMR1wA8pLL2wO+94JEmVz+GdkiRJklTD7OmTJEmSpBpm0idJkiRJNawmCrlMmTIltba25h2GJEmSJOVi5cqV21JKUwe7rSaSvtbWVlasWJF3GJIkSZKUi4hYd6jbHN4pSZIkSTXMpE+SJEmSaphJnyRJkiTVsJqY0ydJkiRJo9Xd3c369evZt29f3qEc0vjx45k1axaNjY1DfoxJnyRJkiQB69evZ+LEibS2thIReYfzLCkltm/fzvr16znxxBOH/DiHd0qSJEkSsG/fPiZPnlyRCR9ARDB58uRh90Sa9EmSJElSplITvn4jic+kT5IkSZJG6I1fvYc3fvWevMM4LJM+SZIkSaphFnKRJEmSpArxiU98ghtvvJHZs2czZcoU5s+fz3vf+95RHdOkT5IGWPfWSwCY+/Xrc45EkiRVg937utm1r4eV63Ywf+6kUR1rxYoVfOtb32LVqlX09PRw1llnMX/+/FHH6PBOSRqgt6uL7o0b2bNqVd6hSJKkCrdy3Q4efmw363fs5c3XLGPluh2jOt6Pf/xjLrroIiZMmMDEiRN59atfXZQ4TfokKbNn1Sr2P/ww3Rs28Ojiy0z8JEnSYS1bs52+VLje3dPHsjXbR3W8lFIRono2kz5JyuxZfi9kb7apu7uwLUmSdAiL5k2mLltBobGhjkXzJo/qeC984Qv593//d/bt20dXVxe33357EaKsgDl9EVEPrAA2pJR+PyKOA24GWoG1wBtSSqPrJ5WkIWheeDZEQEpEY2NhW5Ik6RDmz53EKcdPZNe+Hj5/cfuo5/SdffbZXHjhhbzgBS9g7ty5LFiwgGOOOWbUcVZCT9+7gYcGbP8NcFdK6STgrmxbkkquub2dcaecQuPMmcxZ8jWa29vzDkmSJFW4ieMbmXnshFEnfP3e+973snr1ar7zne+wevXqohRyybWnLyJmARcAVwLvyXZfBLw4u34d8CPg/eWOTdLYNO/Wb+cdgiRJGsOuuOIKHnzwQfbt28ell17KWWedNepj5j2885+AvwYmDtg3PaW0CSCltCkipg32wIi4ArgCYM6cOSUOU5IkSZKe7eZ3nFPU4910001FPR7kOLwzIn4f2JJSWjmSx6eUrkopLUgpLZg6dWqRo5MkVY0lFxQukiRpUHn29P0OcGFE/B4wHjg6Im4ANkfEjKyXbwawJccYJUmSJKmq5dbTl1L6QEppVkqpFbgY+K+U0luA24BLs7tdCnw3pxAlSZIkqepVQvXOg30aeHlEPAK8PNuWJEmSJI1A3oVcAEgp/YhClU5SStuBl+YZjyRJkiQNSf+88sXFWUi9FCqxp0+SNNZYjEWSpGdJKdHX1zfq45j0SZIkSVKFWLt2Laeeeip/+qd/yllnnUVnZ+eoj2nSJ0mSJEkjtX8X7OyEzuVFO+Tq1au55JJLWLVqFXPnzh318Uz6JEmSJGkkOpfD5vvhiXVw3YVFS/zmzp3LokWLinIsMOmTJEmSpJFZuxRSNueu90BhuwiOOuqoohynn0mfJEmSJI1E67kQWUpV31TYrkAVsWSDJEmSJFWd2Qth+hmwbye89prCdgUy6ZMkSZKkkRp3dOFSpISvtbWV+++/vyjH6ufwTkmSJEmqYfb0SZIkSdJILb497wiOyJ4+SZIkSaphJn2SJEmSVMNM+iRJ1W3/LtjZWbQFcSVJqjUmfZKk6tW5HDbfD0+sg+suNPGTJGkQJn2SpOq1dimkvsL13gOFbUmSymjxHYtZfMfioh3vt3/7t4t2rH4mfZKk6tV6LkT2UVbfVNiWJKmK/fSnPy36MU36JEnVa/ZCmH4GHDsXLr2taAvjSpKUl5aWlqIf06RPkpS/0RRjGXc0HDPbhE+SlIuuA11senITHVs68g7lkEz6JEn5shiLJKlKdWzpYPWO1Wzo2sDld15esYmfSZ8kKV8WY5EkVakVm1eQSAB093WzYvOKnCManEmfJClfFmORJFWpBdMXEAQAjXWNLJi+IOeIBteQdwCSpCq15ILCz8W3j+44/cVY9u2E117j3DxJUtVom9bGyZNOZnf3bj597qdpm9aWd0iDyq2nLyLGR8TyiPhFRDwQER/L9h8XET+IiEeyn5PyilGSVCYWY5EkVamWphZmHDWjaAlfV1dXUY4zUJ7DO/cDL0kpvQBoA86PiEXA3wB3pZROAu7KtiVJAy254OmeNkmSpMPIbXhnSikB/WlsY3ZJwEXAi7P91wE/At5f5vAkSZIk6YiWnL8k7xCOKNdCLhFRHxEdwBbgBymlnwHTU0qbALKf03IMUZKKZvEdi1l8x+K8w5AkSWNMrklfSqk3pdQGzAIWRsQZQ31sRFwRESsiYsXWrVtLFqMkSZIkVbOKWLIhpfQEhWGc5wObI2IGQPZzyyEec1VKaUFKacHUqVPLFaqkEbCHS5IkKT95Vu+cGhHHZtcnAC8DHgZuAy7N7nYp8N1cApQkSZKkGpDnOn0zgOsiop5C8nlLSuk/IuIe4JaI+GPgUeD1OcYoSTqExbEZgMqfvi5JUumse+slAMz9+vU5R3JoeVbv/CXQPsj+7cBLyx+RJEmSJNWeipjTJ0mSJElj3Ve+8hXa2tpoa2vjxBNP5LzzzivKcU36JEmSJGmEeru66N64kT2rVo36WO985zvp6Ojg3nvvZdasWbznPe8pQoT5zumTJGn0Ft+edwSSpDFqz6pV7H/4YUiJRxdfxpwlX6O5/Vkz2Ibt3e9+Ny95yUt49atfXYQo7emTJEmSpBHZs/xeSAmA1N1d2B6la6+9lnXr1vGRj3xk1MfqZ9InSVXok/vX8v92PgSdy3OL4Tn79nLBE4/nGoMkSXlqXng2RAAQjY2F7VFYuXIln/3sZ7nhhhuoqyteqmbSJ0kV7lmL23cu532bN/CaJ7bDdRfmk3RVQgySJOWsub2dcaecQuPMmUUZ2vmlL32Jxx9/nPPOO4+2tjbe/va3FyVO5/RJUrVZu5SGlKgH6D0Aa5fC7IVjLwZJkipAfUsL9S0tRZnLt2RJaVa/NemTpGrTei49EZAS9fVN0Hpu1ceweMY0wIXeJUkqBZM+Sao2sxfymekzOWXfXl732m/m08NWCTFIklQB5n79+rxDOCLn9EnSAM+aP1ehfj1+Arcfe1yuydYvxo/jmmOPpmNcU24xSJKkIzPpkyQNW8eWDlbTzQZ6ufzOy+nY0pF3SJIkFUXKlmCoVCOJz6RPkirdY/cVLhVkxeYVJICA7r5uVmxeMarjPbhpFw9u2lWU2CRJGqnx48ezffv2ik38Ukps376d8ePHD+txzumTJA3bgukLCArr0TbWN7Jg+oK8Q5IkadRmzZrF+vXr2bp1a96hHNL48eOZNWvWsB5j0idJGra2aW2cTCO76ePTr7iatmlteYckSdKoNTY2cuKJJ+YdRtGZ9EmSRqSFOlqoM+GTJKnCOadPkqpQF31soscCKpIk6YhM+iSpTLoOdLHpyU3DTtQOTvCsnClJkobDpE+SyqBjSwerd6xmQ9eGYSVqgyV4xa6cKUmSaptJnySVQSFRK5R/Hk6iNliC1185kwSNdbVRObOXvXTzuL2WkiSVgEmfJJVBIVELYHiJ2mAJXn/lzJnUc3UNVM7s2NLB/lhPd2xzuKokSSVg0idJZdA2rY2TJ53MzJaZw0rU2qa1Mae7jqm98YzHtVDHDBqqPuEDsl7P5HBVSZJKxCUbJKlMWppaaGlqGXai9oL9+zl9/z7a9h94eufxZxY3uJEoUgyFXs+AlFzoXZKkEsitpy8iZkfEf0fEQxHxQES8O9t/XET8ICIeyX5OyitGScUx0qqVAjqX8+Ftj/HG3Tvguguhc3neERVd27Q2xqVZNKYpNTFcVZKkSpPn8M4e4K9SSqcCi4B3RcRpwN8Ad6WUTgLuyrYlVamRVq1UZu1SGkjUA/QegLVL846oJOqZQCPHmfBJklQCuQ3vTCltAjZl13dHxEPATOAi4MXZ3a4DfgS8P4cQJRXBYFUr/WI/DK3n0kMAifr6Jmg9N++InrLk/CV5hyBJkoagIgq5REQr0A78DJieJYT9ieG0HEOTNEojrVqpzOyFfHzK8dw8cRJcehvMXph3RCVxYO9Untz+fFau25F3KJIk1Zzck76IaAG+BfxFSmnXMB53RUSsiIgVW7duLV2AkkZlpFUr9bRHmsbznYnH1mzCt3LdDnZ0voqubWfx5muWDTvxW3zHYhbfsbhE0UmSVP1yTfoiopFCwndjSunb2e7NETEju30GsGWwx6aUrkopLUgpLZg6dWp5ApY0Ii1NLcw4akZ1JHyP3Ve4qGyWrdkOqQ6oo7unr7AtSZKKJs/qnQH8K/BQSukfB9x0G3Bpdv1S4Lvljk2SVD6L5k2G6AP6aGyoK2xLkqSiyXOdvt8B3grcFxEd2b4PAp8GbomIPwYeBV6fT3iSpHKYP3cSk2Z/j+49M7ju9X/O/Lmu1CNJUjHlWb3zx5BVd3i2l5YzllJY99ZLAJj79etzjkRSLaq1yplNE7bSNGGrCZ8kSSWQeyGXWtXb1UX3xo3sWbUq71AkqSTe+NV7eONX78k7DEmSdAR5Du+sWXtWrWL/ww9DSjy6+DLmLPkaze3teYcl5ebiLz5QuHJ+vnFUq49sa8k7BEmSVMXs6SuBPcvvhVRYjDp1dxe2pTGsaV8vR+/Yb8+3lUFVRC5VIUkaKnv6SqB54dkQASkRjY2FbWmM2rNqFdM27iES5ev5XnJB4efi20v7PGXy8cmfAeDmnOMopdYD7807BEmSapZJXwk0t7cz96Yb2bP8XpoXnu3QTo1pe5bfS6RC1ab+nu9K/p/ooo/d9NGxpaM86wrWWIIqSZIqj8M7S6S5vZ0p77iior/cSuXQvPBsUkCCiu/57tjSwWq62UAvl995OR1bOvIOSZIkadRM+qRatuSCp3uSctLc3s6WE5rZeVxTxRc1WrF5BQkgoLuvmxWbV+QdklT1nHsoSflzeKekkrv+fc8HYEkFJ3wAC6YvKAxDTdBY38iC6QvyDklD0HWgi93du8s3JFeSpCpjT58kZdqmtXEyjcyknqtfcbUJRBXo2NLB6h2r2dC1wSG5kiQdgkmfJA3QQh0zaKj4hK8SFkbfva+bDU/sZeW6HbnFUBiSW1gixyG5kiQNzqRPqmEPbNrJA5t25h1GVXnyQA9PHujJO4yKt3LdDh5+bDfrd+zlzdcsyy3xKwzJDQAa6xySK0nSYEz6JEnDtmzNdvoKHWx09/SxbM32XOJom9bGyZNOZmbLTIfkShoWiwxpLLGQi6RcrHvrJQDM/fr1OUdSPkvS9LxDKJpF8yZTF9CXoLGhjkXzJo/qeDe/45wRP7alqYWWphYTPkmSDsGePkm56O3qonvjRvasWpV3KBWvEubOHWz+3EmccvxEZk2awI1vX8T8uZPyDkmSJB2CSZ+kstuzahX7H36Y7g0beHTxZSZ+h1Epc+cGM3F8IzOPnWDCJ0lShTPpk1R2e5bfW1gMD0jd3YVtDepQc+cqsfdP5dV1oItNT25ymQpJ0hGZ9Ekqu+aFZ0MUKi5GY2NhW4PqnzsHT8+dq+TeP5VHVa1P+Nh9hYskKTcmfZJK76Avfc3t7cy96Uam/uVfMmfJ12hub88xuMo22Ny5SqmcqfxU0/qEi76/l6kde8uamK576yVPFYuSJJn0ScrJuzZ/gffNvceEbwgOnjs3WO+fxpYF0xcQCUiVvT5hx5YO7joVxu9J/N3XLitb4mehKEl6JpM+SRrgpAP7+IPdT0Dn8rI83wObdvLApp3DeoyVM9U2rY2TaWQm9RW9PuHqu2/jg7f08YalifffsJfVd99W8ufcs2oVex9+iAMjKBRlD6EO5mtCtcKkT5L6dS7nw9se4427d8B1F5Yt8RsJK2eqhTpm0FCxCR/AaY/20dAL9QkaegvbpbZn+b1EgmD4haLsIdTBfE2oVpj0SSq5JWl6dSxMvnYpDSTqAXoPwNqleUckVbXnnHcRAH1AXVPTU9ul1LzwbFJAYniFolxKZuy5+IsPcPEXHzjk7b4mVEtM+iRVhMV3LGbxHYvzDaL1XHoIegHqm6D13OIef/8u2NlZ0T2Iw3HzO87h5necU5RjOYSqNjW3t7N1arDrmKD12mvLMoe3ub2dLSc0s/O4pmEVinIpmbGnaV8vR+/Yf8hkzteEakmuSV9EfC0itkTE/QP2HRcRP4iIR7Kfjl2SVB6zF/LxKcdz88RJcOltMHth8Y7duRw23w9PrHvG0NEJfU8ypXdLzSSCI+UQqtp1/WXNXPUnzWUt2nT9+57PVR8+a1jP6VIyY8ueVauYtnEPxzx+4JC9eL4mVEvy7um7Fjj/oH1/A9yVUjoJuCvbllTjLv6H5Vz8D0VKfEbRo/ZI03i+M/HY4iZ8UBgqmrL5TP1DRzuX09rzG6b1bh72HMJi9rLlbbRDqJacv4Ql5y8pUXQaK5rb2xl3yik0zpzpUjJjwFDmfvqaUC0ZUtIXBW+JiA9n23MiYtTfiFJKdwOPH7T7IuC67Pp1wB+M9nkkVb6m/Ymjd6XR9/Qcokctd63nQmRvuf1DR9cuJUgEjOk5hA6hUqWYd+u3ee5dP/TLfQ7KPcR7qHM/fU2oVgy1p+/LwDnAm7Lt3cA/lyQimJ5S2gSQ/ZxWoueRat5DPTO5Zf85rFy3I+9QDmvPqlVM25o4Zmca/WT5wXrUKsHshTD9DDh27tNDR1vPJRGFJbZLMYewSox2CNUbv3oPb/zqPaUITVKZlHuI90jnfkrVqmGI9/utlNJZEbEKIKW0IyKaShjXEUXEFcAVAHPmzMkzFKkirVy3gw/ueRM91HPzNcsqej23wYbZjPgDuL9HLfVVXiI17ujCpX/o6OyFrG04kaPSk0y/9IbiDymtEs3t7cy96cbC333h2X75kipAf6/b3K9fX/Ln6h/iTSqc+CtXEnb9+54PwBLfczQGDLWnrzsi6in0ghMRUylUYC6FzRExI3ueGcCWwe6UUroqpbQgpbRg6tSpJQpFql7L1mynh3r6qKO7p49la7bnHdIhjbTE+qAG61GrYHvrjmJb/bSKj7PUmtvbmfKOK0z4pApRzp43h3hLpTfUpO8LwK3AtIi4Evgx8KkSxXQbcGl2/VLguyV6HqmmLZo3mQZ6qaOXxoY6Fs2bnHdIh9Tc3s6WqcHOY6I4Z3jHHQ3HzK6KRKpahuCq8jxn314ueOLxypm3qppR7vXp8qqS2XWgi01PbqJjS0dZnk/K05CSvpTSjcBfA38LbAL+IKX0b6N98oj4BnAPcHJErI+IPwY+Dbw8Ih4BXp5tSxqm+XMn8anmb/DWcUsremhnvzzKuuetfwju1/e/iDdfs8zET0PXuZz3bd7Aa57YXlkFiwZhclp9yt3zlkeVzI4tHazesZoNXRu4/M7LTfxU8w47py8ijhuwuQX4xsDbUkoHV94clpTSmw5x00tHc1xJBafd9SCn8SCnfewzeYeiQQw2BLfSk/NK9LZ/+7vClXd8J9c4ymrtUhpSoh6eLlhUiT3bWXLakFIhOS3TkOsHN+0q+XPUsqd63lIqW8/bvFu/XfLnGGjF5hWkwqwluvu6WbF5BW3T2soag1RORyrkspJsmg0wB9iRXT8WeBQ4sZTBSRqlAwm6C8sgVFoPml/Knh6C20OisaGxoofgVrJx+/cyfv+einydl0zrufRkX8rrK61g0UBrlzIu6zGq6ORUzzAWiistmL6ASIUvuY31jSyYviDvkKSSOuzwzpTSiSmlecD3gVenlKaklCYDvw+U95SMpGHZs2oV7OiFrr6yzMnQ8FXbENyDlXtdrcHsWbWK6ds6OXbXtrH1Op+9kHHTz6S+0gsWDbY+papCrRdXapvWxsk0MpN6rn7F1fbyFcniOxaz+I7FeYehQQy1kMvZKaX/7N9IKX0P+N3ShCSpGPrnYAxcBmGsWLNkPb/6yqMjSgA+sq2Fj2xrKUFUg7vl+NU8NPMHVZfwQfnX1RpMYbmPNCZf51VRsKjKqumW3ZILChflooU6ZtBw+ITPv5FqxFCTvm0R8X8jojUi5kbEh4DKrf8ujUEH97r0z8EoyjIIJdDLXrp5vOiT5/esWsX+rQfo3tkztnp+yqzc1f0OpbDcR1Ts61zkkpyW6v1FpbP42gUsvtYhllKpDDXpexMwlcKyDd8BpmX7JFWIg3tdmtvbYVI9tNSVrRraoTywaScPbNr51HbHlg72x3q6Y1vRq6YVqs4Vrldaz8/B7VDNKmVdreb2dn4++/ksn3kme//28zU7FE1DV8r3F0mqVkcq5AJAVqXz3SWORdII9fe6kBKPLr7sqSQvvfpYgIr7Irxi8wogQTxdNa2LPnbTR8eWjlHNrShUncsOb89PyeRR3W8wK9ft4CPz30pfgvE/fZIbT9tRlUNlVTyDvb84X0s1p3/I6eLb841jDOqfs7jk/CU5RzI8Q0r6IuK/eerc+dNSSi8pekSShm2wXpdKS/QGKlRJKyQMjfWNHNN0DKvpJgGX33n5qCbVN7e389jUYPx+eMFX8u3hrGWVUt1v2Zrt9GWfTi57IXj2+4tVGVWLFsdmAKor7VCehpT0Ae8dcH088Fqgp/jhSBqJSul1Gaq2aW2MS7PoYy9Xv+Jz2XpJFO3M/PWXNQOwpEYSvkpdh665vT33pHrRvMnUBfQlaGyoc9kLPev9ZTjvJeU+g7/upo0AzLXYoVQ1LvnMLxm/t4c906trmaChDu9cedCun0TE/5QgHo1QfwGPuV+/PudIlIfm9nbGnXIKfbt2ccJnP1MVb0L1TKCeCU99IQsKnZWemX+2MbkO3RDNnzuJU46fyK59PXz+4vYx1cvXPz/09JzjqETzugvJVKUP6+zd30ff/j7/t6UqsWfVKqZt3EMknjGdphoMqZBLRBw34DIlIl4JHF/i2DQMlVA6PS+VsFZYJZh367d57l0/rJo3n4HGynpJ8f2dxPeHV8hlzK5DNwwTxzcy89gJYyrhU/Wz0rBq0mP3FS41qrBMUHUuhzXU6p0rgRXZz3uAvwL+uFRBaXgKRTweyr10el7GcsJbS4a0XlK1O5Dgyb5nvFbfdssB3nbLgUM+ZM/ye6kbq+vQSeVWxi+slVxpWNLgCssEVe5yWIcz1KTv1JTSvJTSiSmlk1JKrwB8d6oQY/mDo1LWCpOOZM+qVbCjF7r6nvFaHbc/ccyudMjX7lPzNam+DxhJh/ZUpWES0VBfcf/bb/zqPbzxq/fkHYZUUZrb29kyNdh5TFTV0E4YetL300H2+U5wGOV8s3z6g2PsfSncs/xeUgWsFSYdSf9rc2CP3Z5Vqzh+G0zaxSFPWvTP12ycObPqPmBUWiMZLpyHWlqfspiap3Qz57xtTD1zN3NevI3mKd15h6QqcsnX9nDFv+ypuJPdi76/l6kde2t6fczrL2vmqj9prrrP48MmfRFxfETMByZERHtEnJVdXgw0lyNAHVlzezvjpjbReEzDmPtSWOhmj6rsZldl+vjkz/DxyZ8p+nH7X5sDX6tDnRtQzfM1VTqP9Z3AE3uPrbgvfZVg977ZbNzxIlau2zGsxz15oIcnD5SpOPnapRw17QBTTu+i+bi9sHZpeZ53yQVPr/GmI1ocm59aHqFS7Fm1imlbE8fsTBU1yqljSwd3nQrj9yT+7muX1XTiV42OVL3zlcDbgFnAPw7Yvxv4YIli0gjMWzyrcGWMfSlsbm9n85TZjN+/h7avfsEvxQc5fcYxeYegTGH9wDnPeq2mbCH5umGetLBi79i2Z9Uqpm1bT6Q+1r5tMa3XLvH9L7Ny3Q5Wb7qMvlTPm69Zxo1vX1SZRX5az4Wog9QH9U2F7TKw6mv1G+yEYSX8/6+++zY+eEsfDb3Q89O9rH7ubbS9ri3vsJQ5bNKXUroOuC4iXptS+laZYpKG5atv+SgAN1fAG16lcfHWynLwa7W5vZ0tJzQzfm8PL/jn4fXS93Z10bdrl6Xex6hH7ryb+pSoA3r2H+CRO+/mBb4OAFi2Zjt9qR6op7unj2Vrtldm0jd7IUw/A/bthNdeU9gug6eGBHvqvmr1FxMZyQnDUjrt0T6iF+oT0FvYVuU4bNIXEW9JKd0AtEbEew6+PaX0j4M8TJI0RNe/7/nA8BaS7y9gREpVt06QiuOXU57DGXUNpL4eeusbuG/Kc3hB3kEdQv9w6ZvL9HyL5k2mLnrpS9DY0MiieZPL9MzDt3j8XhjfxJIyJXxAoYpwdzriCaMPb39fdu3H5YlLQ9ZfTGT8fnjBVyrn/f85513Emqtvpg+oa2qi9byL8g6pJMo2BLzIjjS886jsZ0upA5Gkojn+zLwjKKlCxd5nFjCqlA99lcdpL/sd/r+fvp3TH/8ND894Hh9+2e/kHVLFmD93EifP+Bq7957I5y/+WGX28uXkqSrCVN/C0nqm774lSH09TGh8iDYq42/Y3N7ON94yntmP9nLJO6/1tVVhjjS886vZz4+VJxxJY8WBvVPp3jODlet2jJkvZbv3dbNrX8+of+enlnFIyQJGY9T8uZPoe34bP9l3Bp+/uH3M/A8N1cTxnUwc31nx7VLuHoPBqgj7xfzwnrNvL6fs2wudy8s2BPdIOu6/iR83Qnc08tN7P8XVQNsZf5R3WABsnFnPxpn1vNPXVcU5Uk8fABExFbgcaB34mJTSZaUJS1ItW7luBzs6XwWp7qlCC5WiWInZwVau28HDj+2mL/GM4hJdB7rY3b2bji0dQ16Yvrm9nbk33Vj4wrbw7DH/pe3md5yTdwi5mDi+kYnjGys+sakmu/fNZvfeE2v2ZNTAKsJHmgv2UM9M7uudw4Vlaosu+thN37DeC0drSZp++Dt0Lud9j22ggQTXXQiX3lYRid+KNd+nO4K+CLpJrFjz/YpJ+lS5hrpO33eBY4AfArcPuOgQ1jZ9lrVNn807jDFh975uNjyxd9iluZWfZWu2Q6oD6p4qtPCcfXu54InHC2dTc9KfmK3fsZc3X7OsqK+pQnGJwvX+37ljSwerd6xmQ9cGLr/z8mGVt25ub2fKO64Y8wmfKl+1vEf3V/1cv+NlRf//rxTN7e0wqR5a6g47tHPluh18cM+b+Pr+F5WlLTq2dLCabjbQO+z3wpJau5QGEvUAvQfKt6zGESyY90oaU6I+JRpTYVvlsycS2+r7Kud1OkRDTfqaU0rvTyndklL6Vv+lpJFJQ1DKL+kqnUXzJkP0AX00NtTx0pa1vG/zBl7zxPbC2dScEr/BErNiKRSXKFxvbKhj0bzJrNi8gkThCbv7ulmxeUXRnk+qBNX0Hr1szXYYv4GmyXfT2/Cbov7/V5IHX3U6N1/wezx0XOsh77NszXZ6qKdvwIm5Uiq8FwJRYe+FrefSQ9ALZV1W40jazvgjzt7bxwv3HODqsz9oL18ZdWzpoLOxl231fVz+/epai3CoSd9/RMTvlTSSg0TE+RGxOiJ+FRF/U87nLoYDe6fy5Pbnl+8Dbv8u2NmZay9JHkr5JV2lM3/uJCbN/h4tU37OjW9fxCn7fkFDyv9s6mCJWbHMnzuJU46fyKxJE54a2rlg+gKCwhM21jWyYPqCoj2fVAnyeo/+yLYWPrJteDXopkzeRPOcf6Vp6p2Mm3MNUyZvKlF0+RlqD96ieZNpoJc6eov+XjiYwnshkCrsvXD2Qj4+5XhunjipYoZ29tveMI4tDc0mfGW2YvWtAKQIunsPPLVdDYaa9L2bQuK3NyJ2RcTuiNhVqqAioh74Z+BVwGnAmyLitFI9X7H1z1fq2nZWec5sdi6HzffDE+ty7SXJQym/pKu0miZs5ajJvyzMFamQs6mDJWbFNHF8IzOPnfDUcdumtXHypJOZ2TKTq19xddnmsUjlUk3v0btjNRE9RCTq63rZHavzDqnolq3ZzpmxhnfW/ztn9D58yCR8/txJfKr5G7x13NKyLG7fNq2NOd11TO2Nsr4Xrtmwkc3rHzns96ZHmsbznYnHVlTCV8lO6Ori5du31ex30blbdz41tLYhJeZu3Zl3SEM2pEIuKaWJpQ7kIAuBX6WU1gBExDeBi4AHyxzHiAw2X6mkb5hrl0LKFsDs7yUZI29O/V/Sd+3rGfMV7N741XuAKi1qMXshGxpO5Kj0JNMvvSHX12+5i2O0NLXQ0tRiwqeaVE3v0f29TSlBU0NTWXqbHulZVNYqxi9tWctlTX9LIz1008C6ljOB5w5631MbNnBqwwZOL9Pf7AX793P6/n207T9QluejczmtPb8hKqxIS1XrXM6Htz1WcYVvimln3Yv450230DGhgba9PXROelHeIQ3ZUKt3njXI7p3AupRSKeoNzwQ6B2yvB36rBM9TEk/NV0rQ2NBQ+jObredC1BUSvwoac14uVrA7tEosNd2v9cB7n7G9t+4o9nIU0yssTqlSVcsJnmp5j+7vbdpTl/jH3yt9b9NgVYxL3Uan7PsFfXGAOqA+ejll3y+Al5X0OYckj2Rh7VKCVBhWOsZOmJfMYIVvaqxNT2w/j08t/0t+68nVfK7uTN7Xfl7eIQ3ZUId3fhlYBlydXZYB3wT+NyJeUYK4YpB96Rl3iLgiIlZExIqtW7eWIISRO3i+Usk/6GYvhOlnwLFza/Ksikaoc3lFFEfJQ9eBLjY9uamqJlhLyl9zCqb01pWl532wUUEl13ouFPq2iEo6SZxHlczWc0lZW4zFE+YlUSFTNUpp/txJ7Dn+bm47upn3vf2Sij+ZNdBQk761QHtKaX5KaT7QBtxP4fTQ35cgrvXA7AHbs4CNA++QUroqpbQgpbRg6tSpJQhhdJ4xX6kcxh0Nx8w24dPT1i6tiOIoUFjv6Zb955SlsFHHlg4efnw1G3YPfxkESSqXRfMmUzdhLU2T/4umls7yzHecvZC1DSeypX76EU8ST+h7kim9W8pzwjCPZGGIbVGt5flzkU3V2DaE11c1mzi+kxMm3V1VCR8MPek7JaX0QP9GSulBCkngmtKExb3ASRFxYkQ0ARcDt5XouaTa1HouPZH/Gbdyr/dUKPWdKq/0tyQNUD9hHUfNvYamqXfSPOca6iesK8vz7q07im310w7/hTyb7zatd3N5RorkVCXzSG3RsaWDRxv72FqfKu4k4u59s9m440UVtwzKkF5fysVQk77VEfEvEfG72eXLFIZ2jgO6ix1UNk/wz4DvAw8BtwxMOvVsb+jewBu6N+QdhnK0tumzrG367NM7Zi/kM9Nncuuxk3M941bu9Z4KxRei8kp/S9IAhRNSfURAb+qprBNUg813K7FKrJI5cP3AA70HKuZvtHLdDlZvuoz1O15W8etfqnIMqZAL8DbgT4G/oDDf7sfAeykkfCWZwZhS+k/gP0txbGms+PX4Cfx6/ARel+OH6KJ5k1kQqzm77mFW1p3Jonm/XdLna5vWxrg0iz72cvUrPmdVTGkMOn3GMSN63PP7ustW/OrpE1SJxvoKO0GVzXeDVFlz/8psYjqZlBqAXnpTPRPTyXmHBPSvf1kP1JenSrxqwlCXbNgL/EN2OVhXUSOSVFPm1z3CTeOuJFIiGv6durpzKKzKUjr1TKCeCSZ8koYuK37VkMpTQbKiT1DNXkjd8WfCvp3w2msqqvetnLZtn8GeR/+Yhua1pD3z2NY6I++QgP71L3vpS9DY0FhR618+1DOT+3rncGGZliHR0A11yYaTgL+lsFD6+P79KaV5JYqr6s3t/nXeIUiVYe1S6rMhTPR112QJZ0mHVi1LSwxa/KrE71V5nKD67An7AFhypDuOO7pwGcPv14vmTYY7Z3Jg7xzGN1ZOcjV/7iROnvE1du89kc9f/LGKSa765/D3UM/NZVqGJA9HNQ11oGRlGWrUS4CPAJ+jMJxzMYMvqyCVXdV8oRirHCYkqdz27yr0Ug1nmGZW/IqUqC/Te9VpM44u+XOM1OLYDAwhOSyS3ftms3vviWVbqH4oKjW5Avhs1w5gB6dXUEyDzeGvpDYb64ZayGVCSukuIFJK61JKHwVeUrqwqp8lfqXMMEqES9KodS6HzffDE+uGV3myQopfjUWVXJikUsvzf3zyZ/j45M/kHcYzLJo3mQZ6qaOXxoa6iukZVcFQk759EVEHPBIRfxYRrwGmlTCuqlbJJX5r0eI7FrP4jsV5h6HDsISzpLJZuxRSX+H6MCtP/nr8BG4/9jjfq8pssMIk5VDONWTHgvlzJ/Gp5m/w1nFLa3ZoZzUb6vDOvwCagT8HPkGhl+/SEsVU9QaW+O1fJ6zU4/UrcViEyst5pJXPochSGbSeC1FXSPwcUl4V8ihMMlbmn5XbqQ0bOLVhQ0UNO1XBkHr6Ukr3ppS6UkrrU0qLU0p/mFJaVurgqtWC6QsKEx7LtE5YJQ+LkIaqmGdcD+ydypPbn+//gjQWzV4I08+AY+c6THMwx59ZuFSQ/rlzsyb9sGzJV7nXkJXydtievoi47XC3p5QuLG44taFtWhsn08hu+vj0K64ueS+f67UIqru3t5hnXFeu28GOzldBquPNnr2VxiYrT1adchcm6Z9/1kM6bO/iusbnlCWe4XLkSH6WpOl5hzAiRxreeQ7QCXwD+BlW7ByyFupooa4sZZgXzZsMkSD10djQ4MTZMai/t7cv1T8j0emij90UCgpV1BpQBylmxa9la7ZDqgOrh0kapmr9Mqfh659/dl/vHC687IN+TmjoFt+edwQjcqSk73jg5cCbgD8Cbge+kVJ6oNSBaejmz53EpNnfo3vPDK57/Z/7xjUGDdbbWz9hHavpJgGX33k5V5eh13k4Bp6lHOoZ16EonATpKwyvroKTIEvOL1dBdEnSQM4/K77+iqI35xyHnu2wSV9KqRe4A7gjIsZRSP5+FBEfTyl9sRwBamjqJzxKTFhN/YSLAN+8xprBJsGv2PyjshcUGqlinnH1JIikauFJH0nlcsRCLhExLiL+ELgBeBfwBeDbpQ5MQ9expYP9sZ7u2OYSEWPUYJPgy11QaLRObdjAG8bdU5QkrWnCVo6a/EsTPkmSJI5cyOU64Azge8DHUkr3lyUqDcuKzSuAVBU9OiqdieM7mTi+86lEp9wFhSRJla/rQBe7u3cfea53hVX4VHWwwEzlOlJP31uB5wHvBn4aEbuyy+6I2FX68DQUhR6cqJoeHZVPC3XMoMGET5JEx5YOVu9YzYauDY4MGqLTZhzNaTOOzjsMadSONKdvSOv4KV9t09oYl2bRx16ufsXn/IIvSZKeZcXmFaTCbG9HBg2R8y5VK45UvVNVop4J1DPBN29JkjSowlzvIJEcGSSNMSZ9NcKhB5Ik6XDaprVx8qST2d29m0+f+2lPFEtjiEmfJEnSGNHS1EJLU8sRE74hF3wpktM/+OOSP4c0ljlnT5IkSU+x4ItUe+zpk1RzWg+8N+8QJKlqjZWCLx+f/BkAbs45DqkcTPpU/R67L+8IKsLknv3U00vH/TfRdsYf5R2OJKlKjZWCL64pp7HEpE9Vr4s+dtNXtnkHlajj/pu4d0Id3VHPz+79FFdD1SV+xTzj6ge5JI2cBV+k2uOcvhJZkqazJE3PO4ya17Glg9V0s4HeMT3vYMWa79MdQV8E3VHYliRppFqaWphx1AwTPqlG5JL0RcTrI+KBiOiLiAUH3faBiPhVRKyOiFfmEZ+qR2HeARBPzzsYixbMeyWNKVGfEo2psF1JTp9xDKfPOOaw97n5HefYQydJklQCefX03Q/8IXD3wJ0RcRpwMXA6cD7w5YioL394RbB/F+zshM7leUdS0wrzDoBETc87OJK2M/6Is/f28cI9B7j67A9W3dBOSaoIi28vXCSpxuSS9KWUHkoprR7kpouAb6aU9qeUfgP8ClhY3uiKoHM5bL4fnlgH111o4ldCbdPaOJlGZlLP1a+4ekwPQ9nXfBRPNh9twidJkqRnqLRCLjOBZQO212f7niUirgCuAJgzZ07pIxuOtUsh9RWu9x4obM8ube665PwlJT1+JWuhjhbqxnTCJ0mSJB1KyZK+iPghcPwgN30opfTdQz1skH1psDumlK4CrgJYsGDBoPfJTeu5EHWFxK++qbAtSZKUs7F8klgay0qW9KWUXjaCh60HZg/YngVsLE5EZTR7IUw/A/bthNdeU/JePkmSJEk6lEpbsuE24OKIGBcRJwInAdU5IW7c0XDMbBM+SZIkSbnKa8mG10TEeuAc4PaI+D5ASukB4BbgQeAO4F0ppd48YpQkSZKkWpBLIZeU0q3ArYe47UrgyvJGJEmSJEm1qdKqd0qSJI2Oa+1J0jOY9EnSADe/45y8Q5AkSSqqSivkIkmSJEkqInv6pBq2JE3POwRJkiTlzKRPkiRJz+Ai7lJtcXinqt5z9u3lgiceh87qXNJRkiRJKiWTPlW3zuW8b/MGXvPEdrjuQhM/SZIk6SAmfapua5fSkBL1AL0HYO3SvCPSUC25oHCRJElSSTmnT9Wt9Vx6IiAl6uuboPXcvCPKz/Fn5h2BJEmSKpBJn6rb7IWMm34m7NsJr70GZi/MOyJJkiSpopj0qfqNO7pwMeGTJEmSnsU5fZIkSZJUw0z6JEmSJKmGmfRJkiRJUg1zTp9UI5acvyTvECRJklSB7OmTJEmSpBpmT5+k0lt8e94RSJIkjVkmfVItM9mSJEka8xzeKUmSJEk1zKRPkiRJkmqYwztLxWF1kiRJkiqAPX2SJEmSVMNySfoi4jMR8XBE/DIibo2IYwfc9oGI+FVErI6IV+YRnyRJkiTVirx6+n4AnJFSej7wv8AHACLiNOBi4HTgfODLEVGfU4ySJEmSVPVySfpSSnemlHqyzWXArOz6RcA3U0r7U0q/AX4FLMwjRkmSJEmqBZUwp+8y4HvZ9ZlA54Db1mf7JEmSJEkjULLqnRHxQ+D4QW76UErpu9l9PgT0ADf2P2yQ+6dDHP8K4AqAOXPmjDpeSZIkSapFJUv6UkovO9ztEXEp8PvAS1NK/YndemD2gLvNAjYe4vhXAVcBLFiwYNDEUGOEy2NIkiRJh5RX9c7zgfcDF6aU9gy46Tbg4ogYFxEnAicBy/OIUZIkSZJqQV6Ls38JGAf8ICIAlqWU3plSeiAibgEepDDs810ppd6cYpQkSZKkqpdL0pdSeu5hbrsSuLKM4UiSJElSzaqE6p2SJEmSpBIx6ZMkSZKkGmbSJ0mSJEk1zKRPkiRJkmqYSZ8kSZIk1TCTPkmSJEmqYSZ9kiRJklTDTPokSZIkqYaZ9EmSJElSDTPpk5SP/btgZyd0Ls87EkmSpJpm0iep/DqXw+b74Yl1cN2FJn6SJEklZNInqfzWLoXUV7jee6CwLUmSpJIw6ZNUfq3nQmRvP/VNhW1JkiSVREPeAUgag2YvhOlnwL6d8NprCtuSJEkqCZM+SfkYd3ThYsInSZJUUg7vlCRJkqQaZtInSZIkSTXMpE+SJEmSaphJnyRJkiTVMJM+SZIkSaphJn2SJEmSVMNM+iRJkiSphuWS9EXEJyLilxHRERF3RsQJA277QET8KiJWR8Qr84hPkiRJkmpFXj19n0kpPT+l1Ab8B/BhgIg4DbgYOB04H/hyRNTnFKMkSZIkVb1ckr6U0q4Bm0cBKbt+EfDNlNL+lNJvgF8BC8sdnyRJkiTVioa8njgirgQuAXYC52W7ZwLLBtxtfbZPkiRJkjQCJevpi4gfRsT9g1wuAkgpfSilNBu4Efiz/ocNcqg0yD4i4oqIWBERK7Zu3VqaX0KSJEmSqlzJevpSSi8b4l1vAm4HPkKhZ2/2gNtmARsPcfyrgKsAFixYMGhiKEmSJEljXV7VO08asHkh8HB2/Tbg4ogYFxEnAicBy8sdnyRJkiTVirzm9H06Ik4G+oB1wDsBUkoPRMQtwINAD/CulFJvTjFKkiRJUtXLJelLKb32MLddCVxZxnAkSZIkqWbltU6fJEmSJKkMTPokSZIkqYaZ9EmSJElSDTPpkyRJkqQaZtInSZIkSTXMpE+SJEmSaphJnyRJkiTVMJM+SZIkSaphJn2SJEmSVMNM+iRJkiSphpn0SZIkSVINM+mTJEmSpBpm0idJkiRJNcykT5IkSZJqmEmfJEmSJNWwhrwDkDRGLb497wgkSZLGBHv6JEmSJKmGmfRJkiRJUg0z6ZMkSZKkGmbSJ0mSJEk1zKRPkiRJkmqYSZ8kSZIk1TCTPkmSJEmqYbkmfRHx3ohIETFlwL4PRMSvImJ1RLwyz/gkSZIkqdrltjh7RMwGXg48OmDfacDFwOnACcAPI+J5KaXefKKUJEmSpOqWZ0/f54C/BtKAfRcB30wp7U8p/Qb4FbAwj+AkSZIkqRbkkvRFxIXAhpTSLw66aSbQOWB7fbZPkiRJkjQCJRveGRE/BI4f5KYPAR8EXjHYwwbZlwbZR0RcAVwBMGfOnBFGKUmSJEm1rWRJX0rpZYPtj4gzgROBX0QEwCzg5xGxkELP3uwBd58FbDzE8a8CrsqOuTUi1hUveg3RFGBb3kGMYbZ/fmz7/Nj2+bHt82X758e2z49tPzxzD3VDpDRoR1rZRMRaYEFKaVtEnA7cRGEe3wnAXcBJFnKpTBGxIqW0IO84xirbPz+2fX5s+/zY9vmy/fNj2+fHti+e3Kp3Dial9EBE3AI8CPQA7zLhkyRJkqSRyz3pSym1HrR9JXBlPtFIkiRJUm3JdXF2Vb2r8g5gjLP982Pb58e2z49tny/bPz+2fX5s+yLJfU6fJEmSJKl07OmTJEmSpBpm0iciYm1E3BcRHRGxYsD+/xMRqyPigYj4+2zfyyNiZXb/lRHxkgH3n5/t/1VEfCGyNTkiYlxE3Jzt/1lEtJb9l6xQxWj7iGiOiNsj4uHs/p8ecBzb/jCK9dof8LjbIuL+Adu2/yEU8X2nKSKuioj/zf4HXpvtt+0PoYht/6Zs/y8j4o6ImJLtt+0PY5jtvzC7X0dE/CIiXjPg/n7mDlMx2t7P3JEp1ut+wOP8vB2ulJKXMX4B1gJTDtp3HvBDYFy2PS372Q6ckF0/A9gw4DHLgXOAAL4HvCrb/6fAV7LrFwM35/07V8qlGG0PNAPnZdebgKW2ffnaf8Dj/pDCkjP3D9hn+5e47YGPAZ/Mrtf1H9O2L23bUygEt2VAe/898FHbvujt3ww0ZNdnZG3ev+1nbg5tj5+5ubX9gMf5eTuSv0HeAXjJ/3KIf8RbgJcd4XEBbAfGZf+UDw+47U3AV7Pr3wfOya43UFhkM/L+vSvhUoy2H+S2zwOX2/bla3+gBfgxcNpBH0K2f+nbvhM4apD72fYlbHugEdhKYSHgAL4CXGHbl7T9TwQ2Z23qZ25ObT/IbX7mlrHt/bwd+cXhnQJIwJ3Z0J0rsn3PA87Nusj/JyLOHuRxrwVWpZT2AzOB9QNuW5/tI/vZCZBS6gF2ApNL8HtUo2K0/VMi4ljg1cBd2S7b/vCK1f6fAP4B2HPQ/Wz/Qxt122evd4BPRMTPI+LfImJ6ts+2P7RRt31KqRv4E+A+YCOFL2D/mt3Ptj+8YbV/RPxWRDxAoa3fmbWpn7kjU4y2Z8Dtx+Jn7lAVq+39vB2h3NfpU0X4nZTSxoiYBvwgIh6m8NqYBCwCzgZuiYh5KTuFEhGnA38HvCI7Rgxy3DSE28a6YrQ92f4G4BvAF1JKa/p3D/Kctv3TRt3+EdEGPDel9JeDzCGw/Q+tGK/9BmAW8JOU0nsi4j3AZ4G3YtsfTjFe940Ukr52YA3wReADwCex7Y9kWO2fUvoZcHpEnApcFxHfw8/ckRp126eU9oGfuSNQjNf9Kfh5O2L29ImU0sbs5xbgVmAhhbOG387+8ZYDfUD/JP1Z2f0uSSn9OjvMegpfvvrNonD2t/+22dljG4BjgMdL+TtViyK1fb+rgEdSSv80YJ9tfxhFav9zgPkRsZbCkJPnRcSPstts/0MoUttvp3C299Zs+9+As7Lrtv0hFKnt27Jj/DpLDG8Bfju7zbY/jOG2/4DHPQQ8SWFupZ+5I1Cktu/nZ+4wFKnt/bwdBZO+MS4ijoqIif3XKZzFvR/4DtBfHfJ5FCYrb8uGMtwOfCCl9JP+46SUNgG7I2JRRARwCfDd7ObbgEuz668D/qv/7PFYVqy2z+73SQpvcH9x0NPY9odQxNf+v6SUTkgptQIvBP43pfTi7GbbfxBFbPsE/Dvw4mzXS4EHs+u2/SCK+L6zATgtIqZm2y8HHsqu2/aHMIL2PzH7AktEzAVOBtb6mTt8xWr7bNvP3GEo4uvez9vRSBUwsdBLfhdgHvCL7PIA8KFsfxNwA4V/yp8DL8n2/18KZ1w6Blz6qy0tyO7/a+BLZBNogfEUzsD/ikK1sXl5/96VcClW21M4w5sofOHq3/9227487X/QMVt55sRy27/EbU+hkMjdwC8pzKuZY9uXre3fmb3v/JJC8j3Zti96+781u19Htv8PBhzLz9wc2h4/c3Nr+4OO2Yqft8O69L9BSJIkSZJqkMM7JUmSJKmGmfRJkiRJUg0z6ZMkSZKkGmbSJ0mSJEk1zKRPkiRJkmqYSZ8kqWZFxOSI6Mguj0XEhux6V0R8uUTP+RcRcckwH/PTQ+y/NiJel13/ZkScVIwYJUljS0PeAUiSVCoppe1AG0BEfBToSil9tlTPly0ofBlw1nAel1L67SHc7V+AvwYuH0FokqQxzJ4+SdKYExEvjoj/yK5/NCKui4g7I2JtRPxhRPx9RNwXEXdERGN2v/kR8T8RsTIivh8RMwY59EuAn6eUerLH/CgiPhcRd0fEQxFxdkR8OyIeiYhPDoinK/sZEfGliHgwIm4Hpg049lLgZVliKUnSkJn0SZIEzwEuAC4CbgD+O6V0JrAXuCBL/L4IvC6lNB/4GnDlIMf5HWDlQfsOpJReBHwF+C7wLuAM4G0RMfmg+74GOBk4k0KP3lM9gCmlPuBXwAtG8XtKksYgzxZKkgTfSyl1R8R9QD1wR7b/PqCVQiJ2BvCDiCC7z6ZBjjMDeOigfbcNONYDKaVNABGxBpgNbB9w3xcB30gp9QIbI+K/DjrWFuAEnp1YSpJ0SCZ9kiTBfij0pkVEd0opZfv7KHxWBoWE7ZwjHGcvMH6wY2fH2j9gf/+xD5YG2ddvfPYckiQNmcM7JUk6stXA1Ig4ByAiGiPi9EHu9xDw3FE8z93AxRFRn80ZPO+g258HPDCK40uSxiCTPkmSjiCldAB4HfB3EfELoIMB8+0G+B6FIZrDfors563AIxSGgv4L8D/9d4iI6cDe/uGhkiQNVTw9gkWSJI1WRNwK/HVK6ZEh3n8yhYqfc49wv78EdqWU/rUIYUqSxhB7+iRJKq6/oVDQ5Ygi4gTgHmAoawc+AVw38rAkSWOVPX2SJEmSVMPs6ZMkSZKkGmbSJ0mSJEk1zKRPkiRJkmqYSZ8kSZIk1TCTPkmSJEmqYSZ9kiRJklTD/n9jFNVv5P74sAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "selected = light_curves[labels_new == 0][0]\n",
    "plt.figure(figsize=(15,5))\n",
    "\n",
    "for i, passband in enumerate(['g', 'r', 'i', 'z']):\n",
    "    plt.errorbar(selected[:,0], \n",
    "                 selected[:,i+1], \n",
    "                 yerr=selected[:,i+5], \n",
    "                 label=passband, \n",
    "                 fmt='.')\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('Time (mjd)')\n",
    "plt.ylabel('Magnitude')\n",
    "plt.title('Supernovae type I')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Data preprocessing using tf.Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "INP_DIM = 9 # Number of attributes e.g., mjd, magnitudes, errors...\n",
    "NUM_CLS = len(np.unique(labels_new))\n",
    "BATCH_SIZE = 128 # Training batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_generator(list_of_arrays, labels):\n",
    "    for i, j in zip(list_of_arrays, labels):\n",
    "        yield {'lightcurve': i,\n",
    "               'label'     : int(j),\n",
    "               'mask'      : np.ones(i.shape[0])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardize(light_curve):\n",
    "    lc_means = tf.reduce_mean(light_curve, 0)\n",
    "    lc_std   = tf.math.reduce_std(light_curve, 0)\n",
    "    return tf.math.divide_no_nan(light_curve - tf.expand_dims(lc_means, 0), lc_std)\n",
    "#     return light_curve-lc_means\n",
    "\n",
    "def min_max_scaler(light_curve):\n",
    "    lc_min   = tf.reduce_min(light_curve, 0)\n",
    "    lc_max   = tf.reduce_max(light_curve, 0)\n",
    "    return tf.math.divide_no_nan(light_curve - lc_min, lc_max-lc_min)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(input_dict):\n",
    "    light_curve = standardize(input_dict['lightcurve'])\n",
    "    cat_label   = tf.one_hot(input_dict['label'], NUM_CLS)\n",
    "    bool_mask   = tf.cast(input_dict['mask'], tf.bool)\n",
    "    return {'x': light_curve, 'mask':bool_mask}, cat_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_class_dataset(lightcurves, labels):\n",
    "    dataset = tf.data.Dataset.from_generator(lambda: create_generator(light_curves, labels),\n",
    "                                             output_types= {'lightcurve':tf.float32,\n",
    "                                                            'label':tf.int32,\n",
    "                                                            'mask':tf.int32},\n",
    "                                             output_shapes={'lightcurve':(None, INP_DIM),\n",
    "                                                            'label':(),\n",
    "                                                            'mask':(None)})\n",
    "    return dataset\n",
    "\n",
    "def create_dataset(light_curves, labels, balanced=False):\n",
    "    if balanced:\n",
    "        datasets = []\n",
    "        for y in np.unique(labels):\n",
    "            cls_samples = light_curves[labels==y]\n",
    "            cls_labels  = labels[labels==y]\n",
    "            datasets.append(load_class_dataset(cls_samples, cls_labels))\n",
    "        dataset = tf.data.Dataset.sample_from_datasets(datasets)\n",
    "    else:\n",
    "        dataset = load_class_dataset(light_curves, labels)\n",
    "        \n",
    "    dataset = dataset.map(preprocess).cache()\n",
    "#     dataset = dataset.shuffle(1000, reshuffle_each_iteration=True) # shuffling in chunks of 1000 objects \n",
    "    dataset = dataset.padded_batch(BATCH_SIZE, \n",
    "                                   padded_shapes=({'x':[None, INP_DIM], 'mask':[None]}, [NUM_CLS]),\n",
    "                                   drop_remainder=True\n",
    "                                  )\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_rest, \\\n",
    "y_train, y_rest = train_test_split(light_curves, labels_new, test_size=0.5, random_state=42)\n",
    "\n",
    "x_val, x_test, \\\n",
    "y_val, y_test = train_test_split(x_rest, y_rest, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches = create_dataset(x_train, y_train, balanced=True) \n",
    "valid_batches = create_dataset(x_val, y_val, balanced=True)\n",
    "test_batches  = create_dataset(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-31 14:45:03.649696: W tensorflow/core/kernels/data/cache_dataset_ops.cc:768] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset  will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAEICAYAAACQ18pCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA2hUlEQVR4nO3de3xcZb3v8c9vcmmapi1pm/RGmlBu5Z7SUIsIGwURRcUjKqJCBbl4l73dR1GPHt37cMTL9uje260UBNkKCl64bBW5bUFUSklpoEBboKX30rQlbZKmSSaZ3/ljrSnTdJLMSmYySef7fr3ympk1a63nt2Yya37zPM96HnN3RERERCRzsXwHICIiIjLWKIESERERiUgJlIiIiEhESqBEREREIlICJSIiIhKREigRERGRiJRAjTFmtt7Mzs1w3Y+a2V8yXPfrZvbzIcY05G377OenZvZ/wvtnmtma4e4zZd/3m9ni8H7Gr0uG+/6wmT2Yrf2JiMCB50QZfZRAZUGY1Owzs3YzazGz35tZTYbb1pmZm1lxruMcS9z9cXc/drD1Mk3e3P3t7n7bcONK9365++3uft5w9y1S6KL8QBxmOVn50SeFTQlU9rzL3SuAmcB24N/yHI8AFtD/uYiIZJW+WLLM3TuBXwPHJ5eZ2QVmtsLMWs1sk5l9PWWTP4e3u8MarNPDba4ys1Vm1mZmL5jZqSnb1JvZs2a2x8zuNLOyTGIzsx+E5bea2XIzO7PPKmXh/trM7GkzOyVl21lm9hsz22Fmr5jZZ/spo8zMfm5mu8xst5k9ZWbT+1l3flhOm5ndCZSlPHe2mW1OefxFM9sSrrvGzM4xs/OBLwMXh6/dM+G6j5rZ9Wb2V6ADmBsuu/LA4u3fwtdwtZmdk/LEAb+C+/xaPej96tskaGZvDI97T3j7xpTnHjWzfzazv4bH8qCZTUv3+ogUsuTnysy+G9bsv2Jmb095/lEz+6aZLQs/a/ea2ZTwuQPOH+Gy9WZ2bn/njTTlH3TOCZcvNLMnwvPbNjP7dzMrTdnOzeyTZvZSuO0/m9mR4TatZnZXcv1knGb2ZTPbGcb44QFek3eaWVNY9t/M7OThvMYyPEqgsszMyoGLgaUpi/cClwGHARcAnzCz94TPnRXeHubuFe7+hJm9H/h6uM0k4N3ArpT9fQA4HzgCOBn4aIbhPQXUA1OAO4Bf9Um+LgR+lfL8PWZWYkENzn8BzwCzgXOAa83sbWnKWAxMBmqAqcDHgX19VwpPIPcAPwvL+xVwUbqgzexY4NPAae4+EXgbsN7d/wj8X+DO8LU7JWWzS4GrgYnAhjS7fQOwDpgG/G/gt8mT7yAOer/6xDoF+D3wrwTH/z3g92Y2NWW1DwGXA9VAKfCPGZQrUojeAKwh+Jx+G/iJmVnK85cBVwCzgB6Cz92ABjlvAP2fc8Kne4G/D2M6neB8+Mk+uzgfWAAsAr4ALAE+THBePBG4JGXdGeG+ZhOcP5eE5feN6VTgFuAagnPLjcB9ZjZusGOW3FAClT33mNluoBV4K/Cd5BPu/qi7r3T3hLs/C/wC+LsB9nUl8G13f8oDL7t7ahLwr+6+1d1fI0hs6jMJ0N1/7u673L3H3f8FGAekflCXu/uv3T1O8MVfRnACOA2ocvd/cvdud18H3AR8ME0xcYIP91Hu3uvuy929Nc16i4AS4PvuHnf3XxMkeOn0hrEeb2Yl7r7e3dcOcrg/dffnw2ONp3m+OaXsOwlO0hcMss9MXAC85O4/C8v+BbAaeFfKOre6+4vuvg+4iwzfP5ECtMHdb3L3XuA2gi4SqTXaP3P359x9L/BV4ANmVpSFcvs954TntKXh53s9QSLT93z+LXdvdffngeeAB919nbvvAe4H5vdZ/6vu3uXujxH8APtAmpiuAm509yfDc+ttQBfBuVTyQAlU9rzH3Q8j+NB9GnjMzGYAmNkbzOxPFjR/7SGolRmo2aYGGChBeDXlfgdQkUmAZvZ5C5oF94TJ3uQ+cWxK3nH3BLCZ4JddLTArrDbeHW77ZQ48kSX9DHgA+KWZbTWzb5tZSZr1ZgFb/MDZrNPVFOHuLwPXEtTKNZvZL81s1iCHu2mQ59OVPdg+MzGLg49jA8Gvy6QhvX8iBWj/Z8XdO8K7qZ+X1M/5BoIfZcNuEh/onGNmx5jZ78zsVTNrJajN6lvm9pT7+9I8Tj2GljABTD2OdOeiWuDzfc7DNf2sKyNACVSWhb8MfkvwC+ZN4eI7gPuAGnefDPwYSFZD+8F7YRNwZDbjsqC/0xcJftlUhsnenpQ4IPgwJtePAYcDW8N4XnH3w1L+Jrr7O/qWE9bofMPdjwfeCLyToJq9r23A7D7V8XP6i9/d73D3NxGcRBz4VvKp/jbpb1+hdGVvDe/vBcpTnpsRYb9bwxhTzQG2DLKdiESXerXzHIIa8J30+QyHtVJVKesO9jke6JzzI4Ja5aPdfRLBj0lLv5eMVJrZhD7HsTXNepuA6/uch8vDWm7JAyVQWWaBC4FKYFW4eCLwmrt3mtlCgj4wSTuABDA3ZdnNwD+a2YJwf0eZWd8v5agmEvQR2AEUm9nXCPpXpVpgZu+14BL9awmqh5cCy4DWsFPleDMrMrMTzey0NMf/ZjM7KTxhtRKc0HrTxPNEGM9nzazYzN4LLEwXuJkda2ZvCdv6Owl+wSX3uR2os+hX2lWHZZeEfc6OA/4QPtcEfDB8rgF4X8p26d6vVH8AjjGzD4XHdTHBBQW/ixifiAzuI2Z2fNj39J+AX4fNfS8SXBRzQVgD/r8IWgeSBjxvDHLOmUhwbms3s3nAJ7JwHN8ws9Lwh+47CfqE9nUT8PGwRcPMbEJ4fBOzUL4MgRKo7PkvM2sn+GBdDywO278h6GD4T2bWBnyNoN8LsL9a+nrgr2G17CJ3/1W47A6gjaCzdSYdnAfyAEHb+4sEVcSdHNzMdS9BB/gWgk7Y7w1rlHoJ+vDUA68Q/MK7maAJsK8ZBFchthIkkI8BB4234u7dwHsJOsC3hOX+tp/YxwE3hOW+SpD8fDl8Lnmi2WVmT/ezfTpPAkeH+7weeJ+7Jzvqf5WgBrAF+AbB+5CM+6D3q89x7SI4AX6eoOP/F4B3uvvOCLGJSGZ+BvyU4LxQBnwWIOxr9EmC89QWghqp1KvyBjtvDHTO+UeCH8FtBEnNncM8hlcJzjVbgduBj7v76r4ruXsjQT+ofw/Xf5nMLyCSHLADu4GIiIiMfmb2KPBzd78537EMlZmdTXAMh+c5FBkC1UCJiIiIRKQESkRERCQiNeGJiIiIRKQaKBEREZGIigdfJXumTZvmdXV1I1mkiOTZ8uXLd7p71eBrjm46f4kUnoHOXyOaQNXV1dHY2DiSRYpInplZ2hHmxxqdv0QKz0DnLzXhiUjBMrPDzOzXZrY6nObo9HzHJCJjw4jWQImIjDI/AP7o7u8zs1IOnMJHRKRfSqBEpCCZ2STgLMLRnMPR8bvzGZOIjB1qwhORQjWXYG7DW81shZnd3GdSV8zsajNrNLPGHTt25CdKERmVlECJSKEqBk4FfuTu8wnmS7sudQV3X+LuDe7eUFU15i8kFJEsUgIlIoVqM7DZ3Z8MH/+aIKESERmUEigRKUju/iqwycyODRedA7yQx5BEZAxRJ3IRKWSfAW4Pr8BbB1ye53hEZIxQAiUiBcvdm4CGbO93w6WXAVD7s//M9q5FZJQYtQlU3XW/z3cI0o/1N1wwIuXof2B0Gqn3X0RkNFMfKBEREZGIlECJiIiIRKQESkRERCQiJVAiIiIiESmBEhEREYlICZSIiIhIREqgRERERCJSAiUiIiISUUYJlJkdZma/NrPVZrbKzE43sylm9pCZvRTeVuY6WBEREZHRINMaqB8Af3T3ecApwCrgOuARdz8aeCR8LCIiInLIGzSBMrNJwFnATwDcvdvddwMXAreFq90GvCc3IYqIiIiMLpnUQM0FdgC3mtkKM7vZzCYA0919G0B4W53DOEVERERGjUwSqGLgVOBH7j4f2EuE5jozu9rMGs2scceOHUMMU0Rk7Ohtbye+dSsdK1bkOxQRyZFMEqjNwGZ3fzJ8/GuChGq7mc0ECG+b023s7kvcvcHdG6qqqrIRs4jIqNWxYgVdq1cT37KFjZdfoSRK5BA1aALl7q8Cm8zs2HDROcALwH3A4nDZYuDenEQoIjKGdCx7CtwB8Hg8eCwih5ziDNf7DHC7mZUC64DLCZKvu8zsY8BG4P25CVFEZOwoX3gamIE7VlISPBaRQ05GCZS7NwENaZ46J6vRiIiMceXz5zNu3jwSra3M+u53KJ8/P98hiUgOZFoDJSIiGSqqqKCookLJk8ghTFO5iIiIiESkBEpEREQkIiVQIiIiIhEpgRIRERGJSAmUiBQsMysKp6j6Xb5jEZGxRQmUiBSyzwGr8h2EiIw9SqBEpCCZ2eHABcDN+Y5FRMYeJVAiUqi+D3wBSPS3giZDF5H+KIESkYJjZu8Emt19+UDraTJ0EemPEigRKURnAO82s/XAL4G3mNnP8xuSiIwlSqBEpOC4+5fc/XB3rwM+CPy3u38kz2GJyBiiufBERLJs78YV+Q5BRHJMCZSIFDR3fxR4NKv7fNvkbO5OREYhNeGJiIiIRKQESkRERCQiJVAiIiIiESmBEhEREYlICZSIiIhIREqgRERERCJSAiUiIiISkRIoEZECtOHSy9hw6WX5DkNkzFICJSIiIhKREigRERGRiDKayiWcsbwN6AV63L3BzKYAdwJ1wHrgA+7ekpswRUREREaPKDVQb3b3endvCB9fBzzi7kcDj4SPRURERA55w2nCuxC4Lbx/G/CeYUcjIiIiMgZkmkA58KCZLTezq8Nl0919G0B4W51uQzO72swazaxxx44dw49YREREJM8y6gMFnOHuW82sGnjIzFZnWoC7LwGWADQ0NPgQYhQREREZVTKqgXL3reFtM3A3sBDYbmYzAcLb5lwFKSIiIjKaDJpAmdkEM5uYvA+cBzwH3AcsDldbDNybqyBFRERERpNMmvCmA3ebWXL9O9z9j2b2FHCXmX0M2Ai8P3dhioiIiIwegyZQ7r4OOCXN8l3AObkISkRERGQ000jkIiIiIhEpgRIRERGJSAmUiIiISERKoEREClBvezvxrVvpWLEi36GIjElKoESkIJlZjZn9ycxWmdnzZva5fMc0UjpWrKBr9WriW7aw8fIrlESJDIESKBEpVD3A5939OGAR8CkzOz7PMWXF5X+8nMv/eHm/z3csewo8mBjC4/HgcYG6+MYnuPjGJ/IdhoxBSqBEpCC5+zZ3fzq83wasAmbnN6qRUb7wNAjG9sNKSoLHIhKJEigRKXhmVgfMB57ss/yQnAy9fP58xs2bR8ns2cy59RbK58/Pd0giY44SKBEpaGZWAfwGuNbdW1Ofc/cl7t7g7g1VVVX5CTBHiioqKJk165BOntQ8J7mkBEpECpaZlRAkT7e7+2/zHY+IjB2ZzIUnInLIsWCCz58Aq9z9e/mOR7Lgx2dC5x646GaoWZjRJm2dcVo7e7jjyY20dHSzaO5UFtRW5jhQORQogRKRQnUGcCmw0syawmVfdvc/5C8kGbJNy2D7c+AJuO3dsPi+QTdZvqGF1a+2kXD48t0rASgriXH7lYuURMmglECJSEFy978Alu84JEvWPx4kTwC93cFj3jjgJkvX7SLhBy6L9yRYum6XEigZlPpAiYjI2Fd3Jlj4lVZUGjwexKK5U4mlpNAxg5LiGIvmTh1028HG2pLRJ9vvmWqgRERk7KtZCNNPDPpAlU2Gh79BW+cXae3sYfmGlrQ1SgtqK5k3YyKtnT188uyjIvWBan/1GdpI0NTcRH11fQ4OSEY71UCJiMih4eOPw7XPsjxxNF9+9SxWbWtjc8s+PnzzUpZvaEm7ycSyEmYfNp4PvWEOn3rzURklT03NTawhzhZ6uerBq2hqbsrygRzaNlx6GRsuvSzfYQybEigRETlkLN/QwiUb380v9hxPsntTsl9TJjIZO+qK3/99sG+DeCJO4/bGYcU8FhwqSU82qQlPREQOGUvX7aKHIjylfiDTfk0AX9v1P8N7f+l3nam9u3i12MGNkqISGqY37O9bc+v5tw459tFs9WurAajNcxyjiRIoERE5JCzf0MKW3fsotgS9DgliVE0s5UcfacjaVXVNzU1sLw7qtmI4Xzjq/UEfqFdXZmX/MnYogRIRkTFv+YYW3v/jv5FwWGgv8bEJj/NA+QVsqThpwOTpzmtOj1RO4/ZGHAczzJ09T/4IZp/NrPZ2TujqDMajynAQTxnb1AdKRETGvGQfp1PtRW4r/RZv7X2Mb7b9L47ufiHSfsYn9jKttzlIhNJomN5ACVDkTok7Dfv2wTN38LWdr3JxW0swiGc/2w5FU3MTN6+8WR3VRyHVQImIFKDan/1nvkPIqkVzp1JaHON0X0UJPcRwiolzQvezme9k0zLqel7B8NdHM+9Tm1Tf1c2SbdtZUVbGaZ2d1PcYYBTjFAG9PV3cddftXPL54ddCNTU3cdWDV9Hd201pUSk3nXfTiA+ZkOzb9cERLXVsUA2UiIiMeQtqK/mXrQ9S1VpKr0Ev0EMJz5eenPlO1j8OOAb4/tHMD15nflc3V+1p5eTObnYcdRGccgk92NDKHEDj9kY6eztJkDikrvYr3fsUk4r+HLmmbrQNXppxAmVmRWa2wsx+Fz6eYmYPmdlL4a3GvRcRkbw5sXsXb25p5TszZnH3YVPZUjybS9oyvypuddkpdHkpPR6jM1HE6rJT0q7THa7TRSmfen4eyxNH80/TZnDnxEp2Fk3lM7u/lZVmvIbpDZQVlVFkRZTEgqv9xrxNy5h+wjYOq81+c+dIi9KE9zlgFTApfHwd8Ii732Bm14WPv5jl+ERERDKSvNR+bdl41paN532Tp0fa/pH2Ov67+0u8IbaaZX4cb26vY16adR7p/hKLYqtZmjiOFX4US9ft4qXSMgAuattG8QBNgH01PXcHjeseoGHu26g/8UMHPFdfXc9N591E4/ZGGqY37G++a2puOmjZmLH+cSzmmPH6nIVjtNN9RgmUmR0OXABcD/xDuPhC4Ozw/m3AoyiBEhGRsejWC1i0bzo/8HezovcoSktK+FKasaMWzZ3K//Mjebr3GABKwzGmbl0/i0mJLTw/roRTu7ozSg6anruDK5+6nrgZpbuWcxOkTaJSk6TR0C9qWOrOxBMG5ljpwXMWJgfrHAt99DJtwvs+8AUgkbJsurtvAwhvq9NtaGZXm1mjmTXu2LFjOLGKiIj0a/3UXh6ZF6f2yX1UNe3j68vW84Vn1md8BduC8dv5v+W/4NJxj3P7lYv6nT/vhvI7eGvRcqonjuMXVy2iaPwGumwzT5YluHJGNd+YWknT+PJBJzRuXPcAcTMSZsQteNxX3xHAG7c30hUfuX5R7d3tbNu7jSeOiPPIvPjwrwasWcj2bW9id8txGdXQjWaD1kCZ2TuBZndfbmZnRy3A3ZcASwAaGhp8kNVFRA5JI/nL+oP/9nxw5/ycF5VTUV6zpuYmbvy7buIxwGLgBLUcDn968KrBa2q6WqFzD/dXV/HcuB6+McDYUb+afi0vbGvl+GkVLKit5OaVvwE8mNqFGL+eOJH7Jpfwk3GlDFAiDXPfRsmuRnqAEg8e91U67jXKxu/cP75Uw/QGStzoSTglJbntF9XU3MSaljU4zm8aIryWg+jumkJ31xQqRzh5au9upy3elrUJoDNpwjsDeLeZvQMoAyaZ2c+B7WY20923mdlMoHnY0YiIHOKGmkj1nYdsLDRxDNVQ5lxr3N5IT4ygXcVfv/WYEe/tpnF7Y/9fmpuWwfbnwBP80DbxqemnRip7oh+LewkQB8AMuhI93Lv68QG/qOtP/BDXfu8/2HR4G2//wBcPar5j0zKmz/wLFnu9T1V9zUKuf/poVla28dbPfDunzXf7Bw0Necz213qNqWZDDkwGr8pCEggZNOG5+5fc/XB3ryMYCuK/3f0jwH3A4nC1xcC9w4pERERkiBqmN1Cc4PWOJuFtzJ2SRC8NVp52uw2XXkbLtz+LJ4INni8txtgZqalq566ZdGz8GPGWN4AX4W7gxfR0HDHotlM3TqP+b0ccnDxB+g7XwHF7JvCB9TNynsQ0TG/AsP2PLeFj9mrA1GQwW02fwxlI8wbgLjP7GLAReP+woxERERmC+up6rnmslLVVCcq7oGxaB0dWtLGnqIiGrjj1Ldv63bZz3zQ8YTxTVsI1M6rpsgTPDFBL8dFffYuq7RvYMb0WrrmHRXOnknjwcLr2zSG+Zz7F5esoih/Ne95yxqBxr5tQzYsTZ1K9oeXgPld1Z5JIxDBLEEvT4TrX6qvr+dQjJaytSlCzrYvD2uDkv/+fnDLMxK23vZ1EaysdK1ZQPn9+xtsNpwkumQw62UsCIyVQ7v4owdV2uPsu4JxhRyAiIpIFdbuKqNtVBMDk5lIWHtuBmWOl4/tNPkr3PkVZ0T5eeGUyf5xbQjcW9GWK0FS1oLaSypr7iXfM5N2NHewtruCy6y8edALj5Rta+P4xF9ATi/HAzUsP7rhes5Blz9UzZfJujv3nG/PS4bpuVxHHbIHqrUEfqNg/fJOOW4+OlPik6lixgq7Vq8GdjZdfwZxbbzlgX8mhKGr7bNfU3MTaHavpiTGkJrj66nqOrTyWtngbN5x5w4j1gRIRETnAaO+TtWfveLY/P5OySfuo/Oe70icf4aCOFnMmJYxnXq2huJagU/cAHbSPnzmJzj0lVM2ctH/Z1+5+CoAyrwEYNHmCYP6+nlgMtxjxngRL1+06aLvdbZPZ3TaZY/N1tVr3XsraHXMPanC6u+lY9tTQE6hlT4EHTWkejx+8r+69abcL+rj5sPphXXnjBgDqL4q2XX+UQImIjKD+fmHL617Y1nrA40xfq3lTgmEvk69xd1sZ3W1l/V/ttf5xnh1fQuP4Mhbs6+S0RAfXPHYYa6sSfOiLuR9fadHcqRQnEvTGoKSkmEVpxp0asIlvhBRXxom1x/AEmPVSXlcx5H2VLzyNcBgoYiUllC88LaPtkn3cesj91YeZUgIlIiKjUq6Th6bKmVw5s5q4GUWHTeKN8RgnrYVzVpeMyFVmC2orufbF3/PixJlcdv21Bx3joE18I2DyxC7qalqZcGSCjh3jKJ/eQ3l5//3JBlM+fz7Ns8op29fDKT+8JeOarPrqeq552Fk7HT705dExeKgSKBERGVXWTahm6dSjebKmDCas58Gf38nPPzJ4n6KkZE0Ur64ccL1G76CbGG6QwHisFp6oiXP903XDPILMzd3bzNy9zWmPrb8mvudKp9JUWs35ua6V2rSMhSfsDIZRAMZPi2Ml/fcny1R3WRHdZUVpk6f102DtdJiepqN43c7gbzQkTxBhMmEREZFcS9a6/HVOOaV1t1Ja/RBFs5Zwz6q/Drptb3s78a1b6W1vDxbMOCn460dyUEpLDn1g0GPOysq2LBzJ8CWb+GKeoCScMmb5hhaunXI2N086kQ/fvJTlG1pyF0DKMArusHP3uJyOHt7U3MSN5xoPnGJc9cAVwx/1vK/uvf32sRoKJVAiUrDM7HwzW2NmL4eToheMi298gotvfCLfYRwkWetSPOEVsB7MHKyH4vJXBtwueXVXfMsWulavfj2JGkB9dT3XP30052+dSnECLAHFbpzUMjFy3GVes78DeRTHz5zE8Smd0VMlm/jetaVxf/NdulqpnAnnrfMEuBtrN03KSvKUnHKnb4LUuOZueosgkRz8dM3dwy4rl9SEJyIFycyKgB8CbwU2A0+Z2X3u/kJ+IytsyVqXxN4jYFoReA/jiku5cN7AzUapV3fhTqK1laKKwTs7H7dnAsftmcCRa9pYW5XgLV1zOW7PhCHH318yNFR9m/gWzZ1KqSeI03/H86ypWciGpmrKD+vklb2l7GkbN+xdJqfc6YkdPC1M7Y49lLjTAxS7U7tjzwHbTp7YxZRJXfuntck3JVAiUqgWAi+7+zoAM/slcCGgBCqCbF9VmNqxetKGeXRO3sqHv/yDQfu9lC88jf1tTWbEJkVLZJJjSM2bMrTkKduJU3/7XVBbyfd2PRr0gbru4znvVL563/GwD463DczMwv6SU+547OCxtvbEzuLHz/6G5l2lVE/tZv3fnfX6hpuWcVqyP1Y4rU3UJKrXffCVIlACJSKFajawKeXxZuANqSuY2dXA1QBz5swZVmHJRONQkcvjeeekTuAV6CmBXbXUZtBpuHz+fMbNm0eitRWKi0m0tpLo6ICensgjXo8m6UbtPu61DRzbupJZr50PeRraYKheH47g4LG2jhw3jdgDk6lN9NJbNJ4jz5v2+oZhf6xY6rQ2ea6FUh8oESlUlmbZAT9R3X2Juze4e0NVVdWwCivt7GVSSxelnb3D2k+UsjpWrMh5WbmU7BSe6XEUVVQQmzSJ+MaNxLdsIb5hA/EtW9h4+RWD7mPelHmvX703ggY6xtR+XcljSLdsLKmvrmfxw8Wc80zRQaOJ12xcTXGilxhBE17NxpQkPeyPlUgARSM/rU06SqBEpFBtBlJ7/R4ObM1FQR0rVlC9tYPJr3VTvaWDyh2dkb/4kl+08ebmAZOK1LLG4hdsUm97+5AShURr6+t9oULJEa9HQpSkb7BkKN2o3emWjYQXvJYXPDsNtXN2xjjr+aKDmmWDZtjgV0ystPTAQTZrFtL8/Ez2bKwc8pWAh03s4sjD24I+VFmgBEpECtVTwNFmdoSZlQIfBO7LRUEdy57CPKjyMqCiNR4pKUj9oh2sViW1rJH8gs2m3vZ24lu2DClRiE2aFPSFSmEDjHjdNzHN5Oq9geKOkvQNlgzt79eVcgzJJAMcKy7KeCTvsaB8/nzGV5VSOrn4oDnyIBhZvnVL5dCa7jYt4w0n7uLo2tagD1UWkiglUCJSkNy9B/g08ACwCrjL3Z/PRVmpX3oQPbk54AqzUH/bly88DQ9/xQ+UOAxXR3cvHd3Zb45MJouJ1tenc4lyHEUVFYybN4+S2bMpqa2lZPbstF/GqWWlJqaZDIHQ295OYt8+utavPyBJSq39yuT9TZcgHfB82K8r9RjKp8WZ8+adVJ3Uxpyzd1I+LZ7R6zJUO2NlbPdSdsbKclpOUtG4GCWTitO+X3u7e9jb3TO0Ha9/nJj16UM1TOpELiIFy93/APwh2/v9xrTgC/iu8HHyS691/Xha15eTSFikecAOuMIs1F9SMdSpMoYq20lU32QxNmkSNTf+ONJxFFVUHDCEQX/bpktMcSe+ZUu/Hc87Vqyga9UqAHp37GDj4o8y57af7o+VrVvBPaOkL7Xj+6zvfidteclj2f/c+scpr+pmQnU3WFFOO1OvuP/PnLT5BYoTPfTEill5+PE5KecAAwx8SrdD3Id2UUDdmbgbJBwrzU4fKtVAiYjkWvilN2vhHmrevJNYbU+/tSLppNZEDFarAsFUGa2V43KaPJV19TKlNc7k9jhTWuNZ62uVWiuDGSWzZ+fsOA4oK0WitXXAJtJUyZqm5NVyJXPmDPr+pCqqqKBk1qzMjzFlcMtcd6be8uhfKU70UIRTlOhhe/GErIx8vrV8AY9Of2ekfXWsWIHvTuDtCdZ/9PLo/281C3nliWo2/HUaHad9LytJpxIoEZFcS15B5FA2pYfdk8siJwXJL9qS6upoX7g50LFiBXN2dFG1J86MljhVe6L16RpIarI4bt68jAbDzEZZJbW1B4wdNVATaSorKaHosMmvNwVu3Ehs0qTcvT81C9n+/Ex2D6MzdaZmn30GPbFiejB6YsU8Mnv+sKePSU7Vc9/shkj7eunBP+MeJC29Xd289OCfI5XbsWIFXZuL6NhSysYvficr/6tKoEREcq1mIU+tqeHlLdN4ak0Ne/aOz3dEw5LaUR2y32H9pe7NrC3LbCTx4TogMZ09e8A+SRAmXccdB8XFFFVVMee2n9K7e89Bo6DnUveE02jtPSvn4yDNf/tZrDz8eB6YezpfetM1rJp6xLCnj1m6bhfxouLIU9E8O+1I4mEy11tUzLPTjoxUbvJ/M5v/q+oDJSIyAvbsHc+eveOZPGEfR8zcNeTpKLZ1rWDKpI68TmeR7KieHDXLIVKfrtEq2QF9oD5JyfVi48czrq7u9XWGMQr6aDYt0UlrySRenFoHzv5JjYdq0dypxIKuSJH2dfy5Z/DVv13JCa+9wuqZx/C1c8+IVG7wP2u4O1ZcnJX/VSVQIiIjZPKEfZx27KYhT0dROu41Tqsb+vbZUj5/PhurxlHelaA3BkUJOO3H2euwvn5qL2urEnjX3mHNSzcUB3XazkBqZ/DYpEkjUnM2kububWbejIm0dvbwgw/OH9b0MQtqK4e0rwW1lbx953O8OHEmX/viByPHsGpKHV9/4zUc/9r6IAGbUseCoRxACiVQIiIjZMqkjmFNR1E2fifPji9h+fgyGrri1OdxOovOcUV0jiva/zhbyVPqZLOP+Etc//TRWZtnL5f6Xvl3qJlYVsLEspKszL031H31nVg5iqXrdvH8tCNYWXUURRY8Hu6xKIESEcmyts4a2vYdwfINLQecpF9rLcdnGomYEyuOfgXV78pm8IOZ7cQNShy+mpjMhWnWy8X4TCMldbLZnoSzsrKN8/MdlIwKw5mwedHcqRQneumJBXPwDacZMkmdyEVEsmj5hhbWbLuCzS3nHnSV0Z694/d3Jh9K89sfJ1XQZUbCjC6LcV/L0Dvzri/9LutLvzvk7XMlOdmsJaA4ASe93JzvkOQQsKC2ks+9eD/v2rKc269clJWaNNVAiYhk0dJ1u0h4EVC0/yqjBbWV+/v1HLmjlLpt4zlmCE1vZ+wq4ZnDi4Fe8GLOm/vGtOutn1DDyxPrmNOnBiybxvk+ALosu1cU1lfXc83DCdZOh7dsL+G4nenmfE6v9mf/CcCGSy/LakyFLFnrc+c1p+c5kuEbThNgOkqgRESyKLjKqDe8yihoKkjt11OcgGseKx3Svi/Y1curf5vBszPLuOR9/8DFJx/cBLh8Qwv/ccxieqyIR25emrVf2/1JJlLZdP7OUtiZ9d3KKHEoJGOQQROemZWZ2TIze8bMnjezb4TLp5jZQ2b2Unibu0+oiMgYsaC2kmNn3sLhlQ/vT14O6NcTg/vr5gx5MMIztu7iE8u3pE2eIKgB67Ei3IoGHGenNr6W2vjaIcWQasM0+NOJRlNz07D3lVQ6sZNJs1uYML2VSbNbsjLxq0i2ZVID1QW8xd3bzawE+IuZ3Q+8F3jE3W8ws+uA64Av5jBWEZExYWLZJiaWbdpf85Ps19MDJChmZewcPhyxdui50qk0lVZTXzqVE7v77/u0aO5Uir2XXrLXWTadv9XM4Kkji3h11nYS5vz3A1fyk7fdTH11/fB2vGkZ00/YFgzVkJTHIRtkZCWbYceCQRMod3cgOTV1SfjnwIXA2eHy24BHUQIlInKQ+up6rnmslPvr5rAydg69nUeAJTK+lHr5hhaunXI2PbEYpZ7ge7se7ffS/gW1lXxz078wbloRR7/vq8zLQfPdnc8+zj1ntYD1AMEYkl293dy7+vHhJ1DrH8dinjouZeQhH8bSl/Bg0h3LoXR8Y1lGV+GZWZGZNQHNwEPu/iQw3d23AYS31f1se7WZNZpZ444dO7IUtojI6NVhzs6ixAHNWks/dDKxE+YS65hNzHsjjcK8dN0uTi5eyyeK/4sTi9bSVJr2dBvYtIx3zXuOd0xtYt4DH8lJ89eD6/4G1rt/Hl53wIvp6Thi+DtPmSx3/zy/OZ40V2QoMupE7u69QL2ZHQbcbWYnZlqAuy8BlgA0NDT4IKuLiIxpTc1NbCrpxYGrHriCm952y/5amcmTX+OkBX9h9+4qvnP+NRk3351TsZ4rSr9JCT3EKWbDFb/of+X1j1PsThEMabDOTJw39408ufyXOL3gMeK7FxDrOI33vCXa9BpphZPllk3aR+WpUyDRAxfdfMg236k2aeyKdBWeu+82s0eB84HtZjbT3beZ2UyC2ikRkYLWuOZuANyMeG83jWvuPqBZa/Lk15g8+bVIV8bN63wGt24MKLJe5nU+A5ybfuW6M+mJFUGil6Ic1dxcfPKZbP3RODYfnqBi+2G0vFbG4usvztrVfjN/uyq4c+sFwW0Ok6e+Ccxwh0BQQlQ4Bk2gzKwKiIfJ03iCT+23gPuAxcAN4e29uQxURGQsaOjspMSdHqDEnYbOTgD27JnC7t1VjJv0El66habmpsz7C9WdiRWPh95ubLCkqGYh3znuTcxr3cn7zvt+bpKPTcv47LTVWLfjk2Fz10pqY4uBLJd1+e+zu78sKps3T8lSgcukBmomcJuZFRH0mbrL3X9nZk8Ad5nZx4CNwPtzGKeIyJhQP+8iftR4KyvKxrEw7tSfexHLN7TwdNNZxMo2U166AuuOc9WDV3HTeTdllkTVLOT/HLsw46Ro7cSprJ04lfcNsF5/081kJOzoHbNgaIY5MzoOmSvlMkmKCilxGk3HOtxYhjMVTDqZXIX3LHDQLJHuvgs4J6vRiIiMdTULuWfiFE7o6qT+krugZiFL//QyeIzi8lf2X7kWT8Rp3N6YcS1UJklRppLTzSS8KPJwCsD+jt4JC66WG8qVcoea0ZRoSHrZfo80F56ISJa9VFrGPRMP259MLJo7lVgsEVyl5sHv1pJYCQ3TG/IS39J1u6BsC6VT/0xv8Sv9DrbZr5qFfG36DB6umEjcoBd0pZwUHE3lIiKSYwtqK6k/5c9BH6iy+XjpFm4484ZIYya1d7fTFm+L1neqH9OmbqNizs249WJexLSpJwFHRdrHS6VlvFRaxmPlFZzQ1cmHPnBXwdY+SWFSDZSIyAiYPPk1amvXMKMywcwJMyMlQU3NTaxpWcOW9i1c9eBVw542pW3Pw5j1gEHMemjb8/CQ99W3tk2kUCiBEhEZ5Rq3N+IEw+gl+04NR/JKwSL3A64UFJHMKYESkYJjZt8xs9Vm9qyZ3R0OEjxqNUxvwAiG5c5G36n6eRfxo1d38MmWPdy0Yw/18y7KRpgiBUUJlIgUooeAE939ZOBF4Et5jmdA9dX1HFt5LLMrZmc09EF7dzvb9m7rv6kvvFKwIlFE/SW/VfObyBAogRKRguPuD7p7T/hwKXB4PuPJREVpRUZ9pzLtL6W+SyLDo6vwRKTQXQHcme4JM7sauBpgzpw5IxnTQW49/9aM1kvXX2q4V+2ls6HkSABq42uzvm+RsUAJlIgckszsYWBGmqe+4u73hut8BegBbk+3j2xOhp5MgC7/4+XD2c2gkv2lHB+RsaYmzDlonGWRgqAESkQOSe7ez2y7ATNbDLwTOMfdh5UcjSbJ/lJt8bbIY01Fke1pMUTGGiVQIlJwzOx84IvA37l7R77jybaK0goqSityljyJiDqRi0hh+ndgIvCQmTWZ2Y/zHZCIjC2qgRKRguPu0eYtieh/76zI5e5HhUw7tYscqlQDJSIiIhKRaqBERApQchgCERkaJVAiIgWorvsf8x2CyJimJjwRERGRiJRAiYiIiESkBEpEREQkIiVQIiIiIhEpgRIRERGJSAmUiIiISERKoEREREQiGjSBMrMaM/uTma0ys+fN7HPh8ilm9pCZvRTeVuY+XBEREZH8y6QGqgf4vLsfBywCPmVmxwPXAY+4+9HAI+FjERERkUPeoAmUu29z96fD+23AKmA2cCFwW7jabcB7chSjiIiIyKgSqQ+UmdUB84Engenuvg2CJAuo7mebq82s0cwad+zYMcxwRURERPIv4wTKzCqA3wDXuntrptu5+xJ3b3D3hqqqqqHEKCIiIjKqZDSZsJmVECRPt7v7b8PF281sprtvM7OZQHOughQRkey685rT8x2CyJiWyVV4BvwEWOXu30t56j5gcXh/MXBv9sMTERERGX0yqYE6A7gUWGlmTeGyLwM3AHeZ2ceAjcD7cxKhiIiIyCgzaALl7n8BrJ+nz8luOCIih7Zbz7813yGISBZoJHIRERGRiJRAiYiIiESkBEpEREQkIiVQIiIiIhEpgRKRgmVm/2hmbmbT8h2LiIwtSqBEpCCZWQ3wVoJhWEREIlECJSKF6v8BXwA834GIyNijBEpECo6ZvRvY4u7PDLKeJkMXkbQymgtPRGSsMbOHgRlpnvoKwWwK5w22D3dfAiwBaGhoUE2ViOynBEpEDknufm665WZ2EnAE8Eww1SeHA0+b2UJ3f3UEQxSRMUwJlIgUFHdfCVQnH5vZeqDB3XfmLSgRGXPUB0pEREQkItVAiUhBc/e6fMcgImOPaqBEREREIlICJSIiIhKREigRERGRiJRAiYiIiESkBEpEREQkIiVQIiIiIhEpgRIRERGJSAmUiIiISERKoEREREQiUgIlIiIiEtGgCZSZ3WJmzWb2XMqyKWb2kJm9FN5W5jZMERERkdEjkxqonwLn91l2HfCIux8NPBI+FhERESkIgyZQ7v5n4LU+iy8Ebgvv3wa8J7thiYiIiIxeQ+0DNd3dtwGEt9X9rWhmV5tZo5k17tixY4jFiYiIiIweOe9E7u5L3L3B3RuqqqpyXZyIiIhIzg01gdpuZjMBwtvm7IUkIiIiMroNNYG6D1gc3l8M3JudcERERERGv0yGMfgF8ARwrJltNrOPATcAbzWzl4C3ho9FRERECkLxYCu4+yX9PHVOlmMRERERGRM0ErmIiIhIRIPWQImISDQnzJyc7xBEJMdUAyUiIiISkRIoERERkYiUQIlIQTKzz5jZGjN73sy+ne94RGRsUR8oESk4ZvZmgjk9T3b3LjPrdzoqEZF0VAMlIoXoE8AN7t4F4O6aTUFEIlECJSKF6BjgTDN70sweM7PT0q2kydBFpD9qwhORQ5KZPQzMSPPUVwjOfZXAIuA04C4zm+vunrqiuy8BlgA0NDR43x2JSOFSAiUihyR3P7e/58zsE8Bvw4RpmZklgGmAqplEJCNKoESkEN0DvAV41MyOAUqBnXmNKItuPf/WfIcgcshTAiUihegW4BYzew7oBhb3bb4TERmIEigRKTju3g18JN9xiMjYpavwRERERCJSAiUiIiISkRIoERERkYiUQImIiIhEpE7kIiLZdvnv8x2BiOSYaqBEREREIlICJSIiIhKREigRERGRiJRAiYiIiESkBEpEREQkIiVQIiIiIhENK4Eys/PNbI2ZvWxm12UrKBEREZHRbMgJlJkVAT8E3g4cD1xiZsdnKzARERGR0Wo4NVALgZfdfV04s/kvgQuzE5aIiIjI6DWckchnA5tSHm8G3tB3JTO7Grg6fNhuZmuGUeZYNg3Yme8gssG+le8IxqRCfv9rcxDGiFu+fPlOM9uQ5d3m+/9C5ef/c5nvGFT+wOX3e/4aTgJlaZb5QQvclwBLhlHOIcHMGt29Id9xSH7o/R/73L0q2/vM9/+Fys//5zLfMaj8oZc/nCa8zUBNyuPDga3D2J+IiIjImDCcBOop4GgzO8LMSoEPAvdlJywRERGR0WvITXju3mNmnwYeAIqAW9z9+axFdugp+GbMAqf3X9LJ9/+Fys+/fMeg8ofI3A/qtiQiIiIiA9BI5CIiIiIRKYESERERiUgJVI5pupvCZma3mFmzmT2X71gk/8zs/Wb2vJklzKzfS6fNbL2ZrTSzJjNrzEP5OTlvmdkUM3vIzF4Kbyv7WS+rxz/Y8VjgX8PnnzWzU4dbZsTyzzazPeHxNpnZ17Jc/oDnoRE4/sHKz/Xx15jZn8xsVfj//7k060R/Ddxdfzn6I+hcvxaYC5QCzwDH5zsu/Y3o/8BZwKnAc/mORX/5/wOOA44FHgUaBlhvPTAtH+Xn8rwFfBu4Lrx/HfCtXB9/JscDvAO4n2B8w0XAk1l8zTMp/2zgdzn8vxvwPJTL48+w/Fwf/0zg1PD+RODFbPwPqAYqtzTdTYFz9z8Dr+U7Dhkd3H2Vu+dtNoYMy8/leetC4Lbw/m3Ae7K034FkcjwXAv/pgaXAYWY2cwTLz6kMzkO5PP68nwfdfZu7Px3ebwNWEcymkirya6AEKrfSTXfT900TEenLgQfNbHk4HdZIyuV5a7q7b4PgSw2o7me9bB5/JseTy2POdN+nm9kzZna/mZ2QpbIzNRq+q0bk+M2sDpgPPNnnqcivwXCmcpHBZTTdjYgcOszsYWBGmqe+4u73ZribM9x9q5lVAw+Z2erwV/xIlD+s89ZA5We6D4Zx/OlCSrOs7/Hk8lydyb6fBmrdvd3M3gHcAxydpfIzke/vqhE5fjOrAH4DXOvurX2fTrPJgK+BEqjc0nQ3IgXG3c/Nwj62hrfNZnY3QTNQRglEFsof1nlroPLNbLuZzXT3bWHzSHM/+xjy8aeRyfHk8lw96L5Tv8zd/Q9m9h9mNs3dR2qS3bx+V43E8ZtZCUHydLu7/zbNKpFfAzXh5ZamuxGRSMxsgplNTN4HzgNG8irOXJ637gMWh/cXAwfViOXg+DM5nvuAy8IrsRYBe5JNjVkwaPlmNsPMLLy/kOC7eVeWys9ELo9/ULk+/nDfPwFWufv3+lkt+muQq17v+jugZ/+LBFdhfCXf8ehvxN//XwDbgDjBL5yP5Tsm/eX1/+F/hP8HXcB24IFw+SzgD+H9uQRXaj0DPJ/N80Ym5YePc3LeAqYCjwAvhbdTRuL40x0P8HHg4+F9A34YPr+SAa6QzFH5nw6P9RlgKfDGLJd/0HlohI9/sPJzffxvImiOexZoCv/eMdzXQFO5iIiIiESkJjwRERGRiJRAiYiIiESkBEpEREQkIiVQIiIiIhEpgRIRERGJSAmUiIiISERKoEREREQi+v+YcuPabKTBTgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes  = plt.subplots(1, 2, figsize=(10,4))\n",
    "for x, y in train_batches:\n",
    "    un, co = np.unique(tf.argmax(y, 1), return_counts=True)\n",
    "    axes[0].bar(range(len(un)), co)\n",
    "    axes[0].set_xticks(range(len(un)))\n",
    "    axes[0].set_xticklabels(un)\n",
    "    axes[0].set_title('Batch labels distribution')\n",
    "    \n",
    "\n",
    "    masked_sample = tf.boolean_mask(x['x'][0], x['mask'][0])\n",
    "\n",
    "    for i, passband in enumerate(['g', 'r', 'i', 'z']):\n",
    "        axes[1].errorbar(masked_sample[:,0], \n",
    "                         masked_sample[:,i+1], \n",
    "                         yerr=masked_sample[:,i+5], \n",
    "                         label=passband, \n",
    "                         fmt='.')\n",
    "    axes[1].set_title('Input sample')\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Long Short Term Memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using ```Sequential()``` class from *Keras Library* we can stack several layers. In this case, we will use recurrent and dense layer for *the middle learning* and dense activated layer for the output. Remember that the output should have the same dimension as number of classes are considered.<br><br>A graphical representation is shown bellow \n",
    "\n",
    "<img src=\"./img/archi.png\" alt=\"Girl in a jacket\" width=\"700\" height=\"400\"> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Activation, LSTM, Dropout, Input, LayerNormalization, LSTMCell\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.losses import CategoricalCrossentropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Params \n",
    "n_neurons    = 32  # Neurons in each gate of LSTM\n",
    "dropout_prob = 0.3 # Fraction of the units to drop \n",
    "num_layers   = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_7\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_17 (InputLayer)          [(None, None, 9)]    0           []                               \n",
      "                                                                                                  \n",
      " lstm_15 (LSTM)                 (None, None, 32)     5376        ['input_17[0][0]']               \n",
      "                                                                                                  \n",
      " lstm_16 (LSTM)                 (None, 32)           8320        ['lstm_15[0][0]']                \n",
      "                                                                                                  \n",
      " dense_14 (Dense)               (None, 32)           1056        ['lstm_16[0][0]']                \n",
      "                                                                                                  \n",
      " input_18 (InputLayer)          [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " dense_15 (Dense)               (None, 2)            66          ['dense_14[0][0]']               \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 14,818\n",
      "Trainable params: 14,818\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "lc_placeholder   = Input(shape=(None, INP_DIM), dtype=tf.float32)\n",
    "mask_placeholder = Input(shape=(None,), dtype=tf.bool)\n",
    "inputs = {\n",
    "    'x': lc_placeholder, \n",
    "    'mask': mask_placeholder\n",
    "}\n",
    "\n",
    "\n",
    "x = LSTM(n_neurons, return_sequences=True, dropout=dropout_prob)(inputs['x'])\n",
    "x = LSTM(n_neurons, dropout=dropout_prob)(x)\n",
    "x = Dense(n_neurons, activation='relu')(x)\n",
    "predictions = Dense(NUM_CLS, activation='softmax')(x)\n",
    "\n",
    "model = Model(inputs=inputs, outputs=predictions)\n",
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['acc'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "    183/Unknown - 4s 15ms/step - loss: 0.6936 - acc: 0.4977"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [45]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m history \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_batches\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[43m                    \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m20\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m                    \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mEarlyStopping\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpatience\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m                    \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalid_batches\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/keras/utils/traceback_utils.py:64\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 64\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[1;32m     66\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/keras/engine/training.py:1384\u001b[0m, in \u001b[0;36mModel.fit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1377\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[1;32m   1378\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m   1379\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39mepoch,\n\u001b[1;32m   1380\u001b[0m     step_num\u001b[38;5;241m=\u001b[39mstep,\n\u001b[1;32m   1381\u001b[0m     batch_size\u001b[38;5;241m=\u001b[39mbatch_size,\n\u001b[1;32m   1382\u001b[0m     _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m):\n\u001b[1;32m   1383\u001b[0m   callbacks\u001b[38;5;241m.\u001b[39mon_train_batch_begin(step)\n\u001b[0;32m-> 1384\u001b[0m   tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1385\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[1;32m   1386\u001b[0m     context\u001b[38;5;241m.\u001b[39masync_wait()\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py:915\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    912\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    914\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[0;32m--> 915\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    917\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[1;32m    918\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py:947\u001b[0m, in \u001b[0;36mFunction._call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    944\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m    945\u001b[0m   \u001b[38;5;66;03m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[1;32m    946\u001b[0m   \u001b[38;5;66;03m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[0;32m--> 947\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_stateless_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=not-callable\u001b[39;00m\n\u001b[1;32m    948\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stateful_fn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    949\u001b[0m   \u001b[38;5;66;03m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[1;32m    950\u001b[0m   \u001b[38;5;66;03m# in parallel.\u001b[39;00m\n\u001b[1;32m    951\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/eager/function.py:2956\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2953\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[1;32m   2954\u001b[0m   (graph_function,\n\u001b[1;32m   2955\u001b[0m    filtered_flat_args) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[0;32m-> 2956\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mgraph_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2957\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiltered_flat_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgraph_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/eager/function.py:1853\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1849\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[1;32m   1850\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[1;32m   1851\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[1;32m   1852\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[0;32m-> 1853\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_call_outputs(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1854\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcancellation_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcancellation_manager\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   1855\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[1;32m   1856\u001b[0m     args,\n\u001b[1;32m   1857\u001b[0m     possible_gradient_type,\n\u001b[1;32m   1858\u001b[0m     executing_eagerly)\n\u001b[1;32m   1859\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/eager/function.py:499\u001b[0m, in \u001b[0;36m_EagerDefinedFunction.call\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    497\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _InterpolateFunctionError(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    498\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m cancellation_manager \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 499\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_num_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    503\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    504\u001b[0m \u001b[43m        \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mctx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    505\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    506\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[1;32m    507\u001b[0m         \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msignature\u001b[38;5;241m.\u001b[39mname),\n\u001b[1;32m    508\u001b[0m         num_outputs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_outputs,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    511\u001b[0m         ctx\u001b[38;5;241m=\u001b[39mctx,\n\u001b[1;32m    512\u001b[0m         cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_manager)\n",
      "File \u001b[0;32m~/miniconda3/envs/tf2/lib/python3.9/site-packages/tensorflow/python/eager/execute.py:54\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 54\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     55\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     57\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit(train_batches, \n",
    "                    epochs=20,\n",
    "                    callbacks = [EarlyStopping(patience=10)],\n",
    "                    validation_data=valid_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# summarize history for accuracy\n",
    "plt.figure(figsize=(15,5))\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# summarize history for loss\n",
    "plt.figure(figsize=(15,5))\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "y_probas = model.predict(test_batches)\n",
    "y_pred   = np.argmax(y_probas, axis=1)\n",
    "\n",
    "y_test   = np.concatenate([y for _, y in test_batches], 0)\n",
    "y_true   = np.argmax(y_test, axis=1)\n",
    "\n",
    "acc_test = accuracy_score(y_true, y_pred)\n",
    "f1_score = f1_score(y_true, y_pred, average='macro') \n",
    "print('Accuracy:{}\\nF1:{}'.format(acc_test, f1_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from visual import plot_confusion_matrix\n",
    "class_names = ['I', 'II']\n",
    "cnf_matrix = confusion_matrix(y_true, y_pred)\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure(figsize=(10,4))\n",
    "plt.subplot(121)\n",
    "plot_confusion_matrix(cnf_matrix, classes=class_names,\n",
    "                      title='Confusion matrix, without normalization')\n",
    "\n",
    "# Plot normalized confusion matrix\n",
    "plt.subplot(122)\n",
    "plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,\n",
    "                      title='Normalized confusion matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}