04_new_features_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement a convolution estimator for the MNIST problem\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import argparse\n",
    "\n",
    "#from skimage.io import imread, imsave, imshow\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(111)\n",
    "\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '0'\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_trn, y_trn), (X_tst, y_tst) = tf.keras.datasets.mnist.load_data(path='/tmp/mnist.npz')\n",
    "\n",
    "#X_trn = np.array(X_trn/255.0, dtype=np.float32)\n",
    "#y_trn = np.array(y_trn, dtype=np.int32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxIAAAJCCAYAAABUPfGIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm83dO9P/7PMWQwJYiqoSKGmIKYhxpSEtWYqSGEoi3lVxQhRZDWrFRJlJJKTZeqITGVaA0tGtdQvg1iSCsaY5BIyCRyfn/cx13W2vfsdK+Tc/Y+5+zn86/Xstb+7NU+LDvvfNZnfRoaGxsLAACAHIvVegIAAED7o5AAAACyKSQAAIBsCgkAACCbQgIAAMimkAAAALIpJAAAgGwKCQAAIJtCAgAAyLZENb+soaHBa7RroLGxsaHWc6B6rLPasM7qj7VWG9ZafbHOaqPSdeaOBAAAkE0hAQAAZFNIAAAA2RQSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQLYlaj0BgKZsscUWSfvHP/5xyEcccUTIN910UzJuxIgRIb/wwgutNDsAwB0JAAAgm0ICAADIppAAAACyNTQ2NlbvyxoaqvdlzbD44osn7W7dulX0uXjv9lJLLRXyeuutl4z7//6//y/kyy67LORBgwYl4+bMmRPyxRdfnPT97Gc/q2hOscbGxobsD9FutfV1tjB9+/YN+dFHH036lltuuYqu8emnn4a84oortszEKmCd1Z/2vNZa2q677hryrbfemvTtvPPOIb/22muL/F3WWn2pt3U2bNiwpB3/uW+xxb76+/9+/fol45544okWnUel68wdCQAAIJtCAgAAyNZhj39dY401Qu7UqVPSt/3224e8ww47hNy9e/dk3AEHHLBIc5gyZUrSvuqqq0Leb7/9Qp45c2Yy7qWXXgq5pW9VQVuz9dZbh3zXXXeFXLq1MN6GGa+ZefPmJePi7UzbbrttyKVHwZZ+Dqphp512Strxv6/33HNPtafTYrbaaquQn3322RrOBNqfI488MuShQ4cmfQsWLGjyM9V8NGFh3JEAAACyKSQAAIBsHWZrU3zaS1GkJ75UevpSS4hvQZU+ef/ZZ5+FHJ9q8d577yXjpk2bFnJLnHABtRafZrb55psnfbfcckvIq6yySkXXe+ONN0K+9NJLk77bb7895Keeeirk0vV40UUXVfRd0JJKT1pZd911Q25vW5viE2R69eoVcs+ePZNxDQ0OWYKFiddMly5dajiTfO5IAAAA2RQSAABANoUEAACQrcM8I/H2228n7Y8//jjklnhG4plnngl5+vTpSd+3vvWtkOMjJW+++eZF/l7oCH7zm9+EXPom9+aIn7NYZpllkr74yOR4P/omm2yyyN8Li+qII45I2n/7299qNJNFFz/T9MMf/jDk+LmnoiiKiRMnVm1O0F70798/5BNOOKHsuHj97LnnniF/8MEHrTOxTO5IAAAA2RQSAABAtg6ztemTTz5J2qeddlrI8a2goiiKv//97yHHb5su9eKLL4Y8YMCAkD///PNk3EYbbRTySSedVOGMoWPbYostQt5jjz1CXthRkPG2pPvuuy/pu+yyy0J+9913Q47Xc1GkxyfvsssuFX0vVEt8ZGp7N2rUqCb/eXw8M/A/dthhh6Q9evTokBe2Bf8Xv/hFyJMnT275iS2ijvNfNAAAoGoUEgAAQDaFBAAAkK3DPCNRasyYMSE/+uijSd/MmTND3nTTTUP+/ve/n4yL92SXPhcRe/nll0M+5phj8icLHUDfvn2T9iOPPBLycsstF3JjY2My7o9//GPI8dGwO++8czJu2LBhIcd7s6dOnZqMe+mll0JesGBByPFzGkWRHiH7wgsvFNBa4qOHV1555RrOpGWV29cdr33gf3zve99L2quuumqT4x5//PGkfdNNN7XWlFqEOxIAAEA2hQQAAJCtw25tis2YMaNs36efflq2L35T5+9///uQ4+0SUM969+4dcnzkclGk2x4++uijkN97771k3I033hjyZ599FvIDDzyQjCtt5+ratWvSPvXUU0M+7LDDFunasDADBw4MufTfw/akdFtWr169mhz3zjvvVGM60Ob16NEj5KOPPjrpi/8sOX369JDPP//81p9YC3JHAgAAyKaQAAAAstXF1qaFGT58eMjxm3iLIj01pn///iGPGzeu1ecFbVHnzp2TdnyyWbx9oyjS09GOOOKIkJ977rlkXK22eqyxxho1+V7qz3rrrVe2Lz71r62L13tRpFudXn/99ZDjtQ/1Zs011wz5rrvuqugzI0aMCPmxxx5r6Sm1KnckAACAbAoJAAAgm0ICAADIVvfPSMRvrI6Pey2K9G23119/fcil+9fiPd9XX311yKVv8IX2brPNNkvapc9FxPbZZ5+Qn3jiiVabE7Rnzz77bK2nkLx5viiKYvfddw958ODBIe+2225lr3HeeeeFHB9lCfUmXj/xW+1L/fnPfw75yiuvbNU5tSZ3JAAAgGwKCQAAIFvdb22KTZo0KWkfeeSRIY8ePTrkww8/PBkXt5deeumQb7rppmRc6Rt9ob355S9/mbQbGhpCLt2+1Ba2My222Fd/V+KN9LRFK6ywQvZnNt1006Qdr8P4qPLVV189GdepU6eQ47e5x+ukKIpi9uzZIT/zzDMhz507Nxm3xBJf/RHi+eefr2ju0NHsu+++Sfviiy9uctyTTz6ZtL/3ve+F/Omnn7b8xKrEHQkAACCbQgIAAMhma9NC3HPPPSG/8cYbIZdu79h1111DvvDCC0Pu2bNnMu6CCy4I+Z133mmxeUJr2nPPPUPu27dv0hefTHbvvfdWbU6VirczlZ6i9uKLL1Z7OtSpeKtQ6b+H1157bchnnnlmRdcrPQkm3to0f/78kGfNmpWMe+WVV0K+4YYbQi5923y8LfGDDz4IecqUKcm4+K30EydOrGju0BE05+3V//znP5N2vLbaM3ckAACAbAoJAAAgm0ICAADI5hmJCk2YMCHkgw46KOnba6+9Qo6PiT322GOTceuuu27IAwYMaOkpQquI90HHx0cWRVF8+OGHIf/+97+v2pxinTt3TtrDhw9vctyjjz6atM8444zWmhIkjj/++JAnT56c9G2//fbZ13v77beT9pgxY0J+9dVXQx4/fnz2tUsdc8wxIa+00kpJX+meb6gXQ4cODbnSo8XLHQvb3rkjAQAAZFNIAAAA2Wxtaobp06cn7ZtvvjnkUaNGhRy/9bMoimKnnXYKuV+/fiE//vjjLTtBqJL4TbfVfHN7vJ1p2LBhSd9pp50Wcnxc5eWXX56M++yzz1ppdlDeJZdcUuspZImPNy9V6bGX0BHEx5/vtttuFX1m7NixIb/22mstPqe2wB0JAAAgm0ICAADIZmtTheI3iX73u99N+rbaaquQS7czxeK3iv7lL39pwdlBbVTzbdbxbeV4+9LBBx+cjItvJR9wwAGtPzGoU/fcc0+tpwBVM27cuJCXX375suPi09KOPPLI1pxSm+COBAAAkE0hAQAAZFNIAAAA2TwjEVlvvfWS9o9//OOQ999//5C//vWvV3S9L7/8MmnHx2NW+iZEqLWGhoYmc1EUxb777hvySSed1KLfe/LJJyfts88+O+Ru3bqFfOuttybjjjjiiBadBwCsuOKKIS/sz3C//vWvQ66HY8bdkQAAALIpJAAAgGx1ubUp3po0aNCgkOOtTEVRFGuuuWb2tZ977rmQL7jggqSvmkdlQktpbGxsMhdFupauuuqqpO+GG24I+eOPPw552223TcYdfvjhIW+66aYhr7766sm4t99+O+SHH3445Pg2MtB6Src29u7dO+T4yEvoCEaPHp20F1ussr97f/rpp1tjOm2WOxIAAEA2hQQAAJCtw25tWnnllUPecMMNk76RI0eGvP7662df+5lnnknav/jFL0KO36rrZCY6usUXXzzk448/PumL3yo9Y8aMkNddd92Krl16e/ixxx4L+ZxzzsmaJ7DoSrc2VrrVA9qLvn37hty/f/+kL/4z3bx580K++uqrk3EffPBBK82ubfJfAQAAIJtCAgAAyKaQAAAAsrXrZyRWWGGFkH/zm98kffE+t7XWWqtZ14/3aF9++eUhx0dPFkVRzJ49u1nXh/bgb3/7W8jPPvts0rfVVluV/Vx8NGz8zFKp+GjY22+/PeSWflM20LK22267kH/3u9/VbiLQQrp37x5y/BtW6p133gl5yJAhrTqnts4dCQAAIJtCAgAAyNbmtzZts802Sfu0004Leeuttw55tdVWa9b1Z82aFXLpm3kvvPDCkD///PNmXR/auylTpoS8//77J33HHntsyMOGDavoeldeeWXSvuaaa0J+8803mzNFoApK32wN4I4EAACQTSEBAABkU0gAAADZ2vwzEvvtt99C2+W88sorId9///1J3/z580OOj3WdPn16c6YIdeO9995L2sOHD28yAx3DH//4x5APPPDAGs4EWt/EiRNDjl8BUBRFscMOO1R7Ou2COxIAAEA2hQQAAJCtobGxsXpf1tBQvS8jaGxsdGZfHbHOasM6qz/WWm1Ya/XFOquNSteZOxIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkE0hAQAAZFNIAAAA2RQSAABAtqq+2RoAAOgY3JEAAACyKSQAAIBsCgkAACCbQgIAAMimkAAAALIpJAAAgGwKCQAAIJtCAgAAyKaQAAAAsikkAACAbAoJAAAgm0ICAADIppAAAACyKSQAAIBsCgkAACCbQgIAAMimkAAAALIpJAAAgGxLVPPLGhoaGqv5ffyPxsbGhlrPgeqxzmrDOqs/1lptWGv1xTqrjUrXmTsSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkE0hAQAAZFui1hMAAFrelVdembRPPPHEkCdMmBDynnvumYybPHly604M6DDckQAAALIpJAAAgGwKCQAAIJtnJIA2adlll03ayyyzTMh77LFHyCuttFIy7pe//GXIc+fObaXZQdu05pprhjx48OCkb8GCBSFvsMEGIa+//vrJOM9IwML17t075CWXXDLp22mnnUL+9a9/HXK8/ppr7NixIR9yyCFJ37x58xb5+s3hjgQAAJBNIQEAAGSztQmoqXgrxtChQ0PebrvtknF9+vSp6HqrrLJKyPFxl1APpk6dGvJf/vKXpG/vvfeu9nSg3dpoo42S9pFHHhnygQceGPJii6V/J7/qqquGHG9namxsXOQ5xWv42muvTfp+8pOfhDxjxoxF/q5KuSMBAABkU0gAAADZbG2KbLPNNkk7PvFi5513Drn0dldsyJAhIb/77rtJ3w477BDyLbfcEvIzzzyTP1loR+JTYeLbr0VRFIcddljIXbt2DbmhoSEZ9+9//zvkmTNnhhyfPlMURXHQQQeFHJ+YMXHixNxpQ7vz+eefh+z0JWi+iy66KGkPHDiwRjNp2hFHHJG0f/vb34b81FNPVW0e7kgAAADZFBIAAEA2hQQAAJCt7p+ROPjgg0O+8sork74ePXqEHO/Xfvzxx5Nx8Zt1f/GLX5T9rvga8WdK304I7VG3bt2S9iWXXBJyvM5K31hdzhtvvJG0v/3tb4ccv0m09NmHeN3GGepB9+7dQ950001rOBNo3x555JGkXe4ZiQ8//DBpx88qxEfDLuzN1ttvv33I8TO57YE7EgAAQDaFBAAAkK0utjYtsUT6P3PLLbcM+frrrw95qaWWSsbFbwU977zzQn7yySeTcZ07dw75jjvuCHm33XYrO6fnnnvuP00b2pX99tsvaf/gBz/IvsakSZNCHjBgQNIXH/+6zjrrZF8b6kH8O7bGGmtU9JmtttoqacfbBR0hS7265pprkvaYMWOaHPfFF18k7ffffz/7u5ZbbrmQJ0yYkPTFb8pe2Hxq9edKdyQAAIBsCgkAACCbQgIAAMhWF89IDB48OGmPGjWqyXGlR33FR1bOmDGj7PXjcQt7LmLKlCkh33jjjWXHQXt04IEHVjTurbfeStrPPvtsyEOHDg05fiai1AYbbJA3OagT7777bsi/+93vkr7hw4c3+ZnSfz59+vSQR44c2VJTg3Zl/vz5SXthv0mLKj7efPnll6/oM/GfKYuiKObOnduic6qUOxIAAEA2hQQAAJCtw25tio9rPfPMM5O+xsbGkH/961+HPGzYsGTcwrYzxc4666yKxp144okhT506taLPQHvxwx/+MGkfc8wxIY8bNy7kN998MxlX+lbQSqy88srZn4F6E/8OFkX5rU1A9R1yyCEhx7+fXbt2rejz55xzTovPqTnckQAAALIpJAAAgGwdZmtT6S2eeDvTvHnzkr6HH3445PiUmNmzZ5e9fpcuXUIuPZkpfntoQ0NDyOeff34ybuzYsWWvD+1dfFpMUbTuNortttuu1a4NHdVii331d4cLFiyo4UygPhx22GEh//SnP0361llnnZCXXHLJiq734osvhlz6Ru1acUcCAADIppAAAACyKSQAAIBs7foZie7du4d8/PHHJ33xEa/xMxFFURT77rtvRdeP96/deuutIW+xxRZlP3PnnXeGfOmll1b0PVDP4mORl1566Yo+s/HGG5fte/rpp0P+29/+1vyJQQcTPxcR/0YC/9eaa66ZtA8//PCQ+/fvX9E1dthhh5ArXXOlrx6In6148MEHQ17Yc73V5I4EAACQTSEBAABka9dbmzp16hRyjx49yo6Lt04URVF87WtfC/moo44Kee+9907G9enTJ+Rlllkm5NLbU3H7lltuCfnzzz8vOyfo6JZaaqmQN9xww5DPPffcZNzAgQOb/Hx8VGVRlD+usvTY2XhNf/nll5VNFoC6F/+5795770364qP+W9Nf//rXpH3ddddV5Xubyx0JAAAgm0ICAADI1q63NsVvrJ46dWrSt9JKK4X8r3/9K+mr9Mn5eMtE/BT9Kquskoz76KOPQr7vvvsqujZ0BPHbODfbbLOk76677go5XjOlJ03E6yw+ZWn33XdPxsVbpWJLLJH+Z2z//fcP+corrwy59A33AFBOQ0PDQtuVaM7b5Pfcc8+k/Z3vfCfkP/7xj9lzaG3uSAAAANkUEgAAQDaFBAAAkK1dPyMxffr0kEvfVn3//feHvMIKKyR9kyZNCnns2LEh/+53v0vGffLJJyHffvvtIZc+IxH3QUcWH7lcFOlzDHfffXfZz/3sZz8L+dFHH036nnrqqZDjtVo6Lj6WLxY/D1UURXHRRReF/Pbbb4c8ZsyYZNzcuXPLzhc6okr3a++0004hjxw5slXnBG3JhAkTQu7Xr1/SN3jw4JAffvjhkOfMmdOs7/r+978f8gknnNCsa7QF7kgAAADZFBIAAEC2hkqPQm2RL2toqN6XtYD49u4TTzwRcukt4Z/85CchjxgxovUnlqmxsTH/zDLarZZeZ/ERrz//+c+TvtNOO63s5+Jj6g4//PCQ4y2JRZFuTXrwwQdD3nzzzZNx8fGtl156acilW5722WefJufzpz/9KWlfcsklIU+bNq3JzxRFUbz44otl+2LWWf1pb79p8ZveK/3t32STTUJ+5ZVXWnxOzWGt1Zf2ts4q1a1bt5A//vjjsuP22muvkKt5/Gul68wdCQAAIJtCAgAAyNauT21qbV27dg053s5UekvYqU10NIsvvnjI5513XshDhgxJxn3++ech//SnP0364nURb2facsstk3HxqTDx27HfeOONZNxxxx0X8mOPPRbycsstl4zbfvvtQz7ssMNC3nvvvZNxjzzySFHOv//975B79epVdhy0J9dee23Ixx57bEWfOeaYY0KOt/ECi+bb3/52rafQItyRAAAAsikkAACAbAoJAAAgm2ckFiJ+cyHUk3hfdPxcxKxZs5Jx8T7rcePGJX3bbrttyEcddVTI3/nOd5Jx8bNI8fGyo0ePTsbFzy3EZsyYkbQfeuihJvOgQYOScYceemiT1yuKojj55JPL9kF7NXHixFpPAWouPtJ8t912S/oeffTRkGfPnt2i3xv/DhZFUVx55ZUtev1acUcCAADIppAAAACyebP1QsRHc8Vv3C39/2yVVVYJeerUqa0/sUzeAlpfWmKdvffeeyHHb56eO3duMi7eKrH00ksnfeuss05F3zV8+PCQL7roopDjt/C2B9ZZ/Wlvv2mx119/PeS111677LjFFvvq7xtL1/SkSZNafmIVsNbqS0ussx122CHks846K+QBAwYk4+Ljvsttp/1PVlhhhZAHDhwY8ogRI5Jxyy67bJOfL91SFR9dHh993tq82RoAAGg1CgkAACCbU5sWYq211qr1FKAm3n///ZDjrU2dO3dOxm266aZlrxFvB/zLX/4S8pgxY5Jxb731VsjtbTsTtFcvv/xyyAv7rVuwYEE1pgOtauTIkSH36dOn7LjTTz895JkzZzbru+LtUptvvnnIC3uU4PHHHw/5mmuuSfqquZ2pOdyRAAAAsikkAACAbAoJAAAgm2ckFuKvf/1ryPERePaM0tHttNNOIe+7774hx/s9i6IoPvzww5BvuOGGpG/atGkhz5s3r6WnCCyC6667LuS99tqrhjOBtuO4445rtWvHv5dFURT33XdfyCeddFLIc+bMabU5tAZ3JAAAgGwKCQAAIJs3W1cofgto6VF58RsTx48fX7U5VcpbQOtLe15n7Zl1Vn/a81rr2bNnyPfff3/St8EGG4Tc0PDVv9a9e/dOxnmzNdXQEuusb9++IZ9wwgkhf+9731vUS/+fdTBr1qyQ4y3y8XbCoiiKCRMmLPJ3tyZvtgYAAFqNQgIAAMimkAAAALJ5RqJCRx55ZMijRo1K+p544omQ4713r7zySqvPqxL2k9aX9rzO2jPrrP5Ya7VhrdWXll5nnTt3Djn+s11RFMX5558f8vLLL5/0jRkzJuRHHnkk5LFjxybj3n///ZaYZs15RgIAAGg1CgkAACCbrU0VWm655UK+4447kr7+/fuHfPfdd4d81FFHJeM+//zzVprdwrkNXF/a8zprz6yz+mOt1Ya1Vl+ss9qwtQkAAGg1CgkAACCbrU3NEG9zKoqiuOCCC0I+7rjjQt5kk02ScbU6xclt4PrSUdZZe2Od1R9rrTastfpindWGrU0AAECrUUgAAADZFBIAAEA2z0jUAftJ64t1VhvWWf2x1mrDWqsv1llteEYCAABoNQoJAAAgW1W3NgEAAB2DOxIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkE0hAQAAZFNIAAAA2RQSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkG2Jan5ZQ0NDYzW/j//R2NjYUOs5UD3WWW1YZ/XHWqsNa62+WGe1Uek6c0cCAADIppAAAACyKSQAAIBsCgkAACCbQgIAAMimkAAAALIpJAAAgGwKCQAAIJtCAgAAyKaQAAAAsikkAACAbAoJAAAg2xK1ngBALfz5z38OuaGhIenbZZddqj0daBEbbrhhyHvuuWfSd8wxx4T87LPPhvz3v/+97PV+9atfhTxv3ryWmCLQgbgjAQAAZFNIAAAA2RQSAABANs9IRJZccsmkvf3224d84YUXhvzNb36zanMCWsYVV1yRtOP1fdNNN1V7OtBijj322JAvu+yykJdZZpmyn1l77bVDPuSQQ8qOi5+leOyxx5o7RaCDckcCAADIppAAAACyNTQ2NlbvyxoaqvdlzdCjR4+k/eGHH4b8/vvvh7z55psn4+K+tqixsbHhP4+io2jr66yaLr744pBPOumkpO+LL74I+Qc/+EHSd8cdd2R/l3VWf9rKWlthhRVCfvXVV0P+2te+tsjXnj59esgHH3xw0jdu3LhFvn5zWGv1pa2ss3pT6TpzRwIAAMimkAAAALI5talCX//615vMRdH2tzZBvdp2221DLj2V7cknnwy5OVuZoK345JNPQj733HNDvvzyy5NxSy21VMhvv/12yGussUbZa3fv3j3k3XffPemr1dYmqFc9e/YMuWvXrknfoEGDQj7uuOPKXuOBBx4I+aijjlrkObkjAQAAZFNIAAAA2RQSAABANs9IVKihwWlz0BJ22mmnpH3WWWeFHO/xjPd954iv0adPn5AnTZqUjBsyZEizrg9t2bXXXhvyj370o6Rv0003DXnGjBnZ1x45cmTzJwZUpH///kl7//33Dzn+fevWrVsyrtLXOcTPDrYEdyQAAIBsCgkAACCbrU0Vim8ZdenSpYYzgfbtuuuuS9rrrrtuyBtuuGHI8fGsOc4888yQV1xxxZB/+MMfJuNeeumlZl0f2ovzzz8/acfbCPv27Zt9vU6dOi3ynID/MWrUqJA33njjkLfaaquKPj9z5sykfeutt4b87LPPJn233XZbyHPmzMma53/ijgQAAJBNIQEAAGRTSAAAANk8I9EMW265ZdIeP358jWYC7c+sWbOS9qI+f1S617tnz54hL1iwYJGuDe3ZnXfembTj547GjRsXcrw/e2FKn7n47ne/uwizg44vfk7voosuSvqOPvrokOPjzp9//vlk3MUXXxzyhAkTQp49e3Yy7u233160yTaTOxIAAEA2hQQAAJDN1qbI/Pnzk/ann34acvwGwbXXXrtqc4KO4Lzzzgu5dBvFq6++GnKlR7IuvfTSIQ8dOjTpW2qppUKOtx2WbvOAju6www5L2vGbreO3vlequUcyQ706++yzQ/7+97+f9I0YMSLk+Gjmzz77rPUn1oLckQAAALIpJAAAgGy2NkWmT5+etP/617+GvOeee1Z7OtCufeMb3wg5fqt06RbCH//4xyFPnTq1omv/8pe/DPnAAw9M+t59992Qv/nNb1Y2WWjH1l9//ZDvueeekNdZZ51k3BJLLNpP/r333rtIn4eOIt5CW7q99vDDDw/5Jz/5SciPPfZYMu7hhx8OuaXfNl1N7kgAAADZFBIAAEA2hQQAAJDNMxJAiyg9TjLeq92jR4+Q4yPviqIonnjiiYquP2TIkJCPPPLIsuMuuOBSqhVSAAAgAElEQVSCiq4HHcUGG2wQcq9evUJe1GciSp188slJ+4QTTmjR60N7MWzYsJBLn5G44447Qo7fIN+en4NYGHckAACAbAoJAAAgm61NzbDiiivWegpQE6VbJQYPHhzyb3/726RvscW++nuKBQsWhLzddtsl484444yQ42NdV1hhhWRcfMxrQ0NDyDfddFMy7je/+U35/wHQAcXbCE8//fSQL7nkkmRcly5dFul7VllllUX6PHQU8e9WY2Nj0nfbbbeF3FG3M8XckQAAALIpJAAAgGy2NjXD3nvvXespQE0ccsghSXvUqFEhl97ejbczvfnmmyFvueWWybi4vc8++4S82mqrJePibRXxG7CPPvroiuYO9eCqq64K+Y033kj6unfv3uRnSrcsjhw5MuTllluuBWcHHcN///d/h1z6mxavn9mzZ4f8yCOPtP7EasAdCQAAIJtCAgAAyKaQAAAAsjWU7mtu1S9raKjel7WA+C2el19+ecgzZsxIxpXbd9pWNDY2NvznUXQULb3ODj744JBvueWWpG/+/PkhT58+Pek79NBDQ542bVrI8VoqiqLYeeedm/ze+IjXokifwYjz+++/n4zr169fyJMmTWry2q3BOqs/7e03rZzStTZ8+PCQzznnnJBL19Ouu+4a8uTJk1tnck2w1upLNdfZNttsE/Lf//73pG/evHkhx8eTn3jiicm4s88+O+TPPvusyWsXRVFMnDhx0SbbyipdZ+5IAAAA2RQSAABANse/LsTbb7/d5D9fcsklk3bPnj1DrubtXaiGY489NuTSNXH++eeHPHr06Iqud8IJJyTt+E3UpW+9LifeivHYY48lfdXczgQdQadOnZJ2vJ0p9sUXXyTtL7/8stXmBK0lPkr8/vvvT/rWWGONkOPt7UWRbu395JNPQo6Pey2KdGvTMsssE3K8HaojcUcCAADIppAAAACy2dq0EPGJNLHSEy46d+5cjelATYwdOzbku+++O+n797//nX29Hj16JO0+ffo0OW7QoEFJe8KECU2OmzJlSvYcgK/EWxQX5re//W3StvZoj1544YWQS9/cPnTo0JBLTyks56STTirb96c//Snkcr9h7Z07EgAAQDaFBAAAkE0hAQAAZPNm6wq98sorIa+//vpJ37XXXhvy8ccfX7U5VcpbQOtLW1xn3bp1C7l0P3a8ZuKjW3v37t36E2tB1ln9aem1tuKKK4Zcepzybbfd1mRurvgIzNI37JbuG/9fa6+9dtL+5z//ucjzaA5rrb609Do744wzQh42bFjS17Vr14qu8cYbb4S87rrrJn3xawAOOOCAkONnM9oDb7YGAABajUICAADI5vjXCo0bNy7k1VZbLek75ZRTqj0daFfi7UvHHXdc0vfhhx+GvMsuu1RtTtDWXHXVVSHvtddeSV+81e/dd99N+t55552Q33zzzZC32GKLstc4/fTTQy63lakoiuLyyy8v+73QHl100UUhl76tfbPNNgu5f//+Za+x/PLLh/zAAw8kfUOGDAk5Xo8dlTsSAABANoUEAACQzdamZig96WrevHk1mgm0XT179gz5Bz/4Qcil6+e6664L2ZtyqWcjRowIuVevXknfdtttF/Ljjz+e9L311lshxycM7rjjjsm4ZZddtsnvLV2T8SlO5557bshz5swpM3Nony677LJaT6Hdc0cCAADIppAAAACyKSQAAIBsnpFohtKj8vbZZ5+Q77nnnmpPB9qkRx55JOT4eYlbbrklGRfvwYZ6Nn78+JD/9re/JX0333xzyL/+9a+TvjXXXLPJXKlp06Yl7Q033DD7GkB9ckcCAADIppAAAACy2dpUoYMOOijkuXPnJn2vvvpqtacDbd7o0aNDPu+880IeO3ZsLaYD7cqpp56atDt37hzyMsssU/Zz8Zt5Bw0aVHbcp59+GvKAAQOaM0UAdyQAAIB8CgkAACCbQgIAAMjW0NjYWL0va2io3pe1sNtvvz3kDTbYIOnbe++9Q548eXLV5lSpxsbGhlrPgeppz+usPbPO6o+1VhvWWn2xzmqj0nXmjgQAAJBNIQEAAGSztakOuA1cX6yz2rDO6o+1VhvWWn2xzmrD1iYAAKDVKCQAAIBsCgkAACCbQgIAAMimkAAAALIpJAAAgGwKCQAAIJtCAgAAyKaQAAAAslX1zdYAAEDH4I4EAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkE0hAQAAZFNIAAAA2RQSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRboppf1tDQ0FjN7+N/NDY2NtR6DlSPdVYb1ln9sdZqw1qrL9ZZbVS6ztyRAAAAsikkAACAbAoJAAAgm0ICAADIppAAAACyVfXUJoDm6t27d8gPPfRQyIsvvngyrmfPnlWbEwDUM3ckAACAbAoJAAAgm61NQJs0YsSIpH3wwQeHvMIKK4R8//33V21OAMBX3JEAAACyKSQAAIBsDY2NjdX7soaG6n0ZQWNjY0Ot50D1tLd1tvLKK4d89913h7ztttsm4+L/Vk2YMCHkXXfdNRn38ccft/QUK2Kd1Z/2ttY6CmutvlhntVHpOnNHAgAAyKaQAAAAsikkAACAbG3y+Ndlllkm5PjIx6Ioijlz5oS8xRZbhLzssssm4w477LCQH3/88aTvnXfeyZ7T+++/H/LYsWOTvueeey77elCv4jdUF0VRXHbZZSFvs802ZT93xhlnhByvuVo9EwFtUUPDV9uab7vttqRv4MCBIW+44YYhT5kypfUnBnRI7kgAAADZFBIAAEC2Nnn866WXXhrykCFDWm0+zbVgwYKk/corr4Qc30ouva381ltvteq8ynFUXn1p60fllR7r+uSTTzY5Lt6iURRFMXjw4JBL11ZbYJ3Vn7a41pZaaqmQX3vttaRvtdVWC/mYY44JedSoUa0/sRZkrdWXtrjO6oHjXwEAgFajkAAAALK1yVOb9t9//+zPlJ7c8v/+3//LvkbpbeD11lsv5O7du4e82WabJeP69OkT8gUXXFB2DrXa2gS1Fp/U9F//9V9JX+kWpv9V+t+B0tPSgP9r1qxZIb/xxhtJX7y1aaWVVqranICiOPXUU5N2p06dQt5ggw1Cjk8dLTVx4sSQN9pooxacXfO5IwEAAGRTSAAAANkUEgAAQLY2+YzEt7/97ZBL34L7+uuvN/mZeF9oURTFe++916Jzit+c/Y9//CPpW2ONNZr8zN577520H3jggRadE7QXhx9+eMil6+XBBx8M+Uc/+lHIzXkDPfCVq6++Omn369cv5HhPNtB8O++8c9KOn5uN+/bbb79kXLnnAxf2WoZ111035PjVA0WRvq2+mtyRAAAAsikkAACAbG3yzdZt0aBBg0K+9dZby46bO3duyDvuuGPS99xzz7X8xCrgLaD1pa2ss6effjrkvn37hvzuu+8m43bfffeQ33zzzdafWCuxzupPW1lr5XzjG99I2pMnTw553rx5Iffq1SsZ19Jbg1uatVZfqrnOVllllZBvu+22pG+ttdZq8jPdunVL2ksvvXTI8fal559/Phm3+eabN3ueRfF/t//27Nlzka5XyputAQCAVqOQAAAAsikkAACAbG3y+NdaiV9XXhRFcdVVV4V8xBFHVHSN7bbbLuQXX3yxZSYG7cA+++yTtLfZZpuQ42ex/vCHPyTj5syZ07oTA4qiSPdrx793pUeV/+Y3v6nanKCW+vfvn7Svv/76kEufMWqO+EjWjz76KOnr0aNHyKuuumrIo0ePTsatvvrqTV679PjXWnFHAgAAyKaQAAAAstX91qZvfetbIcdv3y2KojjyyCOb/MwXX3yRtE888cSQJ06c2HKTgzaue/fuIZced1zOtGnTkvaUKVOyv/ekk04KeWG3n4cMGZJ9beioyh33XrqtF+rF6aefnrQr3c4UH/U/dOjQpG/8+PEhv/baa2Wv8fHHH4cc/6aV28pUFEXx1ltvhVz6Z9ZacUcCAADIppAAAACy1eXWpq233jrkcePGhbz44otX9PnS28Nvv/12yF9++eUizg7aj/jf9y222CLpW2yxr/6eYsGCBSH/5S9/qejaJ598ctm+E044IeSFvc3z1FNPDbn0dnHpW0EB6Ph22223kLfddtuKPxf/WS/eVvTUU08t8pwWtp0pNnbs2JBLT4GqFXckAACAbAoJAAAgm0ICAADIVpfPSBx00EEhV/pcRKz0qLwHHngg5Oeeey7k++67Lxl3zz33hDxhwoTs74W2Zueddw659PjX+LmIeG/pwvZ19u3bt+z1St+++78+//zzpB0fJ7veeuuFfOeddybjDjnkkJAnT55cdk4AdBzxs3NLLbVU2XFPP/100v7Zz34WcnOei1h++eWT9u677x7yTjvtVNE8HnzwwezvbW3uSAAAANkUEgAAQLa63Np09913h7zBBhuEvNVWWyXjevTokX3tLbfcsslcFEVx7rnnhvyrX/0q5EsvvTQZ9+GHH2Z/L1TDsssum7R79epVduy7774b8s033xzym2++mYzr3bt3yKeddlrI++yzTzIu3hIVH9t8+eWXJ+O6desW8qOPPtrkP4d61NDQEHK5t1xDR3fdddeFXPrnvE8//TTkQw89NOl7//33F+l7f/SjHyXt8847r8lxL7/8ctKOt+Mv6hxagzsSAABANoUEAACQrS63NsVPwO+xxx4hr7HGGsm4+JbXyiuvHPL++++fjDv66KNDjm8dl4rf9HvKKaeEXPpG4F133TXk+OQbqLUddtghaV9xxRVlx15//fUh//znPw85XktFURSXXXZZyAMHDgx55syZybg77rgj5CFDhoS87rrrJuOuvfbaJq/x5z//ORnnpCbqje1MUBR33XVXk7k17LXXXiGfc845ZcfNnz8/5Pg3rCja5nammDsSAABANoUEAACQTSEBAABka6jmnsmGhoYOuUHzsMMOC/mEE04Ieeutt27W9X7605+GXHo0bHM0NjaWf3CDDqc119nQoUOT9gUXXFB27BJLNP0IVukbQbfZZpsmx8XPChVFUTzxxBMhb7vttiE/+eSTZecQH7McP1fRGqyz+tPWf9O+8Y1vJO1yzwV961vfStrxWmuLrLX60tbX2cJ8+eWXIS/sz9vHH398yPHxtLVU6TpzRwIAAMimkAAAALLV5fGvLe3WW28N+fe//33If/rTn5JxO+20U0XXW2eddVpmYtDCunfvnrTj447Hjh1b9nN9+/YNec011yx7jVNPPTXk0u0V8Ruw/+u//qvJz5deI97aBDRt0qRJtZ4CdBgXXnhhyPGx/ws7zr+tbydcGHckAACAbAoJAAAgm61NLSx+O+Hzzz+f9FW6ten1119v0TlBa4lPoaj0BLjS27vx5zbZZJOQ33777WRcly5dQv7Xv/4V8o477piM+/TTTyuaBwAsqk6dOiXtzTbbLOT49670N/Kkk04K+Y033mil2bU+dyQAAIBsCgkAACCbQgIAAMjWYZ+RWGWVVUL+4Q9/mPRNnDgx5DvuuKNFv3fxxRcPedNNN63oM/FzFUVRFOPHj2/ROUFLKT3i9bTTTgt5n332Sfrit0/Hx78uu+yyZa9/xBFHhFx6rOtHH30U8vDhw0N+5513/sOsgYXp3LlzracA7cpSSy0V8uDBg5O+AQMGNPmZ2267LWnHrw5Y2NGwbZ07EgAAQDaFBAAAkK3DbG36+te/nrQfeuihkDfeeOOkb/nll2/R71555ZVDPuWUU0LeZZddKvr8q6++mrSffPLJlpkYtLAvvvgiac+aNSvk+FZvURTFU089FXKlR8PGZs6cmbTjbYh//OMfs68HNG3gwIFJe8SIETWaCbRd8bbc66+/PuTvfve7ZT9z8sknhzxy5Mikrz1vZ4q5IwEAAGRTSAAAANk6zNamX/3qV0m7dDtTrFevXiG/9tprIc+ePbvsZ7p27Rry6aefnvTF25kWdiJNfApNvG3jxBNPLPsZaEtK39Y+aNCgkON1UBRF0a9fv4queeONN4b8j3/8I+S///3vybgnnnii0mkCRVF88MEHSfvll18OeaONNqr2dKBdW2211UJe2HamSZMmhXzVVVe16pzaAnckAACAbAoJAAAgm0ICAADI1mGekfjzn/+ctA866KCyY1944YWQ433Yn376adnPdOvWLeTNNtusOVNMnovYb7/9Qrb3m/bqgQceaDIDtTdv3rykPWfOnCbHlb6J1/GvUBTrr79+0j711FObHPf6668n7e985zutNqe2yB0JAAAgm0ICAADI1mG2Nj3yyCNJ+/bbbw/5kEMOKfu55m5TKmf+/Pkhlx5Je9ddd4X8zDPPtOj3AsDCvPjiiyFvscUWIS+zzDK1mA60aWeffXbSPvjgg5scV7oVcPLkya02p7bIHQkAACCbQgIAAMimkAAAALJ1mGck3nrrraR91FFHhXzvvfcmfbvsskvI8bFde++9d9nrT5w4sWzfo48+2uS4eD8qANTSBRdcEHKfPn1CvuOOO2oxHWhzNtpoo5CXW265suOuu+66kOM/A9YjdyQAAIBsCgkAACBbQ2NjY/W+rKGhel9G0NjY2FDrOVA91lltWGf1x1qrDWutvlRznV1yySUhl77JOj7WdeDAgSG/9tprrT+xGqh0nbkjAQAAZFNIAAAA2WxtqgNuA9cX66w2rLP6Y63VhrVWX6q5znbdddeQH3744aTvgAMOCHns2LHVmlLN2NoEAAC0GoUEAACQTSEBAABk84xEHbCftL5YZ7VhndUfa602rLX6Yp3VhmckAACAVqOQAAAAslV1axMAANAxuCMBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkE0hAQAAZFNIAAAA2RQSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANmWqOaXNTQ0NFbz+/gfjY2NDbWeA9VjndWGdVZ/rLXasNbqi3VWG5WuM3ckAACAbAoJAAAgm0ICAADIppAAAACyKSQAAIBsCgkAACCbQgIAAMimkAAAALIpJAAAgGxVfbM1ANC2rbXWWiFfdNFFIe+3337JuE022STkiRMntv7EgDbHHQkAACCbQgIAAMhmaxMA1LHtt98+aT/00EMhT506NeSrr746GffBBx+07sSANs8dCQAAIJtCAgAAyKaQAAAAstX9MxKHH354yLvttlvS17dv35DXW2+9stcYP358yHvttVfIn376aUtMEciw9NJLh/z444+HvOqqqybjvvnNb4b81ltvtfa0oE3ZY489Qr7zzjuTvmuvvTbks846K+RZs2a1/sSAdsUdCQAAIJtCAgAAyNbQ2NhYvS9raKjel0V69OiRtEeNGhVyvBVp+vTpybinn366yev169cvacdbKeK3e2644YbZc20NjY2NDbWeA9VTq3XW0kq3Iq200kpNjps2bVrS/ta3vhXy6NGjQ37ttdeScVtvvXXIM2fObPY8/5d1Vn/a21pbZ511Qn7ppZdC/utf/5qMGzhwYMgLFixo/YllstbqS3tbZx1FpevMHQkAACCbQgIAAMhWF6c2xW/pLIqiWHPNNUO+9NJLQ/7FL36RjPvkk0+avN7666+ftP/7v/875N69e4d8zjnnJON+/vOfVzZh6GD69OkT8oknnpj09ezZs8nPxGupKIpijTXWaHLcxRdfnLTjLYUNDV/dmX3nnXeScZ06dVrIjKH969KlS9KOt/X+4x//CPmggw5KxrXF7UzQXqywwgohH3zwwSGfeeaZybjS7bv/a9iwYUn7oosuasHZtTx3JAAAgGwKCQAAIJtCAgAAyNZhj38dMGBAyKXPSNxxxx0hDxo0aJG/K372Id7bNnny5GRcr169Fvm7msNRefWlLR6VFz8XccUVV1T0mblz5ybtP/zhDyHvsssuIZfbZ1oU6TMSRxxxRNJ3yy23VDSPSlln9actrrVY6XN/P/7xj0Ned911Q54yZUrV5tQSrLX60tbX2bbbbpu049+4+Jjx5v55++abbw75qKOOatY1msPxrwAAQKtRSAAAANk67PGvSyzx1f+0N998M+m7/fbbW/S77rzzzpDjrU2lR+8tt9xyIc+YMaNF5wBtzfDhw0M+7bTTyo678cYbQ546dWrIl112WTIu7uvbt2/IDz/8cDIufpN9/Jl4nUJH1blz55AHDx6c9D3++OMht7ftTNCWxL8z119/fdK3wQYbhBz/Bo0ZMyYZN3bs2JDjrbcHHnhgMi7eOhUfWz5v3rzcabcKdyQAAIBsCgkAACCbQgIAAMjWYZ+ReOyxx0LebLPNkr5Zs2a16HeVHlP5v1ZeeeWkfeihh4Z87bXXtugcoK1ZeumlQ+7atWvIpccin3XWWSG/9957Za+3zjrrhHzmmWeGvNJKKyXjPv/885Dj5zTmzJlTwayhfTv99NNDXmaZZZK+eK0BzRc/3xA/E1EURTFu3LiQBw4cWNH13njjjZD79++f9K2++upNftdLL71U2WRbmTsSAABANoUEAACQrcNubarmNoZ//vOfIb/88sshb7TRRsm4+E2i0NHFx63uvvvuIW+44YbJuIsvvjjk448/PuRu3bol4375y1+GvMcee4T8ySefJOMuuOCCkK+55prcaUO7tttuu4X81FNPJX0vvPBCtacDHdLs2bPL9sXbnlpC/LqAjz76qEWv3RLckQAAALIpJAAAgGwddmtTNX3xxRchz58/v4YzgbbjxRdfDHn8+PEhl25t2mWXXUIeMGBAyFdccUUybo011mjye372s58l7REjRuRPFtqxHXbYIeT4Lbgbb7xxs67Xr1+/kOM388Zbd6GeNTQ0NJmLoiimTZsWcpcuXUJee+21k3FHHnlkyFtssUXI77//fjJu0KBBIb/zzjvNm3ArckcCAADIppAAAACyKSQAAIBsnpFoAZ07dw453g9XaubMmdWYDrQJ8Rvf4+PrSq266qoh33XXXSGX7jttbGwM+be//W3IY8aMWaR5Qns3ePDgkF999dWQ//Wvf5X9TLw/+/LLL0/6ll9++ZDjdTxkyJBk3NVXX509V+gI4uP949+moiiKU045JeRTTz015Pg5iFKHHHJIyPHR6e2BOxIAAEA2hQQAAJDN1qYWsOaaa4a83nrrlR330EMPVXS9Hj16hLzpppsmfdttt13If/jDH0J+7bXXKro21MLkyZMX+RoPPvhgyJdddlnI//73vxf52tCeHX300SEfeuihIcfbkoqiKDp16hTyueeeG/Kxxx6bjHv44YdDHjhwYMijR49Oxk2aNCnkSn/foCP4+OOPQ1522WWTvi233DLkeItu6RaoWbNmhfzKK6+09BSrxh0JAAAgm0ICAADIZmtTheKTmVZfffWkb/vtt6/oGtdee23Izz//fMibb755Mm6FFVYI+Rvf+EbSF5/8tM4664Qcn8ABbcHiiy8e8o477hhy6WlM5TzwwANJe6+99mqZiUE7F58YUxRFscQSX/2Uz58/v+zn4t+aeCvSwk6J+f3vfx9y/AbtoiiKM844o8nrQUcXr8H4bfJFkf4ZMV4/pe6+++6QbW0CAADqikICAADIppAAAACyddhnJLp27Rry1772taQv3ica723bZZddyl4vfmN16f7USsWf69atW9lxN9xwQ8il+8Q/+uijkN96661mzQOq4fbbbw95//33D7n0CLxyKh0H9ebrX/962b6JEyeW7Xv55ZdDHjZsWPb3XnPNNUn7H//4R/Y1oKMZP3580u7Tp09Fn7vwwgtbYzpV544EAACQTSEBAABka9dbm+LtS8OHD0/64qMi119//WZdf8aMGSHHx66WHq8XH70XGzVqVNKOj3994YUXmjUnaEtWXXXVkI866qik74ADDgg53qZU+u/+Sy+91OQ1SrckAv/ZO++8U7Yv/h1rjilTpizS56EebLzxxiEvtthXf1+/YMGCWkyn1bkjAQAAZFNIAAAA2dr11qYxY8aEPGDAgKRv7ty5IZeefPSvf/0r5LFjxzb5maJIT0WKb+mWnorRu3fvkP/5z3+GfMoppyTjPvvss//7PwLasV133TXkn//852XHxSfEjBw5Munbd999Q463NrXnN31Cayp9O3ylb4tfVDvvvHPSXtStUtARzZ49O+R4O9Pjjz+ejJs3b161ptSq3JEAAACyKSQAAIBsCgkAACBbu35GYrfddgs5fu6hKNI36b744ovNun58rOsll1wS8mqrrZaM+/DDD0M+6KCDQvZMBB1Nv379kvZVV11Vduzee+8d8p/+9KeQS9/Ke8455zT5eW9uh6aVvvW9Nd8Cv+SSS4b8ox/9KOm7+eabW+17ob0ofcXA97///ZCnTp0acumb4TvKb5w7EgAAQDaFBAAAkK1db22Kb+dOnz496ZswYUL29bp06ZK0//CHP4S8xx57hFx6TOwhhxwSsjdW05GVHrPcrVu3kJ944omk7/777w853h6x5557lr1GfIxlfEsY+Erp0cjvvfdeyIMHDw65dCtFpeL1Gl9jzTXXTMZ973vfa9b1ob2Lf7cefvjhpC/e/j506NCQ77zzztafWA24IwEAAGRTSAAAANna9dam119/PeS+ffsmfdddd13IK664YtL30ksvhRy/ifq0005Lxq233nohP/PMMyEfd9xxybjmngoF7U38ls6iSLcXlp4cE2+PiN9efeWVVybjpk2bFvKoUaNCbu62DOjo4q1MRVEUF154YciXX3552c/deuutIa+11lohb7rppsm4M888M+Q5c+aEHJ+UWBRF8dFHH1U4Y+hYLr300pBLT/K87bbbQl7Yeuwo3JEAAACyKSQAAIBsCgkAACBbu35GIn6b4HnnnZf0DRkyJOTFFkvrpd13373J6917771J+9RTTzrEx/IAAAJrSURBVA35oYceavY8oaP42te+Vrav9LjWRx55JOQdd9yx7OeOOuqokO+7775FmB3Up6uvvrrJf166P3vkyJFNjps5c2bSjt9Yf/7554c8b9685k4R2r3+/fuHHB+zPHv27GRcRz3mtRx3JAAAgGwKCQAAIFtD6ZGNrfplDQ3V+zKCxsbGhv88io6iNdfZT37yk6S9sKPt4rdUf/LJJyGXbsO4+OKLQy69RdyeWGf1x29abVhr9aVW66z0Te7PP/98yF26dAk53uZUFEVxzz33tOq8qqXSdeaOBAAAkE0hAQAAZFNIAAAA2dr18a9Add14441Ju1OnTiGfffbZSd9zzz0Xcny08hVXXNFKswOA5uvatWvI8SsAiqIounXrFvJdd90Vckd5JqK53JEAAACyKSQAAIBsjn+tA47Kqy/WWW1YZ/XHWqsNa62+VHOdHXfccSGXvgn+6aefDjl+y/XcuXNbf2I14PhXAACg1SgkAACAbLY21QG3geuLdVYb1ln9sdZqw1qrL625zrbeeuukHZ/GdMMNNyR9119/fchTpkxprSm1GbY2AQAArUYhAQAAZFNIAAAA2TwjUQfsJ60v1lltWGf1x1qrDWutvlhnteEZCQAAoNUoJAAAgGxV3doEAAB0DO5IAAAA2RQSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkE0hAQAAZFNIAAAA2RQSAABANoUEAACQTSEBAABkU0gAAADZFBIAAEA2hQQAAJBNIQEAAGRTSAAAANkUEgAAQDaFBAAAkO3/B2MhQ1pLz20cAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x720 with 20 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot some images\n",
    "%matplotlib inline\n",
    "\n",
    "f, ax = plt.subplots(5,4, figsize=(15,10))\n",
    "\n",
    "for i, img in enumerate(X_trn[:20]):\n",
    "    ax[i//4, i%4].imshow(img, cmap='gray')\n",
    "    ax[i//4, i%4].axis('off')\n",
    "plt.show()  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create the input_functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the train and eval datasets\n",
    "# From https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py\n",
    "\n",
    "def decode_image(image):\n",
    "    # Normalize from [0, 255] to [0.0, 1.0]\n",
    "    image = tf.cast(image, tf.float32)\n",
    "    return image / 255.0\n",
    "\n",
    "def decode_label(label):\n",
    "    label = tf.cast(label, tf.int32)\n",
    "    return label\n",
    "\n",
    "\n",
    "def train_input_fn(features, labels, batch_size):\n",
    "    \"\"\"An input function for training\"\"\"\n",
    "    inputs = features\n",
    "\n",
    "    # Convert the inputs to a Dataset.\n",
    "    ds_images = tf.data.Dataset.from_tensor_slices(inputs).map(decode_image)\n",
    "    ds_labels = tf.data.Dataset.from_tensor_slices(labels).map(decode_label)\n",
    "    dataset = tf.data.Dataset.zip((ds_images, ds_labels))\n",
    "    \n",
    "    # Shuffle, repeat, and batch the examples.\n",
    "    dataset = dataset.cache().shuffle(buffer_size=1000).repeat().batch(batch_size)\n",
    "\n",
    "    # GEnerate iterator and return the next elements of the iterator\n",
    "    # in 1.6 yand above ou can pass directly the dataset and the estimator build internaly the iterator.\n",
    "    (images, labels) = dataset.make_one_shot_iterator().get_next()\n",
    "    return (images, labels)\n",
    "\n",
    "\n",
    "def eval_input_fn(features, labels, batch_size):\n",
    "    \"\"\"An input function for evaluation or prediction\"\"\"\n",
    "    \n",
    "    inputs = features\n",
    "    ds_images = tf.data.Dataset.from_tensor_slices(inputs).map(decode_image)\n",
    "\n",
    "    if labels is None:\n",
    "        dataset = ds_images\n",
    "    else:\n",
    "        ds_labels = tf.data.Dataset.from_tensor_slices(labels).map(decode_label)\n",
    "        dataset = tf.data.Dataset.zip((ds_images, ds_labels))\n",
    "\n",
    "    # Batch the examples\n",
    "    dataset = dataset.batch(batch_size)\n",
    "\n",
    "    (images, labels) = dataset.make_one_shot_iterator().get_next()\n",
    "    \n",
    "    return (images, labels)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configure the model architecture\n",
    "\n",
    "class Model(tf.keras.models.Model):\n",
    "    \"\"\"Model to recognize digits in the MNIST dataset.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        \n",
    "        # Define layers to use in the model\n",
    "        self._input_shape = [-1, 28, 28, 1]\n",
    "\n",
    "        self.conv1 = tf.layers.Conv2D(32, 5, padding='same', activation=tf.nn.relu)\n",
    "        self.max_pool2d = tf.layers.MaxPooling2D((2, 2), (2, 2), padding='same')\n",
    "    \n",
    "        self.conv2 = tf.layers.Conv2D(64, 5, padding='same', activation=tf.nn.relu)\n",
    "    \n",
    "        self.fc1 = tf.layers.Dense(1024, activation=tf.nn.relu)\n",
    "        self.dropout = tf.layers.Dropout(0.4)\n",
    "    \n",
    "        self.fc2 = tf.layers.Dense(10)\n",
    "\n",
    "    \n",
    "    def __call__(self, inputs, training):\n",
    "        \"\"\"Add operations to classify a batch of input images.\n",
    "        Args:\n",
    "          inputs: A Tensor representing a batch of input images.\n",
    "          training: A boolean. Set to True to add operations required only when\n",
    "            training the classifier.\n",
    "        Returns:\n",
    "          A logits Tensor with shape [<batch_size>, 10].\n",
    "        \"\"\"\n",
    "        y = tf.reshape(inputs, self._input_shape)\n",
    "        y = self.conv1(y)\n",
    "        y = self.max_pool2d(y)\n",
    "        y = self.conv2(y)\n",
    "        y = self.max_pool2d(y)\n",
    "        y = tf.layers.flatten(y)\n",
    "        y = self.fc1(y)\n",
    "        y = self.dropout(y, training=training)\n",
    "        return self.fc2(y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the model function\n",
    "\n",
    "\n",
    "# Define the model_function compatible with tf.estimators\n",
    "def model_fn(features, labels, mode, params):\n",
    "    \"\"\"The model_fn argument for creating an Estimator.\"\"\"\n",
    "    image = features\n",
    "    if isinstance(image, dict):\n",
    "        image = features['image']\n",
    "    \n",
    "    # Instanciate the model\n",
    "    model = Model()\n",
    "    \n",
    "\n",
    "    # Train step\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)\n",
    "\n",
    "        logits = model(image, training=True)\n",
    "    \n",
    "        loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "        accuracy = tf.metrics.accuracy(labels=labels, predictions=tf.argmax(logits, axis=1))\n",
    "    \n",
    "        # Name the accuracy tensor 'train_accuracy' to demonstrate the\n",
    "        # LoggingTensorHook.\n",
    "        tf.identity(accuracy[1], name='train_accuracy')\n",
    "    \n",
    "        tf.summary.scalar('train_accuracy', accuracy[1])\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=tf.estimator.ModeKeys.TRAIN,\n",
    "            loss=loss,\n",
    "            train_op=optimizer.minimize(loss, tf.train.get_or_create_global_step()))\n",
    "\n",
    "\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.EVAL:\n",
    "        logits = model(image, training=False)\n",
    "        loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=tf.estimator.ModeKeys.EVAL,\n",
    "            loss=loss,\n",
    "            eval_metric_ops={'accuracy':tf.metrics.accuracy(labels=labels,\n",
    "                                                            predictions=tf.argmax(logits, axis=1)),\n",
    "                            })\n",
    "\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        logits = model(image, training=False)\n",
    "        predictions = {'classes': tf.argmax(logits, axis=1),\n",
    "                       'probabilities': tf.nn.softmax(logits),}\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=tf.estimator.ModeKeys.PREDICT,\n",
    "            predictions=predictions,\n",
    "            export_outputs={'classify': tf.estimator.export.PredictOutput(predictions)})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/tmp/mnist', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x12ecbf0f0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /tmp/mnist/model.ckpt.\n",
      "INFO:tensorflow:train_accuracy = 0.15625\n",
      "INFO:tensorflow:loss = 2.2862, step = 1\n",
      "INFO:tensorflow:train_accuracy = 0.453125 (3.708 sec)\n",
      "INFO:tensorflow:global_step/sec: 15.2601\n",
      "INFO:tensorflow:train_accuracy = 0.5729167 (2.846 sec)\n",
      "INFO:tensorflow:loss = 0.48128533, step = 101 (6.554 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.640625 (2.708 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.0546\n",
      "INFO:tensorflow:train_accuracy = 0.675 (2.830 sec)\n",
      "INFO:tensorflow:loss = 0.61875665, step = 201 (5.539 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.71875 (2.705 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3719\n",
      "INFO:tensorflow:train_accuracy = 0.74553573 (2.739 sec)\n",
      "INFO:tensorflow:loss = 0.23609236, step = 301 (5.443 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.7734375 (2.744 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.1184\n",
      "INFO:tensorflow:train_accuracy = 0.7951389 (2.775 sec)\n",
      "INFO:tensorflow:loss = 0.13317513, step = 401 (5.519 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.809375 (2.763 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.6028\n",
      "INFO:tensorflow:train_accuracy = 0.8238636 (2.918 sec)\n",
      "INFO:tensorflow:loss = 0.13595013, step = 501 (5.681 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.8333333 (2.794 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.9068\n",
      "INFO:tensorflow:train_accuracy = 0.84134614 (2.790 sec)\n",
      "INFO:tensorflow:loss = 0.17086647, step = 601 (5.585 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.84821427 (3.042 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.9734\n",
      "INFO:tensorflow:train_accuracy = 0.8541667 (2.851 sec)\n",
      "INFO:tensorflow:loss = 0.13286963, step = 701 (5.893 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.8613281 (2.819 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.4598\n",
      "INFO:tensorflow:train_accuracy = 0.86580884 (2.906 sec)\n",
      "INFO:tensorflow:loss = 0.17967863, step = 801 (5.725 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.8715278 (2.919 sec)\n",
      "INFO:tensorflow:global_step/sec: 17.1391\n",
      "INFO:tensorflow:train_accuracy = 0.87828946 (2.916 sec)\n",
      "INFO:tensorflow:loss = 0.028510211, step = 901 (5.835 sec)\n",
      "INFO:tensorflow:train_accuracy = 0.884375 (2.852 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into /tmp/mnist/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.3382284.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-11-14:52:33\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /tmp/mnist/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-11-14:52:38\n",
      "INFO:tensorflow:Saving dict for global step 1000: accuracy = 0.971, global_step = 1000, loss = 0.09920357\n",
      "\n",
      "Evaluation results:\n",
      "\t{'accuracy': 0.971, 'loss': 0.09920357, 'global_step': 1000}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['classify', 'serving_default']\n",
      "INFO:tensorflow:Restoring parameters from /tmp/mnist/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: b\"/tmp/mnist_model/temp-b'1526050358'/saved_model.pb\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "b'/tmp/mnist_model/1526050358'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Execute the train process\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.DEBUG)    \n",
    "\n",
    "batch_size = 32\n",
    "train_steps = 1000\n",
    "\n",
    "# Create the classifier\n",
    "mnist_classifier = tf.estimator.Estimator(\n",
    "          model_fn=model_fn,\n",
    "          model_dir='/tmp/mnist',\n",
    "          params={})\n",
    "\n",
    "# Set up training hook that logs the training accuracy every 100 steps.\n",
    "tensors_to_log = {'train_accuracy': 'train_accuracy'}\n",
    "logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=50)\n",
    "\n",
    "# Train the model\n",
    "mnist_classifier.train(input_fn=lambda:train_input_fn(X_trn, y_trn, batch_size),\n",
    "                        hooks=[logging_hook],\n",
    "                        max_steps=train_steps)\n",
    "\n",
    "    \n",
    "# Evaluate the model and print results\n",
    "eval_results = mnist_classifier.evaluate(input_fn=lambda:eval_input_fn(X_tst, y_tst, batch_size))\n",
    "print('\\nEvaluation results:\\n\\t%s' % eval_results)\n",
    "\n",
    "# Export the model for serving\n",
    "image = tf.placeholder(tf.float32, [None, 28, 28, 1])\n",
    "input_fn_export = tf.estimator.export.build_raw_serving_input_receiver_fn({'image': image})\n",
    "mnist_classifier.export_savedmodel('/tmp/mnist_model/', input_fn_export)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tm]",
   "language": "python",
   "name": "conda-env-tm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}