01_Intro_to_TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intro to TensorFlow\n",
    " - Basic models over MNIST dataset\n",
    "   - Linear model\n",
    "   - NN one layer node\n",
    "   - Convolutional model\n",
    " - Tensoboard example\n",
    " - Save & load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version:  1.7.0\n"
     ]
    }
   ],
   "source": [
    "# Header\n",
    "# Basic libraries & options\n",
    "from __future__ import print_function\n",
    "\n",
    "#Basic libraries\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "print('Tensorflow version: ', tf.__version__)\n",
    "\n",
    "#Show images\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# plt configuration\n",
    "plt.rcParams['figure.figsize'] = (10, 10)        # size of images\n",
    "plt.rcParams['image.interpolation'] = 'nearest'  # show exact image\n",
    "plt.rcParams['image.cmap'] = 'gray'  # use grayscale \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 16214802234297272380\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From <ipython-input-3-60356600ed94>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/mnist/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/mnist/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/mnist/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/jorge/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "mnist = input_data.read_data_sets('/tmp/mnist', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape:  (55000, 784)\n",
      "Valid shape:  (5000, 784)\n",
      "Test  shape:  (10000, 784)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEYCAYAAAB/QtA+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztvXm0FdWZ9//53stlkEFoMUgAA45ZaOLwOv0kr2mjpBUTDa/zi2lM0sG4XhMTjS6MiVNi2qQTTWJ3pMWwYhKXGod0UJOgdoNtusFGUVFEDILKcEVBGUXgnPP8/qg6pDhe7j3DPkOd83zW2ot7atjnqQ9V56naVbW3zAzHcRzHCUVbvQNwHMdxmgtPLI7jOE5QPLE4juM4QfHE4jiO4wTFE4vjOI4TFE8sjuM4TlA8sTiO4zhB6TaxSNqcKDlJWxOfJ9UqyDiWvpJM0sgS1rm+YBvel5SRNKiasXYTT9p9/p2kFyWtl7RW0n2ShlUzziJiSrXTgvXvrmT9EDSDT0n7SLpX0gZJ70qaUa0Yi4gl1T4l7SvpYUlvxuvuU8x63SYWMxuQL8AbwGcT0+4qNrg4wF6lLB8CM7u2YBt+AjxqZhtrHUscT6p9As8D481sMDASWA3cWoc4dtIETvPffRIwol7fnyftPiUJmAksA0YBw6jjPpp2n0AWeBg4p5SVKmoKkzRO0lPxmcFqSbfkNz6RHS+W9CrwYjz9NEl/ic96fyJpnqQLEnVeJGmJpHckPSIpf7D9Z/zvkjjbf67EWNuAC4A7K9nmatLoPs3sTTPrTEzKAgcE2fgq0ehO4/p6E530fC3YhleJFPj8LLAn8C0z22hm283s2XAGwtLoPs1slZlNAxaUtGFmVlQBXgNOLph2DHA00A7sDywFvhLP6wsY8AgwGOgH7ANsBj4DdABXAjuAC+J1zgMWAwfF878HzC6ob2Ti+/sA64Gjioj/08C7QN9it7maJa0+gQPjZXLANuD8ertsAqffAX7Q1fruszSfwPeBh4B7gXXAPOD4ertMq8/EcgPidfcpalsrkdLFMlOBuws24vjE/Cn5jYw/twFvJaTMBiYl5nfE0oZVetABdwHT6r1zNZHPocBVwP+qt8s0OwX2A/4C9K/0/8R9GsCv4nUuiOuaDKwFBrvP8vcvSkwslTaFjZX0R0lrJG0EriH6wUmyIvH3h5OfzSwHrErM/wgwLb7EWw+8DWSI2vMriXMgMJEGbgaD9PiMv2stcDfwe0mqtL5qkQKntwJXm9mWMtevKSnwuRV42cx+Y2Y7zOxOopaKY8usr6qkwGdZVPq48XSitrf9zWwQcANQ+COT7D65k8QGxvc9kjcsVwAXmtngROlnZs8U1FMq5wArzGxuBXXUgrT4zNOLaEffI0Bd1aLRnX4K+JmkN4nOaAEWSDqzjLpqQaP7XFjmevWi0X2WRaWJZSCwwcw2SzoE+HIPy88EjpU0Ib5BdRkwJDF/GvBtSQcDSBqSP8DMbBuwgajpoFQm0+BXKzEN7VPSWZIOUMQw4EfAvAY/225op8Bo4PC45M+q/46oXb0RaXSf9wMjJZ0nqV3RI72DgadKqKOWNLpPJPUluhcD0EdSn+6WJ/6ystsHgZOAV4huJs0hunH2eEH74MiCdU4nukG1nuhJmAXA2Yn5XwIWARuB10ncFyF6amZNvO7p8cZuBo7uJu4xRJeCo0ptV6xmSaNP4PI47i1EZ053FcbjTkvfRxPrNvw9ljT4JLoKXBQv9xRwXL1dptVnIoZkeb+nbVW8cl2IM+6bRM92N3ozVcPjPsPjTsPiPsPSqD5r3qWLpFMl7RlfXl0LvAc8U+s4mgX3GR53Ghb3GZY0+AySWCSdEr+Qs1TS1B4WPwFYTvSI3EnARDPbHiKOZsF9hsedhsV9hqXZfFbcFCapnaiNcDywEphP9NLcS5WH13q4z/C407C4z7A0o88QVyzHAEvNbFmcNe8BzghQb6viPsPjTsPiPsPSdD5DJJYR7PoCz0p66ExPUf83pZS3A8SZFtxneEpy6j57xH2GperHfFWj74IQvWV29db1BzZE0hSi7gjK4fUy10sj7jM8PTp1nyURxGdbWxvbtm2jo6OjcJb7DH/M15QQVywribqnzpPvTn0XzOx2MzvKzI4K8J3NTDCfuVyOXC7HxIkTqxBmqujRqe+fJRHEZzabZe7chnlCtp4EPeYbgRCJZT5woKQxirr/Po/o7dBuOf/881m7du3OkslkeO+995g9e3aAkFJNWT53qWD+fDKZDB/60Idoa2vj/vvvr0qgKaIip7lcjltv/euQHj/60Y/CR5guKt5Hc7kcvXv35oQTTqhKgCmjYp8NR6A3SicQPdXwKlGHej0tb2Zm8+bNs3nz5tknPvEJ+8QnPmHXX3+9mZn9x3/8R+Gbnk/X843ZWpdyfCbLRz/6UQMsm83u8m+r+izVadLVr3/9a9u+ffsu/nK5nPss0ydgjz76qM2cObPQofus4JjPlzVr1nQ5vebbUyeJu92pzjrrLLNooZbe0Sr1OWXKFMtms5bNZrva2dxnkT5feuklmzVrlh1zzDGWy+Usl8tZZ2en+yzTZ/6kMr6h7IklkNNGSyw1f/N+d1xxxRWsWrWK++67r96hNAUTJ05EEieeeCLDhtV1WPpU8+lPf5rZs2fz1ltvcfXVVwMwatSoHtZydseQIVF/iWZG//792bZtW/6H0qmQIUOG8M///M/1DgMI81RYRTz++OOcdNJJADz33HNMnjyZxx57rM5RpZ9TTz0ViNqy29oa5vwhdaxcuZKbbroJgI0bN5LL5chkMnWOKr1s3x69ID5o0CA2bNhQ52iai5///OdccMEFPS9YA4r+xZE0StJsSYslLZJ0aTz9OkmrJD0XlwmlBLBlyxYkIYkjjjiCtWvXlroNqaQaPl96adcXdfv37x846salWvtnkiuuuILTTjstXNANTDWP92w2uzOp5HK5lrmirvY+esopp5DNZsMGXS4ltOkNB46M/x5IdKNpLHAd8M0Q7YOA7dixw+69996mb3Oths/777/fMpmMzZ8/3zKZjGWzWRs6dGhLtGHXYv/s4qa9+yzDZ2dnp7W3t7fcPZZqOj3yyCO72z+t1tta9BWLmXWa2YL4703AYnp4O7RUzj//fLZu3cq5554bstqGpBo+zzrrLKZPn85rr73G9OnTaW9vb5krwFrsnwDXXHNN6Cobkmr6HD58eOOcWdeQajpdsGBBQzV5l3WPRdJo4AiiQXTGAZdI+nvgaeByM3u3i3V6fGt05MiRDBo0qJyQUk1InxdffHFVY00D1do/G+nArSXV8tnKNLvTkns3ljQAeAK40cweVDRE7VqiS67vAsPN7Is91LEJWFLkVw4F+pvZ3iUFmhLcZ1jcZ1gC+XybaNTRYi6fm9on1HwfrY/PEtv1OoBZwGW7mT8aeLGIeopuQy1l2bQV9+k+G7mE8lmKp2b2GdJpo/ss5akwAb8AFpvZzYnpwxOLTQReLLbOVsZ9hsV9hsV9hqeVnJZyj2Uc8HngBUnPxdO+BZwv6XCiy7jXgIuCRti8uM+wuM+wuM/wtIzTohOLmf2Zrrt3/kMZ33t7lZZNDe4zLO4zLIF9QvGemtIn1G0frYvPiocmdhzHcZwkrfn8pOM4jlM1PLE4juM4Qal5YpF0iqQlkpZKmpqYXvW+npoR9xkW9xkW9xmW1Pis8TPc7UQD2ewH9AaeB8bG84L1o9MqxX26z0Yu7rN1fdb6iuUYYKmZLTOz7cA9wBlQu76emgz3GRb3GRb3GZbU+Kx1YhkBrEh8XkkXG1/Qjw5E/egslDRD0pBqB5ki3GdY3GdY3GdYUuOz1omlq2e4d3neOe5H5wHg62a2EbgN2B84HOgEflztIFOE+wyL+wyL+wxLanzWOrGsBJLjuo4EVuc/SOogknKXmT0IYGZrzCxrZjlgOtHloBPhPsPiPsPiPsOSGp+1TizzgQMljZHUGzgPmAmt1Y9OQNxnWNxnWNxnWFLjs6Zj3ptZRtIlRL17tgMzzGxRPLtl+tEJhfsMi/sMi/sMS5p8epcujuM4TlD8zXvHcRwnKJ5YHMdxnKB4YnEcx3GC4onFcRzHCYonFsdxHCconlgcx3GcoHhicRzHcYLiicVxHMcJiicWx3EcJyieWBzHcZygeGJxHMdxguKJxXEcxwmKJxbHcRwnKN0mFkmbEyUnaWvi86RaBRnH0leSSRpZ4nqXS3pd0kZJT0k6rloxFhFLqn1K2lfSw5LejNfdp5oxFhlTqp3G602W9EYc8/2S9qxWjEXEkmqfktokXS9pRXzM/1pS/2rG2UM8qfYZr1f6b6iZFVWI+vI/udjlu1i/V7nrxuv3JRpTYGQJ65wAbAQOI0qiXwdWVRJHqJJSnyOAr8ReDdin3h6bwOkRwAbg/wMGEo0A+Mt6u0yxz4uAF4APA4OAPwL/Wm+XKfZZ1m9oRVKIBpd5Kj4wVgO35Dc+sREXA68CL8fTTwP+AqwHfgLMAy4o2DGWAO8AjwAj4un/E9e3BdgMfK6ImCcD/5n4vFdcx5BG3Mka3WeizgGkJLE0ulPgZqIBm/KfDwG2An3dZ1k+Hwa+mvj8qXjd3u6zdr+hlUo5BjiaaDSz/YGlwFcKpDwCDAb6AfvEG/QZoAO4EtiRl0I01OZi4KB4/veA2QX1jUx8f59Y7lG7iflvgAXAkUSjZV4BzKv3DpZWn4nl0pRYGtop0WiAlxZM2w4c4j7L8vkI8LXE55PiOg52n7X7Da1IShfLTAXuLtiI4xPzp+Q3Mv7cBryVkDIbmJSY3xFLG9aVlCJibgOuBTJxWQMcXu8dLK0+E/WkJrE0ulPgv4ALC6atA45zn2X5vARYBIwChgB/ius4wn3W7je0oqfCJI2V9EdJayRtBK4BhhYstiLx94eTn80sB6xKzP8IME3SeknrgbfjjSnpZlOC/wecCxxMlJm/DPxJ0t5l1ldVUuAzdaTA6WaiewFJBgKbyqyvqqTA523A74kS9kLgsXj6yjLrqyop8FnWb2iljxtPJ7pM2t/MBgE3ACpYxhJ/d5LYQEltRDeE86wgOnsbnCj9zOyZgnqK5TDg92b2qpllzWwm0WXfsWXUVQsa3WcaaXSni4j20/z3jQWyRG3qjUhD+4yP82+Z2b5mNoroXsRyM3u71LpqREP7pMzf0EoTy0Bgg5ltlnQIUTbrjpnAsZImSOoFXEZ0uZpnGvBtSQcDSBoi6UwAM9tGdINrvxLimw+cLukjiphAlNFfKqGOWtLoPpHUl+jMBaCPpD7dLd8ANLrT3wBnSjpO0gDgeuBeM3u/hDpqSUP7lLS3pNHx8f4x4IfAdcWuXwca2ifl/oZW0j5IdGPsFaLL+TnA94HHC9oHRxasczrRDar8Ew0LgLMT879EdBa3EXgdmJaY9zWiNr71cT194u8+upv2wZuIsvimuN5zi93mapaU+szHkCzv19tlmp3G61xI1FSzhehx4z3r7TKtPoFDia5S3gOWk3hCrN4lpT7L+g1VvHJdiDPum8BnzWxu3QJpEtxneNxpWNxnWBrVZ827dJF0qqQ94yaVa4nOLJ6pdRzNgvsMjzsNi/sMSxp8Bkkskk6RtETSUklTe1j8BKJL1LeILgMnmtn2EHE0C+4zPO40LO4zLM3ms+KmMEntRG2E44naiecD55tZo94gb2jcZ3jcaVjcZ1ia0WeIK5ZjgKVmtizOmvcAZwSot1Vxn+Fxp2Fxn2FpOp8hEssIdn2BZyW7PlcNgKQpkp6Oi5VYGvUZ9GrgPsPTo1P3WRLuMyxVP+ZrtiUxIRJL4cs80MWLOGZ2u5kdZWZH7a6icePGkcvlyOVy/M3f/E1y1usB4kwLwXyuWLGCs846q6tZreQTinDak8/JkyeTzWYBuOCCCwpnu88SffaA+yzjmH/11VfJZrNkMhm+8Y1vMGvWrOCBFkuIxLKSqF+ePCOJeuksiR/84Af86U9/4sgjj6RPnz7MmTMnQGipJIhPgBEjPnDS06pU5PSuu+5ixowZnHDCCQDceeedrF+/PmyE6aIinz/4wQ944YUXGDlyJJ/4xCeCB5dCKj7m165dy7Jly2hvb6dXr1780z/9E//2b/8WNMiSCPDSTy9gGTAG6A08Tw89s/LBl+wsl8vt/PsLX/hC4fyn6/1yU61KKJ95p8cdd1xX81rGZzlOk66OO+44y2az9uSTT+7iMJvNus8yfN500032/PPP77KP9u/f3/fPCo/5Bx980Do7Ow2wo446yjZu3GgXX3zxzvk136ZAYiYQPdXwKnB1Ect/4Mcuf6Bms9nCg7YVd7SKfRYm61b2WarTwv1yzpw5u/g74IAD7N1333WfZfjctGnTzr+HDx++u33Uffa8/Ae8ZTIZy2azlslkbO7cufbCCy+kO7GUIXEXIePGjbNcLmdmZr/73e98R6vQJ2Bz5syxG264wRNLBT6vvPJK++///u8uT4JOOukk91miz/zJTr50dHR4Ygng9EMf+pBls1nbsmXLLifl9UwsNX/zvpCtW7fy5JNPMm/ePCRx+umn1zukpuCEE07gmmuuqXcYqefKK6/c5fNpp50GwL//+7/XI5zU09bWtrPs2LGDFStW9LyS0y2rV6/mzDPPpH///jz66KNMmjQJgI997GN1i6noxCJplKTZkhZLWiTp0nj6dZJWSXouLhNKCaCzs5O2tjaOP/54AB599NGdfzcz1fIJsPfeDTncTFWpls877rhj599vvPEGM2fODBx5Y1LN/TPJqFGjel6oSaiWU0k7b9RPmjSJn/70p1WIvkRKuPQaDhwZ/z2QqD1wLFGX1N8s9zIum83a2rVr7Xe/+53lcrkumx5owkvjavkE7OSTT7ZcLmcXXXRRyzSFVcvnunXrdt73SzYtuM/y989ksWiBpvdZTaednZ07761ks1l75plnDLAZM2bUrSmsV/dp56+YWSfRIDOY2SZJi+niJZ5SaW9vZ/r06Zx88sm0tdW9Za5mVMsnwOOPP95SLqF6Pvfaa69Kq0gl1dw/W5VqOR0+fHiX07/4xS9WWnXZlPXrI2k0cATwVDzpEkkLJc2QNGQ36+x8a7Rw3pe//GXGjBlTTihNQWifrY77DEs1fUpdvRvY/DT7PlpyJ5SKRrl7ArjRzB6UNAxYS3TJ9V1guJl1myolbQKWFPmVQ4H+ZtaUNw7cZ1jcZ1gC+XybaBCztUV8ZVP7hJrvo/XxWWK7XgcwC7hsN/NHAy8WUU/RbailLJu24j7dZyOXUD5L8dTMPkM6bXSfpTwVJuAXwGIzuzkxPdnANxF4sdg6Wxn3GRb3GRb3GZ5Wclr0zXtgHPB54AVJz8XTvgWcL+lwosu414CLgkbYvLjPsLjPsLjP8LSM01KeCvszXffC+Ycyvvf2Ki2bGtxnWNxnWAL7hOI9NaVPqNs+WhefFY8g6TiO4zhJWutlB8dxHKfqeGJxHMdxglLzxCLpFElLJC2VNDUxvSZ9EzUb7jMs7jMs7jMsqfFZ42e424nGG9iPvw5oMzaeF6wfnVYp7tN9NnJxn63rs9ZXLMcAS81smZltB+4BzoCoHx0zWxD/vQnwvol6xn2GxX2GxX2GJTU+a51YRgDJARhW0sXGl9OPToviPsPiPsPiPsOSGp+1TixdPcO9y/POcT86DwBfN7ONwG3A/sDhRD2D/rjaQaYI9xkW9xkW9xmW1PisdWJZCSRH9hkJrM5/kNRBJOUuM3sQwMzWmFnWzHLAdKLLQSfCfYbFfYbFfYYlNT5rnVjmAwdKGiOpN3AeMBNaqx+dgLjPsLjPsLjPsKTGZyl9hVWMmWUkXULUu2c7MMPMFsWzW6YfnVC4z7C4z7C4z7Ckyad36eI4juMExd+8dxzHcYLiicVxHMcJiicWx3EcJyieWBzHcZygeGJxHMdxguKJxXEcxwmKJxbHcRwnKJ5YHMdxnKB4YnEcx3GC4onFcRzHCYonFsdxHCconlgcx3GcoHhicRzHcYLSbWKRtDlRcpK2Jj5PqlWQcSx9JZmkkSWuN1nSG3HM90vas1oxFhFLqn1K+jtJL0paL2mtpPskDatmnEXElHan+0p6WNKb8br7VDPGIuJJu8/PSZoraYOkTkm3SdqjmnH2EE/afV5fsA3vS8pIGtTdet0mFjMbkC/AG8BnE9PuKja4OMCajv0Sf+cRwM+Ac4HhREN7/rTWceRJu0/geWC8mQ3mr6PX3VqHOHbSBE6zwMPAOXX47g/QBD4HAdcA+wCHAgcB369DHED6fZrZtQXb8BPg0XjY491SUVOYpHGSnorPDlZLuiW/8YnseLGkV4lHLpN0mqS/xGe9P5E0T9IFiTovkrRE0juSHpE0Ip71n/G/S+LM+bkiQvw88ICZzTWzTUQ73LmS+lay3dWi0X2a2Ztm1pmYlAUOCLLxVSIFTleZ2TRgQdgtrw4p8PkrM3vMzLaa2TqiURXHBZUQkEb3WRBrG3ABcGePC5tZUYVo9LGTC6YdAxxNNJrZ/sBS4CvxvL5Eo5Y9AgwG+hGdRWwGPgN0AFcCO4AL4nXOAxYTnWV0AN8DZhfUNzLx/X2A9cBRu4l5FnBpwbTtwCHFbne1Shp9xsscGC+TA7YB59fbZdqdxssNiNfdp94em8FnYvlpwC/r7bIZfAKfBt4F+va4bCVSulhmKnB3wUYcn5g/Jb+R8ec24K2ElNnApMT8jljasK6kFBHzfwEXFkxbBxzXiDtZo/ssiG0ocBXwv+rtshmckpLEkhafcV2fiY/3MfV22SQ+7wKmFbNspU1hYyX9UdIaSRuJmpqGFiy2IvH3h5OfzSwHrErM/wgwLb7EWw+8DWSI2vPLYTNRm2uSgcCmMuurKinwuRMzWwvcDfxekiqtr1qkyWkaSItPSf8b+CXwOTNbXkld1SRFPgcCEymmGYzKHzeeTtQ2vL+ZDQJuILpBnsQSf3eS2MC4zW5EYv4KoiuMwYnSz8yeKainWBYBhyW+byzRfYFXy6irFjS6z0J6Ee3odXvqpgjS5rTRaXifko4FHiQ6c3+ynDpqSMP7jDkHWGFmc4tZuNLEMhDYYGabJR0CfLmH5WcCx0qaEN+gugwYkpg/Dfi2pIMBJA2RdCaAmW0DNgD7lRDfb4AzJR0naQBwPXCvmb1fQh21pKF9SjpL0gGKGAb8CJhnZluKraMONLTTuI6+RG3dAH0k9elu+TrT0D4VPQn6MDDFzGYVu14daWifCSZT5NUK8ZeV3T4InAS8QtTkNIfosb7HC9oHRxasczrRDar1RI+uLQDOTsz/EtGVxkbgdRJtesDXgDXxuqcTHYybgaO7iftCYCWwBXgA2LOc9sXQJY0+gcvjuLcQnTndVRiPOy3ZaT6GZHm/3i5T7PNuolaJzYnyTL1dptVnvM4Youa0UcVuq+IV60Kccd8kera7qEssZ/e4z/C407C4z7A0qs+ad+ki6VRJe8aX/9cC7wHP1DqOZsF9hsedhsV9hiUNPoMkFkmnxC/kLJU0tYfFTwCWEz0idxIw0cy2h4ijWXCf4XGnYXGfYWk2nxU3hUlqJ2ojHE90L2M+0UtzL1UeXuvhPsPjTsPiPsPSjD5DXLEcAyw1s2Vx1rwHOCNAva2K+wyPOw2L+wxL0/kMkVhGsOsLPCvZ9blqACRNkfR0XKzE8naAONOC+wxPj07dZ0m4z7BU/Ziv2ZbEhOgts6u3rj+wIWZ2O3A7QBkb+noZcaUV9xmeHp26z5Jwn2GpxTFfU0JcsawERiU+57tTLxpJvPzyy5gZL7/8Mr161aO37YahYp87duwgm82SzWbZtm1b0OBSSsVOAW688cZgAaWcinx+4Qtf4KGHHuLEE09kzz33pL29nY6OjuBBpogg+2cjESKxzAcOlDRGUm+i3jVnFrtye3s72WyW++67D0l85zvfYfv2hnrAodZU5BPgvPPOo729nfb2dm688UYee+yxqgSaIip2CjB1ak8P67QMZfvMZDKcdtppXHXVVWSzWdatW8f27dvZsWNHVQNucCraPzOZDNlsdpd/lyxZUrVgiyLQG6UTiJ5qeBW4uojld75lfPHFF9u2bdt2efO4s7PT2tvbk9Oervdbs7UslfgsLM8++6zNmjWrcHpL+SzV6e5cZrPZ3Xl2n0X63LFjh2Wz2Z1ly5YttmnTJvcZ6JgfP368vfPOOx+YXvPtqZPEnRucy+VsyZIlu0j4+7//e7v55ptbekcr12e+TJ482bLZrOVyOf8hDODTE0sYn6NHj7bx48fb+PHjbfjw4QbYD3/4Q/dZgdPC39Nrr7227oml5m/ed8X3vve9eofQNAwYMIAnnngCM+O1117j+eefr3dIjrOT1157jccee4zHHnuMzs5O+vXrxxVXXFHvsFLN4MGDmThxIvfeey9mxne+8x0ymQx9+tSvL9OGSCxvvPHGB6bVvY0wpWzevJlPfvKT/OpXv2L//ffnvffe45xzGmI4dcf5AJs2bWLBglSMytywrFu3jhtuuIE5c+bQ1tZGr1696NWrV30f3Cnh0msU0ehki4l6zrw0nn4d0UAzz8VlQimXcblczubOnfuBy7lmv8dSDZ977733Lh47Ojrs5Zdftr/85S9N39RQrf0zWVqpKaxaPu+55x5bvXq1rV692jKZjGUymZbwWe99tObbWoKU4cCR8d8DiW40jY2lfLNEwTs3eNiwYZbL5eyVV16xZ5991nK5nF188cVNv6NVy+e2bdssl8tZLpezHTt22Lhx41riwK2Wz3wZMWKEbd26tZUSS3CfAwcO3JlM8mXFihUt4bNaTgvL7vbRWm9r0S+MmFkn0RgcmNkmSYvp4u3QUlmzZg29e/fmmmuuAeDoo48mk8lUWm3DUy2f9WxXrSfV8pln1apV9OvXL1R1DU81fA4cOBCAq666iltuuYX+/ftz5JFHVhxrWqj2Pgpw9913h6yufMrMvKOBN4jGk7+OaACbhcAMYMhu1pkCPB2X3Z35tcwZoft0n2kp7jP9Tmu9fSX3bqxoiN8ngBvN7EFFQ9SujTfgu8BwM/tiD3VsAoq9Oz8U6G9me5cUaEpwn2Fxn2EJ5PNtolFH1xbxlU3tE2q+j9bHZ4lZtgOYBVzWTRasZCliAAAYO0lEQVR+sYh6ij4jKWXZtBX36T4buYTyWYqnZvYZ0mmj+yz6cWNJAn4BLDazmxPThycWmwi8WGydrYz7DIv7DIv7DE8rOS2lt8dxwOeBFyQ9F0/7FnC+pMOJLuNeAy4KGmHz4j7D4j7D4j7D0zJOS3kq7M903b3zH8r43turtGxqcJ9hcZ9hCewTivfUlD6hbvtoXXxWPDSx4ziO4yRpiC5dHMdxnObBE4vjOI4TlJonFkmnSFoiaamkqYnpoyTNlrRY0iJJl8bTr5O0StJzcZlQ65gbGfcZFvcZFvcZltT4rPEz3O1EA9nsB/QGngfGxvOC9aPTKsV9us9GLu6zdX3W+orlGGCpmS0zs+3APcAZEPWjY2YL4r83EfUAGrQfnSbEfYbFfYbFfYYlNT5rnVhGACsSn1fSxcZLGg0cATwVT7pE0kJJMyQNqXaQKcJ9hsV9hsV9hiU1PmudWLp6hnuX553jfnQeAL5uZhuB24D9gcOJegb9cbWDTBHuMyzuMyzuMyyp8VnrxLKSaLCbPCOB1fkPkjqIpNxlZg8CmNkaM8uaWQ6YTnQ56ES4z7C4z7C4z7CkxmetE8t84EBJYyT1Bs4DZkJr9aMTEPcZFvcZFvcZltT4LKWvsIoxs4ykS4h692wHZpjZonh2y/SjEwr3GRb3GRb3GZY0+fQuXRzHcZyg+Jv3juM4TlA8sTiO4zhB8cTiOI7jBMUTi+M4jhMUTyyO4zhOUDyxOI7jOEHxxOI4juMExROL4ziOExRPLI7jOE5QPLE4juM4QfHE4jiO4wTFE4vjOI4TFE8sjuM4TlC6TSySNidKTtLWxOdJtQoyjqWvJJM0ssT1Jkt6I475fkl7VivGImJJtU9Jn5M0V9IGSZ2SbpO0RzXjLCKmtDttk3S9pBWSNkr6taT+1Yyzh3hS7TNebx9J98b76buSZlQrxiJiaQafJf+GdptYzGxAvgBvAJ9NTLurxOBqOvZL/J1HAD8DzgWGEw3t+dNax5En7T6BQcA1wD7AocBBwPfrEMdOmsDpl4H/AxxLNCLgUODmbteoImn3GQ94NRNYRjTa4jDg1lrHkacJfJb3G2pmRRWiQWJOLpg2DngK2EA0ROYtQK94Xl+iwWUuBl4FXo6nnwb8BVgP/ASYB1yQqPMiYAnwDvAIMCKe/j9xfVuAzcDnioj5ZqLBcPKfDwG2An2L3e5qlTT67GIb/i8wv94u0+wUeBj4auLzp+J1e7vPsnyeHtelevtrEp9l/YZWKuUY4Gii0cz2B5YCXymQ8ggwGOhHdKa7GfgM0AFcCezISyEaanMx0ZlwB/A9YHZBfSMT398nlnvUbmKeBVxaMG07cEiD7mQN7bOLbZgG/LLeLtPsNP7uryU+nxTXcbD7LMvn94GHgHuBdUQ/usfX22WKfZb1G1qRlC6WmQrcXbARxyfmT8lvZPy5DXgrIWU2MCkxvyOWNqwrKUXE/F/AhQXT1gHHNeJO1ug+C2L7TOxyTL1dptkpcAmwiKjZZgjwp7iOI9xnWT5/Fa9zQVzXZGAtMNh91u43tKKnwiSNlfRHSWskbSRqfx9asNiKxN8fTn42sxywKjH/I8A0SeslrQfeBjJEbc/lsJnovkCSgcCmMuurKinwmY/zfwO/JLqUXl5JXdUmBU5vA35PdAAvBB6Lp68ss76qkgKfW4majH5jZjvM7E7gXaJ7WA1HCnyW9Rta6ePG04EFwP5mNgi4gejmThJL/N1JYgMltQEjEvNXEGXHwYnSz8yeKainWBYBhyW+byyQJWqvbEQa3SeSjgUeJDorerKcOmpMQzs1s6yZfcvM9jWzUURt58vN7O1S66oRDe2TKDmXtW/XiUb3WdZvaKWJZSCwwcw2SzqE6AmX7pgJHCtpQvyEw2VEl/95pgHflnQwgKQhks4EMLNtRDe49ishvt8AZ0o6TtIA4HrgXjN7v4Q6aklD+4yfEHkYmGJms4pdr840utO9JY1WxMeAHwLXFbt+HWhon8D9wEhJ50lqjx/pHUx0g7wRaXSf5f2GVtI+SHSj8RWiy6U5RDfOHi9oHxxZsM7pRDeo8k80LADOTsz/ElGW3Ai8DkxLzPsasCZe93SiG0+bgaO7iftComaFLcADwJ7FbnM1Sxp9AncTna1sTpRn6u0y5U4PJbpKeQ9YTuIJsXqXNPqM1/lUXN9mooRS93uqKfd5ISX+hipesS7EGfdNome759YtkCbBfYbHnYbFfYalUX3WvEsXSadK2lNSX+BaojO1Z2odR7PgPsPjTsPiPsOSBp9BEoukUyQtkbRU0tQeFj+B6JL/LaLLwIlmtj1EHM2C+wyPOw2L+wxLs/msuClMUjtRG+F4ona4+cD5ZvZS5eG1Hu4zPO40LO4zLM3oM8QVyzHAUjNbFmfNe4AzAtTbqrjP8LjTsLjPsDSdzxCJZQS7vsCzkl2fqwZA0hRJT8fFSiyN+kx/NQjmc8SIEda7d+9W9wlFOPX9sySC+RwxYoQNGDDAfVb5N7RmWxITIrEUvswDXbyIY2a3m9lRZnbU7irKZDJks1kymQz77rtvctbrAeJMCxX7vP/++8lms1x33XW8//779OvXr3CRVvIJRTgtZv/sBvdZhs9ly5axcuVKNm36wEvc7rOM39CZM2eybt065s2bFzzAUgmRWFYS9XOUZyRRL51F8y//8i9kMhluvfVW2tvb+ehHP8qyZcsChJZKKvZ5//33097ezpQpU2hvb+frX/960ABTSEVOs9ks2WyWt99+m5deeokdO3YwePDg4EGmiIr30ddee43Ro0dz44030tbW8uMNVuTzjDPOIJPJMGHCBPbaay86OjqYP38+hx12WM8rV4sAL/30Ihr7YAzQG3iennq+jLKxATZ16lTLZDLW2dm5c9oBBxxgmUzGEss9Xe+Xm2pVKvVZWE466SS76qqrCqe3jM9ynCZdZbNZy2azNmrUKAOsf//+ls1m7R//8R/dZxk+zzvvPMvlcpbL5eyrX/3q7vZb91nCMZ/JZGz58uXWr18/A+yqq66ybDZr27Zt27lMzbcpkJgJRE81vApcXcTyuxy4y5Yt22XHOuCAAyybzbbyjla2z8KyadMmO/XUU1v6wC3Vad5Tv3797IUXXtjF3SuvvFK4b7rPIn0CO5PKQw89tHPa6aef7j7LPOb32GOPwpNw22OPPeywww7bZXrNt6dOEnfJtoU/hplMxvbcc8+W3tHK9QnYhz70IXvqqafs/ffft2w2a4cffnjLH7jl+Lzpppt2Ohs0aNDOq5dPfvKT7rMMn/nE8s4779jWrVt3Jpl8cZ+lO+3q9xOwgw46qK6JpaEaN/v37082mwVgw4YNdY4mncybN48nn3ySn//856xbt44TTzyRhx56iKuuuoq1a9fWO7xU8eqrr7J06VKy2SzvvvvuzulPPPFEHaNKP4MHD6ZPnz788Ic/ZP369QA8++yzdY6qeTj00ENZvHgxK1as6HnhalFChhxFNIjMYqIOzi6Np19HNB7Ac3GZUMoZTDKrZjIZy2az9t3vfrfpzwir5XPhwoX25ptvfqC55uqrr3afZfh8/vnn7Z133rHx48fb3Llzbfny5V2dIbrPIn0WXqV0cbXSlD6r5TR/nB955JH2jW98Y+dVtUULNX5TGDAcODL+eyBRe+DYWMo3SxS8SzLJlyeffLLLy7pm3NGq5bN37967c+g+y/CZLNls1oYPH+4+K/T5D//wD7Z27VqbNGlSy+yf1XK6fPly++lPf2pr1qyxbdu22fLly7u6p2q13tZeFImZdRINMoOZbZK0mC5e4imVXr2KDqGpqJbP7dsbqsugmlEtn3lGjBjBrFmz6OzsDFVlQ1NNn3fccQd33HFHiKpSRTWcjhkzBoBLL7204vhCUtY9FkmjgSP46+A5l0haKGmGpCG7WWfnW6NlRdrEuM+wVMPnqlWrmDBhQlXibXR8/wxPszstuRNKRaOIPQHcaGYPShoGrCW65PouMNzMvthDHZuAJUV+5VCgv5ntXVKgKcF9hsV9hiWQz7eJBokq5umRpvYJNd9H6+OzxHa9DmAWcNlu5o8GXiyinqLbUEtZNm3FfbrPRi6hfJbiqZl9hnTa6D6LbgqTJOAXwGIzuzkxfXhisYnAi8XW2cq4z7C4z7C4z/C0ktNS7pyPAz4PvCDpuXjat4DzJR1OdBn3GnBR0AibF/cZFvcZFvcZnpZxWspTYX+m6144/1DG995epWVTg/sMi/sMS2CfULynpvQJddtH6+Kz4hEkHcdxHCdJQ3Xp4jiO46QfTyyO4zhOUGqeWCSdImmJpKWSpiamj5I0W9JiSYskXRpPv07SKknPxaU131LbDe4zLO4zLO4zLKnxWeNnuNuJxhvYj78OaDM2nhesH51WKe7TfTZycZ+t67PWVyzHAEvNbJmZbQfuAc6AqB8dM1sQ/72JqAfQYH09NSnuMyzuMyzuMyyp8VnrxDICSA4SsJIuNr6cfnRaFPcZFvcZFvcZltT4rHVi6eoZ7l2ed4770XkA+LqZbQRuA/YHDifqGfTH1Q4yRbjPsLjPsLjPsKTGZ60Ty0qiwW7yjARW5z9I6iCScpeZPQhgZmvMLGtmOWA60eWgE+E+w+I+w+I+w5Ian7VOLPOBAyWNkdQbOA+YCa3Vj05A3GdY3GdY3GdYUuOzpqNsmVlG0iVEvXu2AzPMbFE8u2X60QmF+wyL+wyL+wxLmnx6ly6O4zhOUPzNe8dxHCconlgcx3GcoHhicRzHcYLiicVxHMcJiicWx3EcJyieWBzHcZygeGJxHMdxguKJxXEcxwmKJxbHcRwnKJ5YHMdxnKB4YnEcx3GC4onFcRzHCYonFsdxHCco3SYWSZsTJSdpa+LzpFoFGcfSV5JJGlnCOvtKeljSm/G6+1QzxiLiSbXPeL19JN0raYOkdyXNqFaMRcaTaqeSri/YhvclZSQNqmas3cSTdp9+zO8+lnJ8tsX76ApJGyX9WlL/ntbrNrGY2YB8Ad4APpuYdlexwcUB1nTsl5gs8DBwTh2++wOk3Wc8mNBMYBnRSHbDgFtrHUeStDs1s2sLtuEnwKPxsLI1J+0+8WM+NF8G/g9wLNGIlUOBm7tdA8DMiipEg8ScXDBtHPAUsIFoiMxbgF7xvL5Eg8tcDLwKvBxPPw34C7Ce6CCaB1yQqPMiYAnwDvAIMCKe/j9xfVuAzcDnSoh9QLzuPsWuU+2SRp/A6XFdqre/ZnFaEGsb0fCz59bbZdp9+jEf7Jh/GPhq4vOn4nV7d7tehVKOAY4mGs1sf2Ap8JUCKY8Ag4F+wD5xUJ8BOoArgR15KURDbS4GDornfw+YXVDfyMT394nlHtUkO1lD+wS+DzwE3Ausi3fo4+vtMs1OC2L9NPAu0LfeLtPuEz/mQx3zjwBfS3w+Ka7j4G63tRIpXSwzFbi7YCOOT8yfkt/I+HMb8FZCymxgUmJ+RyxtWFdSSog9FTtZo/sEfhWvc0Fc12RgLTC43j7T6rQgtruAafX22Aw+/ZgP4xO4BFhE1PQ9BPhTXMcR3a1X0VNhksZK+qOkNZI2AtcQtcElWZH4+8PJz2aWA1Yl5n8EmCZpvaT1wNtAhqhtr+lJgc+tRJfjvzGzHWZ2J9EZ9rFl1ld1UuA0H+dAYCJwZyX1VJu0+EwLKfB5G/B74L+AhcBj8fSV3a1U6ePG04EFwP5mNgi4AVDBMpb4u5PEBkpqA0Yk5q8ALjSzwYnSz8yeKainWWl0nwvLXK+eNLrTPOcAK8xsbgV11IK0+EwLDe3TzLJm9i0z29fMRhHd21luZm93t16liWUgsMHMNks6hOgJgu6YCRwraUL8hMNlRJdXeaYB35Z0MICkIZLOBDCzbUQ3uPYrJUBJfYnaEQH6SOrT3fJ1ptF93g+MlHSepPb4ccnBRDcfG5VGd5pnMg1+tRLT8D79mA/nU9LekkYr4mPAD4HrelyxkvZBohs5rxDdTJpDdHP38YL2wZEF65xOdIMq/0TDAuDsxPwvEbXpbQReJ9HmDHwNWBOvezrRzrMZOHo3MedjSJb3y20jDVnS6DNe51NxfZuJEspx9XbZBE7HEDVXjKq3w7T79GM+uM9Dia5S3gOWk3hCrLuieOW6EGfcN4me7W70JoCGx32Gx52GxX2GpVF91rxLF0mnStozvly9ligTPlPrOJoF9xkedxoW9xmWNPgMklgknSJpiaSlkqb2sPgJRJdUbxFdBk40s+0h4mgW3Gd43GlY3GdYms1nxU1hktqJ2gjHEz2CNh8438xeqjy81sN9hsedhsV9hqUZfYa4YjkGWGpmy+KseQ9wRoB6WxX3GR53Ghb3GZam8xkisYxg1xd4VrLrc9UASJoi6em4WIml22emmwz3GZ4enbrPknCfYan6MV+zLYkJkVgKX+aBLl7EMbPbzewoMzuqjO94vYx10kownzNmzCCXy3U1q5V8QhFOff8sCfcZllr8htaUEIllJVE/MnlGEvXSWRKf/OQnA4TSFATxCXDhhRdyyy23BAkq5QRz6gCBfC5cuJCzzjorWFAppun2zxCJZT5woKQxknoT9a45s9RKxo4dGyCUpiCIzzlz5mBmXH755cEDTCFlO126dGlXL421On7Mh6Vin2+88QYAX/3qV8NHVwYVJxYzyxD1gDmLqLvm35rZolLrOeGEEyoNpSkI6bO9vT10eKmkEqcf//jH6ezsZMyYMQBs3ry5anGmhVD7KMDKld32ZdgSVOrzjjvu4PDDDwfgb//2b6sRYunUqWuDwi4X7Oqrr/7AtER5uh5xpqUU+rrkkkssl8u5z0A+88XM7MQTT3SfAXzutddelslkfP8M4DR5rM+dO3d3+25N46v5m/e7489//nO9Q2gafvazn3HffffRq1cvcrkcy5Ytq3dIqee3v/0tAEuWLKlzJM3Bz3/+cz7+8Y/XO4ym4L777iOXy5HL5Vi6dCkLFy6sd0iNk1gAjjqq4R92SAU7duzg8ccfZ/Xq6P7fkCFDeljD6Y5zzjmHs88+G4BVq1bxjW98o84RpZ9jjz2W5cuX1zuMpuDcc8+lra2NtrY2Pv/5z3PooYfWO6TiE4ukUZJmS1osaZGkS+Pp10laJem5uEwoJ5AnnniinNVSSzV9Xn755fzrv/4rQ4dG4wUdfPDBgaNvPKrp87e//S3btm3jgQce4Oyzz+bZZ58NvwENRjV9HnTQQYwaNYqtW7eGD7yBqfZvaENRQpvecODI+O+BRF0QjCXqm/+blba5ErcVDhs2rCXaXGvhs5viPkv0eeONN9oLL7zgPgP4PPvss7u7v9KUPqvtNFnmzZtnQ4cOTcc9FjPrNLMF8d+biJ5e+MDboZXQ1tbGmjVrQlbZsNTCZytRbZ9XX311QzQx1Ipq+szf/2s1anXMX3vttdx9992hqy2Jsu6xSBoNHMFfRw68RNJCSTMkddmgn+yOoKxImxj3GZZq+ZS6ekG6+fH9MzzVdDpr1izGjx8fNN5SKbl3Y0kDgCeAG83sQUnDgLVEl1zfBYab2Rd7qGMTUOzjNUOB/ma2d0mBpgT3GRb3GZZAPt8GtsTr9URT+4Sa76P18Vliu14H0Us8l+1m/mjgxSLqKboNtZRl01bcp/ts5BLKZymemtlnSKeN7rOUp8IE/AJYbGY3J6YPTyw2EXix2DpbGfcZFvcZFvcZnlZyWsodtHHA54EXJD0XT/sWcL6kw4ku414DLgoaYfPiPsPiPsPiPsPTMk6LTixm9me67t75D2V87+1VWjY1uM+wuM+wBPYJxXtqSp9Qt320Lj4rHprYcRzHcZI0VJcujuM4TvrxxOI4juMEpeaJRdIpkpZIWippamJ66/SjExD3GRb3GRb3GZbU+KzxM9ztwKvAfkBv4HlgbDwvWD86rVLcp/ts5OI+W9dnra9YjgGWmtkyM9sO3AOcAd53Vpm4z7C4z7C4z7CkxmetE8sIYEXi80q62Phy+tFpUdxnWNxnWNxnWFLjs9aJpatnuHd53jnuR+cB4OtmthG4DdgfOBzoBH5c7SBThPsMi/sMi/sMS2p81jqxrARGJT6PBFbnP0jqIJJyl5k9CGBma8wsa2Y5YDrR5aAT4T7D4j7D4j7DkhqftU4s84EDJY2R1Bs4D5gJrdWPTkDcZ1jcZ1jcZ1hS47Omo+2YWUbSJUS9e7YDM8xsUTy7ZfrRCYX7DIv7DIv7DEuafHqXLo7jOE5Q/M17x3EcJyieWBzHcZygeGJxHMdxguKJxXEcxwmKJxbHcRwnKJ5YHMdxnKB4YnEcx3GC8v8DEZGPkllQw3IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 25 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Examine the data\n",
    "print('Train shape: ', mnist.train.images.shape)\n",
    "print('Valid shape: ', mnist.validation.images.shape)\n",
    "print('Test  shape: ', mnist.test.images.shape)\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(25):\n",
    "    a = fig.add_subplot(5,5,i+1)\n",
    "    a.set_title('Target: ' + str(np.argmax(mnist.train.labels[i])))\n",
    "    fig.tight_layout()\n",
    "    plt.imshow( np.reshape(mnist.train.images[i],(28,28)) )\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fist model: Linear model\n",
    " - Start an interactive session\n",
    " - Define the graph\n",
    " - Train the graph\n",
    "   - Initialize variables\n",
    "   - Loop over the data runing the trainer\n",
    " - Validate model with test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start an interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Define the graph\n",
    "# - Placeholders\n",
    "# - Model\n",
    "# - Loss\n",
    "# - Trainer\n",
    "\n",
    "# Inputs\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10])\n",
    "\n",
    "\n",
    "#------------------------------------\n",
    "#-----------   MODEL  ---------------\n",
    "#------------------------------------\n",
    "W = tf.Variable(tf.zeros([784,10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "y_pred = tf.nn.softmax(tf.matmul(x,W) + b)\n",
    "#------------------------------------\n",
    "\n",
    "\n",
    "# Loss\n",
    "cross_entropy = -tf.reduce_sum(y * tf.log(y_pred))\n",
    "\n",
    "# Trainer\n",
    "train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Train the graph\n",
    "# Intialize vars\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Iterate running the trainer\n",
    "batch_size = 128\n",
    "num_epoch = 50\n",
    "for epoch in range(num_epoch):\n",
    "    for i in range(430):  # 215 * batch_size is aprox the train size (55000)\n",
    "        batch = mnist.train.next_batch(batch_size)\n",
    "        train_step.run(feed_dict={x: batch[0], y: batch[1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy:  0.9263\n"
     ]
    }
   ],
   "source": [
    "# Predict and evaluate    \n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_pred,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print('Test accuracy: ', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#When finish, close the interactive session\n",
    "sess.close()\n",
    "\n",
    "# Reset the graph to the next experiments\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2: Neural network model\n",
    " - Add a dense layer between the inputs & the output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense_layer(x, input_dim=10, output_dim=10, name='dense'):\n",
    "    '''\n",
    "    Dense layer function\n",
    "    Inputs:\n",
    "      x: Input tensor\n",
    "      input_dim: Dimmension of the input tensor.\n",
    "      output_dim: dimmension of the output tensor\n",
    "      name: Layer name\n",
    "    '''\n",
    "    W = tf.Variable(tf.truncated_normal([input_dim, output_dim], stddev=0.1), name='W_'+name)\n",
    "    b = tf.Variable(tf.constant(0.1, shape=[output_dim]), name='b_'+name)\n",
    "    dense_output = tf.nn.relu(tf.matmul(x, W) + b)\n",
    "    return dense_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-12-a4413b4a5969>:24: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "Epoch:  0  - Accuracy:  0.3829\n",
      "Epoch:  1  - Accuracy:  0.5818\n",
      "Epoch:  2  - Accuracy:  0.6723\n",
      "Epoch:  3  - Accuracy:  0.6824\n",
      "Epoch:  4  - Accuracy:  0.6851\n",
      "Epoch:  5  - Accuracy:  0.6862\n",
      "Epoch:  6  - Accuracy:  0.6878\n",
      "Epoch:  7  - Accuracy:  0.6897\n",
      "Epoch:  8  - Accuracy:  0.6893\n",
      "Epoch:  9  - Accuracy:  0.6873\n",
      "Epoch:  10  - Accuracy:  0.6866\n",
      "Epoch:  11  - Accuracy:  0.6863\n",
      "Epoch:  12  - Accuracy:  0.6868\n",
      "Epoch:  13  - Accuracy:  0.6879\n",
      "Epoch:  14  - Accuracy:  0.6872\n",
      "Epoch:  15  - Accuracy:  0.687\n",
      "Epoch:  16  - Accuracy:  0.6841\n",
      "Epoch:  17  - Accuracy:  0.6856\n",
      "Epoch:  18  - Accuracy:  0.6882\n",
      "Epoch:  19  - Accuracy:  0.6887\n",
      "Epoch:  20  - Accuracy:  0.689\n",
      "Epoch:  21  - Accuracy:  0.6847\n",
      "Epoch:  22  - Accuracy:  0.6866\n",
      "Epoch:  23  - Accuracy:  0.6869\n",
      "Epoch:  24  - Accuracy:  0.6871\n",
      "Epoch:  25  - Accuracy:  0.6894\n",
      "Epoch:  26  - Accuracy:  0.776\n",
      "Epoch:  27  - Accuracy:  0.7784\n",
      "Epoch:  28  - Accuracy:  0.7815\n",
      "Epoch:  29  - Accuracy:  0.7829\n",
      "Test accuracy:  0.7829\n"
     ]
    }
   ],
   "source": [
    "# Start an interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n",
    "\n",
    "\n",
    "### Define the graph\n",
    "\n",
    "# Inputs\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10])\n",
    "\n",
    "\n",
    "#------------------------------------\n",
    "#-----------   MODEL  ---------------\n",
    "#------------------------------------\n",
    "# First layer\n",
    "dense_1 = dense_layer(x, input_dim=784, output_dim=500, name='dense1')\n",
    "\n",
    "# Final layer\n",
    "dense_2 = dense_layer(dense_1, input_dim=500, output_dim=10, name='dense2')\n",
    "#------------------------------------\n",
    "\n",
    "# Loss function\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=dense_2, labels=y)\n",
    "\n",
    "#Optimizer\n",
    "train_step = tf.train.AdamOptimizer(0.01).minimize(cross_entropy)\n",
    "\n",
    "\n",
    "# Predict and evaluate    \n",
    "y_pred = tf.nn.softmax(dense_2)\n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_pred,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "\n",
    "### Train the graph\n",
    "# Intialize vars\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "\n",
    "# Iterate running the trainer\n",
    "batch_size = 128\n",
    "num_epoch = 30\n",
    "for epoch in range(num_epoch):\n",
    "    for i in range(430):  # 430 * batch_size is aprox the train size (55000)\n",
    "        batch = mnist.train.next_batch(batch_size)\n",
    "        train_step.run(feed_dict={x: batch[0], y: batch[1]})\n",
    "    print('Epoch: ',epoch,' - Accuracy: ', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "\n",
    "print('Test accuracy: ', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "    \n",
    "\n",
    "#When finish, close the interactive session\n",
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 3: Convolutional model\n",
    " - Add 2 convolution and max pooling layers previous to the dense layers\n",
    " - Add a dropout regularization layer in the first dense layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_layer(x, size=2, input_channels=1, output_channels=32, name='conv'):\n",
    "    '''\n",
    "    Function to configure a convolution layer\n",
    "    Inputs:\n",
    "      x: Input tensor\n",
    "      size: Convolution filter size x size\n",
    "      input_channels: Num of input channels\n",
    "      output_channels: Num of output channels\n",
    "    '''\n",
    "    W_conv = tf.Variable(tf.truncated_normal([size, size, input_channels, output_channels], stddev=0.1), name='W_'+name)\n",
    "    b_conv = tf.Variable(tf.constant(0.1, shape=[output_channels]), name='b_'+name)\n",
    "    conv_out = tf.nn.relu(tf.nn.conv2d(x, W_conv, strides=[1, 1, 1, 1], padding='SAME') + b_conv)\n",
    "    return conv_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv - pool 1:  Tensor(\"MaxPool:0\", shape=(?, 14, 14, 20), dtype=float32)\n",
      "Conv - pool 2:  Tensor(\"MaxPool_1:0\", shape=(?, 7, 7, 50), dtype=float32)\n",
      "Dense 1:  Tensor(\"Relu_2:0\", shape=(?, 500), dtype=float32)\n",
      "Dense 2:  Tensor(\"Relu_3:0\", shape=(?, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Start an interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n",
    "\n",
    "#Create the net\n",
    "# Inputs\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10])\n",
    "\n",
    "\n",
    "#------------------------------------\n",
    "#-----------   MODEL  ---------------\n",
    "#------------------------------------\n",
    "#Reshape input data to the original image shape\n",
    "x_image = tf.reshape(x, [-1,28,28,1])\n",
    "\n",
    "# First convolution\n",
    "h_conv1 = conv_layer(x_image, size=5, input_channels=1, output_channels=20, name='conv1')\n",
    "h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "print('Conv - pool 1: ', h_pool1)\n",
    "\n",
    "#Second convolution\n",
    "h_conv2 = conv_layer(h_pool1, size=5, input_channels=20, output_channels=50, name='conv2')\n",
    "h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "print('Conv - pool 2: ', h_pool2)\n",
    "\n",
    "#First dense layer\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*50])\n",
    "h_fc1 = dense_layer(h_pool2_flat, input_dim=7*7*50, output_dim=500, name='dense1')\n",
    "print('Dense 1: ', h_fc1)\n",
    "\n",
    "#Dropout \n",
    "dropout_keep_prob = tf.placeholder(tf.float32)\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, dropout_keep_prob)\n",
    "\n",
    "#Second dense layer\n",
    "h_fc2 = dense_layer(h_fc1_drop, input_dim=500, output_dim=10)\n",
    "print('Dense 2: ', h_fc2)\n",
    "\n",
    "#Prediction\n",
    "y_pred = tf.nn.softmax(h_fc2)\n",
    "#------------------------------------\n",
    "\n",
    "\n",
    "# Loss function\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=h_fc2, labels=y, name='cross_entropy')\n",
    "\n",
    "#Optimizer\n",
    "train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)\n",
    "\n",
    "#Accuracy\n",
    "correct_prediction = tf.equal(tf.argmax(y_pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.140625\n",
      "step 500, training accuracy 0.125\n",
      "step 1000, training accuracy 0.117188\n",
      "step 1500, training accuracy 0.117188\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-2835881225d6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mtrain_step\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_keep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m500\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mtrain_accuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_keep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, feed_dict, session)\u001b[0m\n\u001b[1;32m   2278\u001b[0m         \u001b[0mnone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mdefault\u001b[0m \u001b[0msession\u001b[0m \u001b[0mwill\u001b[0m \u001b[0mbe\u001b[0m \u001b[0mused\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2279\u001b[0m     \"\"\"\n\u001b[0;32m-> 2280\u001b[0;31m     \u001b[0m_run_using_default_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2282\u001b[0m \u001b[0m_gradient_registry\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRegistry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"gradient\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_run_using_default_session\u001b[0;34m(operation, feed_dict, graph, session)\u001b[0m\n\u001b[1;32m   5049\u001b[0m                        \u001b[0;34m\"the operation's graph is different from the session's \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5050\u001b[0m                        \"graph.\")\n\u001b[0;32m-> 5051\u001b[0;31m   \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5052\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5053\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1140\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1141\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1142\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1321\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1310\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1311\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1312\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tm/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1418\u001b[0m         return tf_session.TF_Run(\n\u001b[1;32m   1419\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1420\u001b[0;31m             status, run_metadata)\n\u001b[0m\u001b[1;32m   1421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1422\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Inicialization.\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "# Train proccess\n",
    "for i in range(3000):\n",
    "    batch = mnist.train.next_batch(128)\n",
    "    train_step.run(feed_dict={x: batch[0], y: batch[1], dropout_keep_prob: 0.5})\n",
    "    if i%500 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict={x: batch[0], y: batch[1], dropout_keep_prob: 1})\n",
    "        print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_test = 0\n",
    "for i in range(200):\n",
    "    batch = mnist.test.next_batch(50)\n",
    "    acc_test += accuracy.eval(feed_dict = {x:batch[0], y: batch[1], dropout_keep_prob: 1.0})\n",
    "print(\"test accuracy: \", acc_test/200.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use tensorboard to show the net & the training process.\n",
    "    - The same previous convolutional model with the commands that need tensorboard\n",
    "\n",
    "Based on https://www.tensorflow.org/how_tos/summaries_and_tensorboard/index.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def variable_summaries(var, name):\n",
    "    \"\"\"Attach a lot of summaries to a Tensor.\"\"\"\n",
    "    with tf.name_scope('summaries'):\n",
    "        mean = tf.reduce_mean(var)\n",
    "        tf.summary.scalar('mean/'   + name, mean)\n",
    "        tf.summary.scalar('sttdev/' + name, tf.sqrt(tf.reduce_mean(tf.square(var - mean))))\n",
    "        tf.summary.scalar('max/'    + name, tf.reduce_max(var))\n",
    "        tf.summary.scalar('min/'    + name, tf.reduce_min(var))\n",
    "        tf.summary.histogram(name, var)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_layer(x, size=2, input_channels=1, output_channels=32, name='conv'):\n",
    "    W_conv = tf.Variable(tf.truncated_normal([size, size, input_channels, output_channels], stddev=0.1), name='W_'+name)\n",
    "    b_conv = tf.Variable(tf.constant(0.1, shape=[output_channels]), name='b_'+name)\n",
    "    conv_out = tf.nn.relu(tf.nn.conv2d(x, W_conv, strides=[1, 1, 1, 1], padding='SAME') + b_conv)\n",
    "    # Add summary ops to collect data\n",
    "    variable_summaries(W_conv, \"weights_\"+name) #TENSORBOARD\n",
    "    variable_summaries(b_conv, \"biases_\"+name) #TENSORBOARD\n",
    "    return conv_out\n",
    "\n",
    "\n",
    "def dense_layer(x, input_dim=10, output_dim=10, name='dense'):\n",
    "    W = tf.Variable(tf.truncated_normal([input_dim, output_dim], stddev=0.1), name='W_'+name)\n",
    "    b = tf.Variable(tf.constant(0.1, shape=[output_dim]), name='b_'+name)\n",
    "    dense_output = tf.nn.relu(tf.matmul(x, W) + b)\n",
    "    variable_summaries(W, \"weights_\"+name) #TENSORBOARD\n",
    "    variable_summaries(b, \"biases_\"+name) #TENSORBOARD\n",
    "    return dense_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Start an interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n",
    "\n",
    "\n",
    "#Create the net\n",
    "# Inputs\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784], name='x')\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10] , name='y')\n",
    "\n",
    "\n",
    "\n",
    "#Reshape input data to the original image shape\n",
    "x_image = tf.reshape(x, [-1,28,28,1])\n",
    "\n",
    "# First convolution\n",
    "# use a name scope to organize nodes in the graph visualizer\n",
    "with tf.name_scope(\"conv1\") as scope:\n",
    "    h_conv1 = conv_layer(x_image, size=5, input_channels=1, output_channels=20, name='conv1')\n",
    "    h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "    print('Conv - pool 1: ', h_pool1)\n",
    "    # Add summary ops to collect data\n",
    "    variable_summaries(h_pool1, \"h_pool1_summary\") #TENSORBOARD\n",
    "\n",
    "#Second convolution\n",
    "with tf.name_scope(\"conv2\") as scope:\n",
    "    h_conv2 = conv_layer(h_pool1, size=5, input_channels=20, output_channels=50, name='conv2')\n",
    "    h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "    print('Conv - pool 2: ', h_pool2)\n",
    "    variable_summaries(h_pool2, \"h_pool2_summary\") #TENSORBOARD\n",
    "\n",
    "#First dense layer\n",
    "with tf.name_scope(\"dense1\") as scope:\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*50])\n",
    "    h_fc1 = dense_layer(h_pool2_flat, input_dim=7*7*50, output_dim=500, name='dense1')\n",
    "    variable_summaries(h_fc1, \"dense1_summary\") #TENSORBOARD\n",
    "\n",
    "    #Dropout over \n",
    "    dropout_keep_prob = tf.placeholder(tf.float32, name='dropout')\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, dropout_keep_prob)\n",
    "    \n",
    "#Second dense layer\n",
    "with tf.name_scope(\"dense2\") as scope:\n",
    "    h_fc2 = dense_layer(h_fc1_drop, input_dim=500, output_dim=10, name='dense2')\n",
    "    print('Dense 2: ', h_fc2)\n",
    "    variable_summaries(h_fc2, \"dense2_summary\") #TENSORBOARD\n",
    "\n",
    "#Prediction\n",
    "y_pred = tf.nn.softmax(h_fc2)\n",
    "\n",
    "\n",
    "\n",
    "# Loss function\n",
    "with tf.name_scope(\"xent\") as scope:\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=h_fc2, labels=y, name='cross_entropy')\n",
    "    ce_summ = tf.summary.histogram(\"cross_entropy\", cross_entropy) #TENSORBOARD\n",
    "\n",
    "\n",
    "#Optimizer\n",
    "with tf.name_scope(\"train\") as scope:\n",
    "    train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)\n",
    "\n",
    "#Accuracy\n",
    "with tf.name_scope(\"test\") as scope:\n",
    "    correct_prediction = tf.equal(tf.argmax(y_pred,1), tf.argmax(y,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    accuracy_summary = tf.summary.scalar(\"accuracy\", accuracy) #TENSORBOARD\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge all the summaries and write it into /tmp/tensorboard/mnist_logs\n",
    "summaries_dir = '/tmp/tensorboard/mnist_conv'\n",
    "with tf.name_scope('summaries') as scope:\n",
    "    merged = tf.summary.merge_all()\n",
    "    train_writer = tf.summary.FileWriter(summaries_dir + '/train', sess.graph)\n",
    "    test_writer  = tf.summary.FileWriter(summaries_dir + '/test')\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Inicialization.\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "# Train proccess\n",
    "for i in range(3000):\n",
    "    batch = mnist.train.next_batch(128)\n",
    "    train_step.run(feed_dict={x: batch[0], y: batch[1], dropout_keep_prob: 0.5})\n",
    "    \n",
    "    if i % 50 == 0:  # Record summary data for one batch\n",
    "        summary_str = merged.eval(feed_dict={x: batch[0], y: batch[1], dropout_keep_prob: 1.})\n",
    "        train_writer.add_summary(summary_str, i) #TENSORBOARD\n",
    "        \n",
    "        batch_test = mnist.test.next_batch(128)\n",
    "        summary_str = merged.eval(feed_dict={x: batch_test[0], y: batch_test[1], dropout_keep_prob: 1.})\n",
    "        test_writer.add_summary(summary_str, i) #TENSORBOARD\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At the end execute tensorboar with:\n",
    "\n",
    "    cd /tmp\n",
    "\n",
    "    tensorboard --logdir=./tensorboard\n",
    "\n",
    "And accest to it in:\n",
    "\n",
    "    http://localhost:6006\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save and load models\n",
    "\n",
    "### Create and save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n",
    "\n",
    "# Define graph\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784], name='x')\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10], name='y')\n",
    "\n",
    "with tf.name_scope(\"model\") as scope:\n",
    "    W = tf.Variable(tf.zeros([784,10]), name='W1')\n",
    "    b = tf.Variable(tf.zeros([10]), name='b1')\n",
    "#Prediction\n",
    "y_pred = tf.nn.softmax(tf.matmul(x,W) + b, name='y_pred')\n",
    "\n",
    "#Loss\n",
    "cross_entropy = -tf.reduce_sum(y*tf.log(y_pred), name='cross_entropy')\n",
    "\n",
    "# Train graph\n",
    "train_step = tf.train.GradientDescentOptimizer(0.01, name='train_step').minimize(cross_entropy)\n",
    "\n",
    "\n",
    "# Inicialize graph vars\n",
    "sess.run(tf.global_variables_initializer())\n",
    "for i in range(100):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    train_step.run(feed_dict={x: batch[0], y: batch[1]})\n",
    "\n",
    "# Predict and evaluate    \n",
    "correct_prediction = tf.equal(tf.argmax(y_pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n",
    "\n",
    "print('cross_entropy test', cross_entropy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "print('Accuracy test', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "\n",
    "\n",
    "# Create a saver and save weigths.\n",
    "tf.add_to_collection('cross_entropy', cross_entropy)\n",
    "tf.add_to_collection('x', x)\n",
    "tf.add_to_collection('y', y)\n",
    "tf.add_to_collection('accuracy', accuracy)\n",
    "saver = tf.train.Saver(max_to_keep=0)\n",
    "saver.save(sess, '/tmp/my-model',)\n",
    "\n",
    "\n",
    "#Close session\n",
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the model and evaluate it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pretrained model and evaluate it\n",
    "\n",
    "# Start interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n",
    "\n",
    "#Load model\n",
    "new_saver = tf.train.import_meta_graph('/tmp/my-model.meta')\n",
    "new_saver.restore(sess, '/tmp/my-model')\n",
    "cross_entropy = tf.get_collection('cross_entropy')[0]\n",
    "x = tf.get_collection('x')[0]\n",
    "y = tf.get_collection('y')[0]\n",
    "accuracy = tf.get_collection('accuracy')[0]\n",
    "\n",
    "\n",
    "# Evaluate over the test data\n",
    "print('cross_entropy test', cross_entropy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "print('Accuracy test', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "\n",
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recurent neural networks example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# Start interactive session\n",
    "gpu_options = tf.GPUOptions(allow_growth = True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=True))\n",
    "\n",
    "# Define graph\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784], name='x')\n",
    "y = tf.placeholder(tf.float32, shape=[None, 10], name='y')\n",
    "\n",
    "# Reshape X to a sequnce of columns of dim (batch, time, features)\n",
    "x_seq = tf.reshape(x, [-1,28,28])\n",
    "\n",
    "### RNN layer\n",
    "lstm_size= 128\n",
    "#lstm = tf.nn.rnn_cell.LSTMCell(lstm_size)\n",
    "lstm = tf.contrib.rnn.LSTMCell(lstm_size)\n",
    "rnn_out, _ = tf.nn.dynamic_rnn(lstm, x_seq, dtype=tf.float32)\n",
    "\n",
    "#Prediction\n",
    "W = tf.Variable(tf.zeros([lstm_size,10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "y_pred = tf.nn.softmax(tf.matmul(rnn_out[:,-1,:],W) + b, name='y_pred')\n",
    "\n",
    "#Loss\n",
    "cross_entropy = -tf.reduce_sum(y*tf.log(y_pred), name='cross_entropy')\n",
    "\n",
    "# Train graph\n",
    "train_step = tf.train.GradientDescentOptimizer(0.01, name='train_step').minimize(cross_entropy)\n",
    "\n",
    "# Predict and evaluate    \n",
    "correct_prediction = tf.equal(tf.argmax(y_pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n",
    "\n",
    "\n",
    "\n",
    "# Inicialize graph vars\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "batch_size = 128\n",
    "num_epoch = 10\n",
    "for epoch in range(num_epoch):\n",
    "    for i in range(430):  # 430 * batch_size is aprox the train size (55000)\n",
    "        batch = mnist.train.next_batch(batch_size)\n",
    "        train_step.run(feed_dict={x: batch[0], y: batch[1]})\n",
    "    print('Epoch: ',epoch,' - Accuracy: ', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "\n",
    "print('Test accuracy: ', accuracy.eval(feed_dict={x: mnist.test.images, y: mnist.test.labels}))\n",
    "#Close session\n",
    "sess.close()\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tm]",
   "language": "python",
   "name": "conda-env-tm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}