lab-12-1-many-to-one-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-1 many to one\n",
    "### word sentiment classification \n",
    "* many to one \n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', 'a', 'b', 'd', 'e', 'g', 'o', 'r', 's', 'w']\n",
      "{0: '<pad>', 1: ' ', 2: 'a', 3: 'b', 4: 'd', 5: 'e', 6: 'g', 7: 'o', 8: 'r', 9: 's', 10: 'w'}\n",
      "{'<pad>': 0, ' ': 1, 'a': 2, 'b': 3, 'd': 4, 'e': 5, 'g': 6, 'o': 7, 'r': 8, 's': 9, 'w': 10}\n"
     ]
    }
   ],
   "source": [
    "# example data\n",
    "words = ['good', 'bad', 'worse', 'so good']\n",
    "y_data = [1,0,0,1]\n",
    "\n",
    "# creating a token dictionary\n",
    "char_set = ['<pad>'] + sorted(list(set(''.join(words))))\n",
    "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
    "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
    "\n",
    "print(char_set)\n",
    "print(idx2char)\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6, 7, 7, 4], [3, 2, 4], [10, 7, 8, 9, 5], [9, 7, 1, 6, 7, 7, 4]]\n",
      "[4, 3, 5, 7]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "x_data = list(map(lambda word : [char2idx.get(char) for char in word], words))\n",
    "x_data_len = list(map(lambda word : len(word), x_data))\n",
    "\n",
    "print(x_data)\n",
    "print(x_data_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 6  7  7  4  0  0  0  0  0  0]\n",
      " [ 3  2  4  0  0  0  0  0  0  0]\n",
      " [10  7  8  9  5  0  0  0  0  0]\n",
      " [ 9  7  1  6  7  7  4  0  0  0]]\n",
      "[4, 3, 5, 7]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# padding the sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,\n",
    "                       padding = 'post', truncating = 'post')\n",
    "\n",
    "# checking data\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating simple rnn for \"many to one\" classification\n",
    "input_dim = len(char2idx)\n",
    "output_dim = len(char2idx)\n",
    "one_hot = np.eye(len(char2idx))\n",
    "hidden_size = 10\n",
    "num_classes = 2\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
    "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_size))\n",
    "model.add(layers.Dense(units=num_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 11)            121       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10)                220       \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 363\n",
      "Trainable params: 242\n",
      "Non-trainable params: 121\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y):\n",
    "    return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=model(x))\n",
    "\n",
    "# creating an optimizer\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 10), (?,)), types: (tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.156\n",
      "epoch :  10, tr_loss : 0.031\n",
      "epoch :  15, tr_loss : 0.009\n",
      "epoch :  20, tr_loss : 0.005\n",
      "epoch :  25, tr_loss : 0.003\n",
      "epoch :  30, tr_loss : 0.003\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 ==0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc : 100.00%\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print('acc : {:.2%}'.format(np.mean(yhat == y_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fb2fc28a6a0>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHeNJREFUeJzt3Xt0HOWZ5/Hv062bJcuSbQnfJGEHzMX4grFwCMNmssmQNbABJhAumZCQZUJmNz5Jlt09QzI5bJZMMplkh509G58AGchthxiGJIxnYpZkEiYZYIItxzeMLwjbseSr8EWWkWSpu5/9o0ty25Gstt1yqap/n3N0uqr67eqnUuHX5bfeqjJ3R0RE4iURdgEiIlJ4CncRkRhSuIuIxJDCXUQkhhTuIiIxpHAXEYkhhbuISAwp3EVEYkjhLiISQyVhfXFdXZ3PnDkzrK8XEYmkNWvWvOXu9SO1Cy3cZ86cSUtLS1hfLyISSWb223za5dUtY2ZLzGyrmbWa2YPDtLnDzF43s01m9tSZFCsiIoU14pG7mSWBZcD1QDuw2sxWuPvrOW1mA58Dfs/dD5vZBaNVsIiIjCyfI/fFQKu7b3f3PmA5cMspbT4BLHP3wwDufqCwZYqIyJnIJ9xnAG058+3BslyXAJeY2ctm9mszW1KoAkVE5MwV6oRqCTAbeA/QAPzKzOa5+5HcRmZ2P3A/QFNTU4G+WkRETpXPkftuoDFnviFYlqsdWOHu/e6+A9hGNuxP4u6Pu3uzuzfX1484kkdERM5SPuG+GphtZrPMrAy4C1hxSpvnyB61Y2Z1ZLtpthewThEROQMjhru7p4ClwAvAZuAZd99kZg+b2c1BsxeAg2b2OvAi8N/c/eBoFNyy8xBffX4LejygiMjw8upzd/eVwMpTlj2UM+3AA8HfqHptdyeP/vJNPvquC5leO260v05EJJIid2+Z+Y21AGxoPzJCSxGR4hW5cJ8zbQIlCWNDe2fYpYiIjFmRC/eK0iSXTq1WuIuInEbkwh1gfkMtG9qP6KSqiMgwIhnuCxpqONqbYufB7rBLEREZkyIZ7vMbdFJVROR0Ihnul0wZT0VpgvVt6ncXERlKJMO9JJngiuk1OnIXERlGJMMdYH5DDa/t6SSVzoRdiojImBPZcF/QUEtvf4Y3DhwLuxQRkTEnsuE+v6EG0ElVEZGhRDbcZ06uorqihPW6mElE5HdENtwTCWN+g06qiogMJbLhDtnx7lv2dtHbnw67FBGRMSXS4b6goYZUxtm892jYpYiIjCmRDvcTV6qq311EJFekw31aTQV148tZr353EZGTRDrczQZOqurIXUQkV6TDHbLj3d/sOMax46mwSxERGTMiH+4LGmpxh406ehcRGRT5cB+4UnXjbvW7i4gMiHy4Tx5fzozacbpSVUQkR+TDHWBBo65UFRHJFYtwn99QS9uhHg693Rd2KSIiY0JMwl13iBQRyZVXuJvZEjPbamatZvbgEO/fa2YdZrYu+Pvjwpc6vHkzajDTlaoiIgNKRmpgZklgGXA90A6sNrMV7v76KU2fdvelo1DjiKorSnlHXZWO3EVEAvkcuS8GWt19u7v3AcuBW0a3rDO3oKGW9e2duHvYpYiIhC6fcJ8BtOXMtwfLTnWbmW0ws2fNrLEg1Z2B+Q01dHQdZ9/R3vP91SIiY06hTqj+AzDT3ecDPwO+O1QjM7vfzFrMrKWjo6NAX501vzF7h8j1bep3FxHJJ9x3A7lH4g3BskHuftDdjwezfwMsGmpF7v64uze7e3N9ff3Z1DusOdMmUJIw9buLiJBfuK8GZpvZLDMrA+4CVuQ2MLNpObM3A5sLV2J+KkqTXDq1WiNmRETIY7SMu6fMbCnwApAEnnT3TWb2MNDi7iuAT5vZzUAKOATcO4o1D2t+Qy0/2bAHd8fMwihBRGRMGDHcAdx9JbDylGUP5Ux/DvhcYUs7cwsaavjBql3sPNjNrLqqsMsREQlNLK5QHXDisXvqdxeR4harcJ89ZTzlJQmNmBGRohercC9NJrhi+gQduYtI0YtVuEO2a+a1PZ2k0pmwSxERCU3swn1BYw29/RneOHAs7FJEREITu3AfOKmqZ6qKSDGLXbjPmlxFdXkJ69XvLiJFLHbhnkgY8xpqdKWqiBS12IU7ZLtmtuw7yvFUOuxSRERCEctwX9BQQ3/a2by3K+xSRERCEctwH7j9r8a7i0iximW4T6+poG58ma5UFZGiFctwNzPmN9TqyF1EilYswx2yj91r7TjGseOpsEsRETnvYhvuCxpqcYfXdqtrRkSKT2zDfX5DDaCTqiJSnGIb7pPHl9MwcZxOqopIUYptuANc2VjL2l2Hwy5DROS8i3247+nsZf/R3rBLERE5r2Id7gubJgKwdpf63UWkuMQ63K+YPoHSpLGuTeEuIsUl1uFeUZpkzrQJrGtTv7uIFJdYhztk+903tHeSznjYpYiInDfxD/emWrr70mzbrztEikjxiH24L2zMnlRVv7uIFJO8wt3MlpjZVjNrNbMHT9PuNjNzM2suXInn5sLJlUysLNV4dxEpKiOGu5klgWXADcAc4G4zmzNEu2rgM8CrhS7yXJgZVzbW6shdRIpKPkfui4FWd9/u7n3AcuCWIdp9CfhLYMxdMXRl40TeOHCMrt7+sEsRETkv8gn3GUBbznx7sGyQmV0FNLr7TwpYW8EsbMreIVIPzRaRYnHOJ1TNLAE8AvyXPNreb2YtZtbS0dFxrl+dtwXBY/fUNSMixSKfcN8NNObMNwTLBlQDc4F/NrOdwDXAiqFOqrr74+7e7O7N9fX1Z1/1GaoZV8o76qt0UlVEikY+4b4amG1ms8ysDLgLWDHwprt3unudu89095nAr4Gb3b1lVCo+SwsbJ7Ku7QjuuphJROJvxHB39xSwFHgB2Aw84+6bzOxhM7t5tAsslCubannrWB/th3vCLkVEZNSV5NPI3VcCK09Z9tAwbd9z7mUV3sKg331t2xEaJ1WGXI2IyOiK/RWqAy6bWk1FaYJ1uv2viBSBogn3kmSCeTNqWKs7RIpIESiacIfswzs27TnK8VQ67FJEREZVUYX7lY219KUybN6rO0SKSLwVVbgvbAouZtJ4dxGJuaIK92k145gyoVxXqopI7BVVuEO2a2atwl1EYq7own1h00R+e7CbQ2/3hV2KiMioKbpwv3LwJmLqdxeR+Cq6cJ/fUEPC0MVMIhJrRRfulWUlXDp1gvrdRSTWii7cgcHH7mUyukOkiMRTUYb7wqZaunpTbH/r7bBLEREZFcUZ7noyk4jEXFGG+0X146kuL9GTmUQktooy3BMJY0HQ7y4iEkdFGe6QPam6ZV8XPX26Q6SIxE/RhvvCplrSGWfj7s6wSxERKbiiDXddqSoicVa04T55fDlNkypZqytVRSSGijbc4cTFTCIicVP04b63s5d9nb1hlyIiUlBFHe6DT2ZSv7uIxExRh/uc6RMoSyZ0EzERiZ2iDvfykiRzpk/QSVURiZ28wt3MlpjZVjNrNbMHh3j/T8xso5mtM7OXzGxO4UsdHVc21rKxvZNUOhN2KSIiBTNiuJtZElgG3ADMAe4eIryfcvd57n4l8DXgkYJXOkoWNtXS059m6/6usEsRESmYfI7cFwOt7r7d3fuA5cAtuQ3c/WjObBUQmRulL2ycCOgOkSISL/mE+wygLWe+PVh2EjP7lJm9SfbI/dNDrcjM7jezFjNr6ejoOJt6C65x0jgmVZXpsXsiEisFO6Hq7svc/SLgT4EvDNPmcXdvdvfm+vr6Qn31OTEzFjbWasSMiMRKPuG+G2jMmW8Ilg1nOXDruRR1vl114URaDxxj/1FdzCQi8ZBPuK8GZpvZLDMrA+4CVuQ2MLPZObM3AW8UrsTRd+O8aQD8eO3pfrNERKJjxHB39xSwFHgB2Aw84+6bzOxhM7s5aLbUzDaZ2TrgAeBjo1bxKJhVV0XzhRN5dk077pE5FywiMqySfBq5+0pg5SnLHsqZ/kyB6zrvblvUwOd+tJEN7Z0sCG4HLCISVUV9hWqum+ZPo7wkwbNr2sMuRUTknCncAxMqSlkydyor1u+ht1+P3hORaFO457h9UQOdPf38fPOBsEsRETknCvcc115Ux7SaCp5d0zZyYxGRMUzhniOZMD541Qx+9cZbHNCYdxGJMIX7KT54VQPpjPPcOo15F5HoUrif4qL68VzVVKsx7yISaQr3Idy+qJFt+4+xcXdn2KWIiJwVhfsQNOZdRKJO4T6EmnGl/LsrsmPej6c05l1EokfhPozbFjVwpLufX2jMu4hEkMJ9GNddXMeUCeXqmhGRSFK4DyM75r2Bf97WwYEujXkXkWhRuJ/GbcGY979fuyfsUkREzojC/TQuvmA8CzXmXUQiSOE+gtuuamDr/i427TkadikiInlTuI/gA/OnU6Yx7yISMQr3EdRUlvL+OVN4bt1ujXkXkchQuOfh9mDM+4tbNOZdRKJB4Z6HfzO7XmPeRSRSFO55SCaMWxfO4MWtHXR0HQ+7HBGRESnc83T7wJh33eddRCJA4Z6n2VOqWdCoMe8iEg0K9zNw+6IGtuzTmHcRGfsU7mfg5mDM+7df3hl2KSIip5VXuJvZEjPbamatZvbgEO8/YGavm9kGM/u5mV1Y+FLDV1NZysd/byY/WtvOxnY9pUlExq4Rw93MksAy4AZgDnC3mc05pdlaoNnd5wPPAl8rdKFjxdJ/ezGTq8r40j++rr53ERmz8jlyXwy0uvt2d+8DlgO35DZw9xfdvTuY/TXQUNgyx47qilIeuP5SVu08xPOv7Qu7HBGRIeUT7jOAtpz59mDZcO4Dnh/qDTO738xazKylo6Mj/yrHmDuvbuSyqdV8ZeVmevt1SwIRGXsKekLVzD4CNANfH+p9d3/c3Zvdvbm+vr6QX31eJRPGQ/9+Du2He3jy5R1hlyMi8jvyCffdQGPOfEOw7CRm9gfAnwE3u3vsL+O89uI6rp8zhWW/aNWTmkRkzMkn3FcDs81slpmVAXcBK3IbmNlC4DGywV40d9f6/I2X05fO8MhPt4VdiojISUYMd3dPAUuBF4DNwDPuvsnMHjazm4NmXwfGA39nZuvMbMUwq4uVWXVVfOxdM3m6pY1NezQ0UkTGDgtrOF9zc7O3tLSE8t2F1NnTz3u+/iKXTKlm+f3XYGZhlyQiMWZma9y9eaR2ukL1HNWMK+WB91/KqzsO8cKm/WGXIyICKNwL4u6rG7lkyni+snKzntYkImOCwr0ASpIJvnDTHHYd6uY7uu+MiIwBCvcCefcl9bz3sgv4xi9aeetY7EeCisgYp3AvoM/feDk9/Wke+ZmGRopIuBTuBXTxBeO5510XsnzVLjbv1T3fRSQ8CvcC+8z7ZjNhXCl//hPdNVJEwqNwL7DayjI++77ZvNx6kH/aXDQX64rIGKNwHwV/dM2FXFRfxZd/8jpvH0+FXY6IFCGF+ygoTSb40q1z2XWom//6d+vJZNQ9IyLnl8J9lFx7UR2fv/Fynn9tH994sTXsckSkyCjcR9F9183igwtn8MjPtvHTTXpqk4icPwr3UWRmfOWD81jQUMN/fnod2/Z3hV2SiBQJhfsoqyhN8tg9zVSWl/CJ77VwpLsv7JJEpAgo3M+DqTUVPPqRRew90svSp9aSSmfCLklEYk7hfp4sunAif/6Hc3mp9S3+4vktYZcjIjFXEnYBxeSO5kZe33OUJ17aweXTJnD7ooawSxKRmNKR+3n2ZzddzrUXTebzP97I2l2Hwy5HRGJK4X6elSYTLPvwVUyZUM4nv7+G/Ud7wy5JRGJI4R6CiVVlfOujzRw7nuKT319Db7+e3iQihaVwD8llUyfwVx9awLq2I3zhudd0B0kRKSiFe4humDeNT79vNs+uaedb/7I97HJEJEY0WiZkn33fbN48cIyvrNxCbWUZdzQ3hl2SiMSAwj1kiYTxyJ0LONrbz4M/3MCEilKWzJ0adlkiEnF5dcuY2RIz22pmrWb24BDvv9vMfmNmKTO7vfBlxlt5SZLH7lnEgsZaPv2Dtbzc+lbYJYlIxI0Y7maWBJYBNwBzgLvNbM4pzXYB9wJPFbrAYlFZVsK3772aWXVVfOJ7LaxrOxJ2SSISYfkcuS8GWt19u7v3AcuBW3IbuPtOd98A6KYp56C2sozv37eYuvHl3PvtVbyhu0iKyFnKJ9xnAG058+3BMhkFF0yo4P/e905KkwnueWIVbYe6wy5JRCLovA6FNLP7zazFzFo6OjrO51dHStPkSr5/32K6+1Lc88SrdHQdD7skEYmYfMJ9N5A7Pq8hWHbG3P1xd2929+b6+vqzWUXRuGzqBL798cXsP3qcjz65is6e/rBLEpEIySfcVwOzzWyWmZUBdwErRrcsgextgh+7ZxGtB7r44++upqdPtykQkfyMGO7ungKWAi8Am4Fn3H2TmT1sZjcDmNnVZtYOfAh4zMw2jWbRxeTdl9Tz13cupOW3h/lPf7uGfj3oQ0TyYGHd06S5udlbWlpC+e4o+sGqXXzuRxv5wILp/K87FlCS1J0jRIqRma1x9+aR2ukK1Yi4e3ETR3v6+Yvnt9DTl+L/3H0V48qSYZclImOUDv8i5JO/fxFfunUuP99ygHueeJXObp1kFZGhKdwj5p5rLmTZh69iQ3snH3rsFfZ16mEfIvK7FO4RdOO8aXzn41ez50gvt33zFVoPHAu7JBEZYxTuEXXtxXUsv/8ajqfSfOjRV3QvGhE5icI9wubOqOHZP7mW6opS7n781/xym676FZEshXvEzayr4tn/+C5m1lVx33dW89zas7p4WERiRuEeAxdUV/D0J6+heeZEPvv0Op54aUfYJYlIyBTuMTGhopTvfHwxS66Yypf+8XW++vwWPXRbpIgp3GOkojTJsj+6ig+/s4lHf/kmH/zmK/xkw15SumWBSNHRFaoxk0wYX751LnOn1/DoL9/kU0/9hhm147j32pncubiRCRWlYZcoIueB7i0TY+mM80+b9/PESztYteMQVWVJ7ri6kY9fO4umyZVhlyciZyHfe8so3IvExvZOnnx5B/+wfg8Zd66fM4X7rnsHV8+ciJmFXZ6I5EnhLkPa19nL9/51J0+t2sWR7n7mzajhP1w3kxvmTqOiVDciExnrFO5yWj19aX74m3aefHkH2zvepmZcKbdeOZ07rm7kiuk1YZcnIsNQuEteMhnnX7cf5OnVbfy/TfvoS2WYN6OGO65u5OYF06kZpxOwImOJwl3O2JHuPp5bu5unW9rZvPco5SUJbpo3jTuubuSdsyapb15kDFC4y1lzd17bfZTlq3exYt0euo6nmDm5kjuubuT9c6ZyUX2Vgl4kJAp3KYievjTPv7aX5avbWLXjEACTq8pYPGvS4N9lUyeQTCjsRc4HhbsUXNuhbl558y1e3XGIVTsO0X64B4Dq8hKaZ05k8azJLJ41iXkzaigr0cXPIqNB4S6jbs+RHlbvPDQY9gMPDakoTTC/oZbLp1ZzydRqLptazewp1bo6VqQAFO5y3r117DgtQdivazvCtn1dvN2XHnx/ek0Fl0yt5tIp1VwypZpLp1Zz8QXjNb5e5AzkG+66t4wUTN34cpbMncaSudOA7InZ3Ud62La/iy37uti2r4ut+4/xSutB+oKbmSUMptWMo2lSJY2TBl6zf02TKplcVaaTtyJnQeEuo8bMaJhYScPESt572ZTB5al0hp0H32brvmNs3d/FroNvs+tQNy9u7aCj6/hJ66gsS9I4MRv2M2ormFhVxsTKsuC19KTpcaVJ/RCIBBTuct6VJBNcfEE1F19QzU1MO+m9nr407Ye72XUo+9d2qCd47ebVHQfp6k0Nu96yksRg4NdWllIzrpTacWXUDEznLgvmqytKGFeWpCyZ0A+DxEpe4W5mS4D/DSSBv3H3r57yfjnwPWARcBC40913FrZUKQbjypLMnpI9ATuUVDrDkZ5+Dr/dx+Hufg539w0x3UdnTz/bO96ms+cIR3r66Uud/p72yYQxrjRJRWmSyrIk40qTjMt9LUtSUZKkojQx2K6iNBG85syXZKfLShKUlSQoD17Lktnp8pIT72n4qIymEcPdzJLAMuB6oB1YbWYr3P31nGb3AYfd/WIzuwv4S+DO0ShYiltJMkHd+HLqxpef0ed6+9N09vRzpLufzp7+YLqPrt4UPf1pevrS9PSn6e5L09ufprsvRU9/hp6+FAe6+unuS3O8P0Nvf7Zdb3+azDmORUgmjNKkUZpMUJpMUJIYmDZKgmWlSaMkkZ0vSRjJhAWvwXzSfmd5MgEliQQJM0qSln1NGInEibbJhJG07LKEZWtJWPYvmSBnOqeNGWYDn892uyWDdongM8mgrQ0stxPrSgyu9+T3DcOMwe9JWHbeOGV+cPmJZYngX1uJId4r9n+J5XPkvhhodfftAGa2HLgFyA33W4AvBtPPAt8wM3M9503GiIGj6ykTKgqyPnenP+30prJBnxv8x1MZ+lIZjqfSwWtmcFlfzvTxVJpUxulPZ+hPZ0ils+vsT2dIZTInptNOXzr7mVTGybiTSjvpjJPKZILX7Hx/Ovt+OnPKX7Cs2AwEvuX8kDC47MQPiJlhkPOenfTZgfYMtj/xozTQZvD7hnhv8GcmWPbZP7iEDyyYPqrbnk+4zwDacubbgXcO18bdU2bWCUwG3ipEkSJjjZlRVmKUlSQiM37f3ck4pDIZMpng1bM3j0t79kchkyE7nTnxI5EJPjc4nwmm3XF30sH8wHo8aOvOSZ/3nPVkctoNLOOUeXfHya4jeDu7zMEZWGfwWbLbkW0TfD670dl2DHxfdppT1pfbZuB/q4F1DSwfmIfs/wanLv+d9Q4uP3mdONRWjv7/Z87rCVUzux+4H6Cpqel8frVI0ct2o0AyMXBdga4viLN8rhHfDTTmzDcEy4ZsY2YlQA3ZE6sncffH3b3Z3Zvr6+vPrmIRERlRPuG+GphtZrPMrAy4C1hxSpsVwMeC6duBX6i/XUQkPCN2ywR96EuBF8j+O+5Jd99kZg8DLe6+AngC+L6ZtQKHyP4AiIhISPLqc3f3lcDKU5Y9lDPdC3yosKWJiMjZ0n1ZRURiSOEuIhJDCncRkRhSuIuIxFBoD+swsw7gt2f58Trid/Vr3LYpbtsD8dumuG0PxG+bhtqeC919xAuFQgv3c2FmLfk8iSRK4rZNcdseiN82xW17IH7bdC7bo24ZEZEYUriLiMRQVMP98bALGAVx26a4bQ/Eb5vitj0Qv2066+2JZJ+7iIicXlSP3EVE5DQiF+5mtsTMtppZq5k9GHY958rMdprZRjNbZ2YtYddzNszsSTM7YGav5SybZGY/M7M3gteJYdZ4JobZni+a2e5gP60zsxvDrPFMmVmjmb1oZq+b2SYz+0ywPJL76TTbE9n9ZGYVZrbKzNYH2/Q/guWzzOzVIPOeDu7OO/L6otQtEzzPdRs5z3MF7j7lea6RYmY7gWZ3j+zYXDN7N3AM+J67zw2WfQ045O5fDX6EJ7r7n4ZZZ76G2Z4vAsfc/X+GWdvZMrNpwDR3/42ZVQNrgFuBe4ngfjrN9txBRPeTZZ/VV+Xux8ysFHgJ+AzwAPAjd19uZo8C6939myOtL2pH7oPPc3X3PmDgea4SInf/FdlbPee6BfhuMP1dsv/hRcIw2xNp7r7X3X8TTHcBm8k+HjOS++k02xNZnnUsmC0N/hx4L9lnU8MZ7KOohftQz3ON9A4lu/N+amZrgscQxsUUd98bTO8DpoRZTIEsNbMNQbdNJLovhmJmM4GFwKvEYD+dsj0Q4f1kZkkzWwccAH4GvAkccfdU0CTvzItauMfRde5+FXAD8KmgSyBWgqdyRaf/b2jfBC4CrgT2An8Vbjlnx8zGAz8EPuvuR3Pfi+J+GmJ7Ir2f3D3t7leSfZzpYuCys11X1MI9n+e5Roq77w5eDwA/JrtD42B/0C860D96IOR6zom77w/+w8sA3yKC+ynox/0h8Lfu/qNgcWT301DbE4f9BODuR4AXgXcBtcGzqeEMMi9q4Z7P81wjw8yqgpNBmFkV8H7gtdN/KjJyn6v7MeDvQ6zlnA0EYOAPidh+Ck7WPQFsdvdHct6K5H4abnuivJ/MrN7MaoPpcWQHjmwmG/K3B83y3keRGi0DEAxt+mtOPM/1yyGXdNbM7B1kj9Yh+8jDp6K4PWb2A+A9ZO9gtx/478BzwDNAE9m7f97h7pE4STnM9ryH7D/1HdgJfDKnr3rMM7PrgH8BNgKZYPHnyfZTR24/nWZ77iai+8nM5pM9YZoke+D9jLs/HOTEcmASsBb4iLsfH3F9UQt3EREZWdS6ZUREJA8KdxGRGFK4i4jEkMJdRCSGFO4iIjGkcBcRiSGFu4hIDCncRURi6P8D4SsQ3Ivf9lEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}