lab-12-4-many-to-many-bidirectional-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-4 many to many variable bidirectional\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length\n",
    "* bi-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating bidirectional rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.InputLayer(input_shape=(max_sequence,)))\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                                 trainable=False, input_length=max_sequence,\n",
    "                                 embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.Bidirectional(keras.layers.SimpleRNN(units=hidden_dim, return_sequences=True)))\n",
    "model.add(layers.TimeDistributed(keras.layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 10, 20)            520       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             168       \n",
      "=================================================================\n",
      "Total params: 913\n",
      "Trainable params: 688\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)\n",
    "    sequence_loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=model(x),\n",
    "                                                           reduction='none') * masking\n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step\n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)\n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 10), (?, 10), (?,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.052\n",
      "epoch :  10, tr_loss : 0.002\n",
      "epoch :  15, tr_loss : 0.000\n",
      "epoch :  20, tr_loss : 0.000\n",
      "epoch :  25, tr_loss : 0.000\n",
      "epoch :  30, tr_loss : 0.000\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fa354caa7f0>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFmRJREFUeJzt3X+QHGWdx/HPd2d3ZpOdTWJmNiHZJEQIkiASIBFjqVfhhxZ4CnJEhSpPscqLnlBq3dWVnn+oR51VlndanGJB5ZRS7lT0QCFaeEoED7kSzg2GX/khIWKREMgmIcnm1/6a7/0xPctkM7szm8ym00+/X5Wpnenpmfl2deUzzzzdz9Pm7gIAhKUl7gIAAM1HuANAgAh3AAgQ4Q4AASLcASBAhDsABIhwB4AAEe4AECDCHQAC1BrXBxeLRV+4cGFcHw8AibR+/frd7t5Vb73Ywn3hwoXq6emJ6+MBIJHM7M+NrEe3DAAEiHAHgAAR7gAQIMIdAAJEuANAgAh3AAgQ4Q4AAUpcuG95uU//8svNevXQQNylAMBpK3Hh/qfdh/Sth5/Xjn1H4i4FAE5biQv3Yj4rSdp9sD/mSgDg9JXAcM9JkvYcpFsGAMaSuHAvRC33PYdouQPAWBIX7vlcq7KtLbTcAWAciQt3M1OxI6vdhDsAjClx4S5Jxc4cB1QBYByJDPdCR5Y+dwAYR91wN7P5ZvawmW00s2fN7NM11jEz+4aZbTWzp8zs4skpt6yQz9HnDgDjaORKTEOS/t7dnzCzTknrzexBd99Ytc5Vks6Jbm+RdHv0d1IU8lntOTggd5eZTdbHAEBi1W25u/tOd38iut8naZOk7lGrXSPpLi97TNIMM5vT9GojxY6cBoZL6usfmqyPAIBEm1Cfu5ktlHSRpMdHPdUt6cWqx9t1/BeAzGy1mfWYWU9vb+/EKq1S7IxGqfbR7w4AtTQc7maWl3SvpM+4+4ET+TB3X+Puy919eVdX3Yt3j6nQEY1SZfIwAKipoXA3szaVg/377v6TGqvskDS/6vG8aNmkGBmlyumQAFBTI2fLmKTvSNrk7l8fY7W1kj4cnTWzQtJ+d9/ZxDqPUZlfhoFMAFBbI2fLvE3SX0t62sw2RMs+L2mBJLn7HZIekPRuSVslHZb00eaX+pqZHcwMCQDjqRvu7v6opHHPN3R3l3RTs4qqpy3TohlT2zjXHQDGkMgRqhKjVAFgPMkN93yOPncAGENiw72Yz3K2DACMIcHhTssdAMaS2HAvdOS0/8igBoZKcZcCAKed5IZ7NJDp1cO03gFgtMSGezHPue4AMJYEh3s0vwz97gBwnMSGe2FkCgJa7gAwWoLDvTJ5GC13ABgtseHemWtVNtOi3YxSBYDjJDbczWzkcnsAgGMlNtyl8kFVRqkCwPESHe6FfJZRqgBQQ7LDvYOWOwDUkuhwL+az2n1oQOXp5AEAFQkP95wGhko62D8UdykAcFpJdLgXRqYgoN8dAKolPNwrUxDQ7w4A1ZId7h203AGglkSH+8jkYYxSBYBjJDrcZ3YwvwwA1JLocM+2tmj6lDZmhgSAURId7pKYXwYAakh8uBc7crTcAWCUxId7IZ/VnkO03AGgWuLDnZkhAeB4iQ/3Qj6rVw8PanC4FHcpAHDaCCDcy+e6v0rXDACMSHy4FxmlCgDHSX64dzJKFQBGS3y4FxilCgDHSX64R33unOsOAK9JfLhPa29VW8bocweAKokPdzPjWqoAMEriw12Sip2MUgWAakGEOy13ADhW3XA3szvNbJeZPTPG8yvNbL+ZbYhuX2h+meMr5LP0uQNAldYG1vmupNsk3TXOOr919/c0paITUMyXZ4Z0d5lZXGUAwGmjbsvd3R+RtPcU1HLCivms+odKOjQwHHcpAHBaaFaf+1vN7Ekz+4WZvbFJ79mwQkc0SpV+dwCQ1Jxwf0LSme6+VNI3Jd031opmttrMesysp7e3twkfXVbIV+aXIdwBQGpCuLv7AXc/GN1/QFKbmRXHWHeNuy939+VdXV0n+9EjiiOjVDmoCgBSE8LdzM6w6CimmV0Sveeek33fiai03JlfBgDK6p4tY2Y/lLRSUtHMtkv6oqQ2SXL3OyStkvS3ZjYk6Yik693dJ63iGuhzB4Bj1Q13d7+hzvO3qXyqZGyyrS2a1t7KKFUAiAQxQlUq97v30nIHAEkBhXshn6VbBgAiwYR7MZ/jgCoARIIJ90KemSEBoCKccO/I6dXDAxoaLsVdCgDELphwL+azcpf2Hqb1DgDBhHvlWqr0uwNAQOFeJNwBYEQw4T4yBcEhTocEgGDCvdjB5GEAUBFMuE+b0qq2jDHtLwAooHA3My6UDQCRYMJdqkxBQLcMAAQW7jntZpQqAIQV7sWOrHb30S0DAEGFe3l+mX6d4muFAMBpJ6hwL+ZzOjpY0uGB4bhLAYBYBRXuTEEAAGWBhXt5lOpuRqkCSLmgwn1klCoHVQGkXFjh3lmZX4ZuGQDpFlS4z+yIwp1RqgBSLqhwz7Vm1NneyuRhAFIvqHCXogtl0y0DIOWCC/cCo1QBILxwL7fcCXcA6RZcuDMzJAAEGe457T08oOES88sASK/gwr2Yz8pdevUwrXcA6RVguFeupUq/O4D0Ci7cCyMDmWi5A0iv8MKdljsAhBfuxTwtdwAILtyntbeptcVouQNIteDCvaXFONcdQOoFF+6SVOhglCqAdAsz3PNZZoYEkGpBhjvzywBIu7rhbmZ3mtkuM3tmjOfNzL5hZlvN7Ckzu7j5ZU5MMZ/V7j5a7gDSq5GW+3clXTnO81dJOie6rZZ0+8mXdXIK+ZyODA7r8MBQ3KUAQCzqhru7PyJp7zirXCPpLi97TNIMM5vTrAJPBKNUAaRdM/rcuyW9WPV4e7QsNswvAyDtTukBVTNbbWY9ZtbT29s7aZ9TYJQqgJRrRrjvkDS/6vG8aNlx3H2Nuy939+VdXV1N+OjaaLkDSLtmhPtaSR+OzppZIWm/u+9swvuesJmVPnculA0gpVrrrWBmP5S0UlLRzLZL+qKkNkly9zskPSDp3ZK2Sjos6aOTVWyj2tsy6sy10nIHkFp1w93db6jzvEu6qWkVNQnzywBIsyBHqEqMUgWQbsGGe4FRqgBSLOBwp+UOIL2CDfdiR1Z7Dw1ouORxlwIAp1yw4V7I51Ryad9humYApE+w4V4ZyMS57gDSKNhwr0xBsLuPfncA6RNsuBcr4U7LHUAKBRvuhY6oW4ZRqgBSKNhwnz6lTa0txihVAKkUbLi3tJhmdmSZXwZAKgUb7pK0YOZUbdp5IO4yAOCUCzrcV57bpSe379euA0fjLgUATqmgw/3yJbMlSQ9v2RVzJQBwagUd7ovP6FT3jClat4lwB5AuQYe7menyJbP06HO7dXRwOO5yAOCUCTrcJemyxbN0ZHBYv3t+T9ylAMApE3y4rziroKnZjNZteiXuUgDglAk+3NvbMnrHOUU9tHmXylcEBIDwBR/uUvmsmZ37j2oj57wDSIlUhPul586SmfRrzpoBkBKpCPeuzpyWzpuhX9PvDiAlUhHuknTFklnl0ap9jFYFEL7UhPvIaNXNdM0ACF9qwn3xGZ2aO72d0aoAUiE14V4erTqb0aoAUiE14S5Jly+JRqtuY7QqgLClKtwro1U5awZA6FIV7iOjVTcxWhVA2FIV7pJ0+eLZeonRqgACl7pwv3RxebTqQ5w1AyBgqQv3ymjVdZzvDiBgqQt3KRqt+uI+RqsCCFYqw/2yxYxWBRC2VIb7kjnl0arMEgkgVKkM98po1d8yWhVAoFIZ7hKjVQGELbXhzmhVACFrKNzN7Eoz22JmW83sczWev9HMes1sQ3T7WPNLba72tozevojRqgDCVDfczSwj6VuSrpJ0nqQbzOy8Gqv+yN0vjG7fbnKdk+KKJeXRqpt29sVdCgA0VSMt90skbXX3be4+IOluSddMblmnxqWLZ0kSXTMAgtNIuHdLerHq8fZo2WjXmdlTZnaPmc2v9UZmttrMesysp7e39wTKba6uzpyWzme0KoDwNOuA6s8kLXT3CyQ9KOl7tVZy9zXuvtzdl3d1dTXpo0/OFYsZrQogPI2E+w5J1S3xedGyEe6+x937o4fflrSsOeVNvsq1VX+zOf5fEgDQLI2E++8lnWNmrzezrKTrJa2tXsHM5lQ9vFrSpuaVOLkqo1XX0e8OICCt9VZw9yEzu1nSLyVlJN3p7s+a2S2Setx9raRPmdnVkoYk7ZV04yTW3FRmpsuWzNK963fo6OCw2tsycZcEACetoT53d3/A3d/g7me7+5ejZV+Igl3u/o/u/kZ3X+rul7r75sksutnedd4ZOjI4zERiAIKR2hGq1d62qKjZ03K694ntcZcCAE1BuEvKtJiuvWieHt7Sq96+/vovAIDTHOEeWbWsW8Ml1/0bdtRfGQBOc4R7ZNGsTi2dP0P3rKdrBkDyEe5VVl3crc0v9+nZl/bHXQoAnBTCvcp7l85VNtNC6x1A4hHuVWZMzeqd583W/Rte0sBQKe5yAOCEEe6jXLesW3sPDeg3WzjnHUByEe6j/MU5XSrmc3TNAEg0wn2U1kyLrr1orh7avEt7DnLOO4BkItxruG7ZPA2VXGuffCnuUgDghBDuNSw+Y5rO757GdAQAEotwH8Oqi+fpmR0HtGnngbhLAYAJI9zHcPWF3WrLmO7lwCqABCLcxzCzI6vLFs/SfRte0uAw57wDSBbCfRyrls3X7oP9euSPXIIPQLIQ7uNYeW6XCh1ZDqwCSBzCfRxtmRZdc2G31m3cpX2HB+IuBwAaRrjXcd2ybg0Ml/QzznkHkCCEex1vnDtdS+ZMYzoCAIlCuDfguou79eT2/Xrulb64SwGAhhDuDXjfRd1qbTHdw4FVAAlBuDegmM9p5bld+ukTOzTEOe8AEoBwb9CqZfO0q69fj27dHXcpAFAX4d6gyxbP1uumtnFgFUAiEO4Nyra26Oqlc/Wrja9o/5HBuMsBgHER7hOwatl8DQyVmOcdwGmPcJ+A87un6YJ50/XPP9+onz9FwAM4fRHuE2BmuvPGN+tN3dN18w/+oFvX/VHuHndZAHAcwn2Civmcvv83b9FfXdytW9c9p0/dvUFHB4fjLgsAjtEadwFJlGvN6GvvX6pFs/L66n9v0Yt7D2vNh5dpVmd73KUBgCRa7ifMzPTJlYt0x4eWacvLfXrfbf+rZ1/aH3dZACCJcD9pV55/hv7rE2+VS3r/Hb/Tr559Oe6SAIBwb4bzu6fr/pvepnNmd+rj/7led/zP8xxoBRArwr1JZk1r149Wr9BfvmmOvvKLzfqHe55S/xAHWgHEgwOqTdTeltE3b7hIi2bldeu657R110HdcMl8rTiroAUzp8rM4i4RQEoQ7k1mZvrMFW/Q2V153fLzjfrsvU9Lks6Y1q4VZ83UirMKWnFWQWcWCHsAk6ehcDezKyX9m6SMpG+7+1dGPZ+TdJekZZL2SPqgu7/Q3FKT5b1L5+o9F8zR872H9Ni2PXps2x49unWP7ttQHtlK2AOYTFbvwJ+ZZST9UdI7JW2X9HtJN7j7xqp1PinpAnf/hJldL+lad//geO+7fPly7+npOdn6E8XdR8L+8T/t1WPb9qi3r1+SNKUto7kz2jV3xhTNnT5Fc2dM0ZwZ7eqeMUVzppeXt7dlYt4CAHEzs/Xuvrzeeo203C+RtNXdt0VvfLekayRtrFrnGklfiu7fI+k2MzPnlJFjmJkWzcpr0ay8PrTiTLm7tu0uh/223kN6ad8RvbT/qDa/vGsk9KvN7Mhq9rR2dWQzam8r36ZkM5rS1lK+X7WsvbVFubaMspkWZVtb1JZpUa61fD/b2qJspryscr+lRWoxi27lWlvstWUWPW+K/tqxfyuvAXB6aCTcuyW9WPV4u6S3jLWOuw+Z2X5JBUlc2WIcZqazu/I6uyt/3HP9Q8N6ZX+/duw7op37j4wE/64DR3VkcFiHB4a059CA+geHdaRyGxhW/1C8V4oaCX2V/6r8T+W7r30JmCRVfUFED6P3sJHHr31f2Mj7j37Oaj439hfN6KdqrWo6fuFEvrtqrVqrpgl9HU7Cd+dkfB3zJV/f9W+er4+946xJ/YxTekDVzFZLWi1JCxYsOJUfnTi51owWFKZqQWHqhF5XKrn6h0o6PDCkgeGSBoaiW437g8Ml9Q+VNDjsKrnL3VVyqRT9dXeVStXLXO6SSyP3SyUfeVxySVXvUVmu6DVeeY1Lrui9oh93lZ94ld96ledrPafq56rWrzwe/XOx+vejj362xm/LWj83J/IjtPbrG1tvzPechB/Bk/Kzmt/qDSnmc5P+GY2E+w5J86sez4uW1Vpnu5m1Spqu8oHVY7j7GklrpHKf+4kUjPG1tFi5qyZL/zyQZo0MYvq9pHPM7PVmlpV0vaS1o9ZZK+kj0f1Vkh6ivx0A4lO35R71od8s6Zcqnwp5p7s/a2a3SOpx97WSviPpP8xsq6S9Kn8BAABi0lCfu7s/IOmBUcu+UHX/qKT3N7c0AMCJYm4ZAAgQ4Q4AASLcASBAhDsABIhwB4AA1Z04bNI+2KxX0p9P8OVFhTe1QWjbFNr2SOFtU2jbI4W3TbW250x376r3wtjC/WSYWU8js6IlSWjbFNr2SOFtU2jbI4W3TSezPXTLAECACHcACFBSw31N3AVMgtC2KbTtkcLbptC2Rwpvm054exLZ5w4AGF9SW+4AgHEkLtzN7Eoz22JmW83sc3HX0wxm9oKZPW1mG8wscReWNbM7zWyXmT1TtWymmT1oZs9Ff18XZ40TNcY2fcnMdkT7aYOZvTvOGifCzOab2cNmttHMnjWzT0fLE7mfxtmeJO+jdjP7PzN7Mtqmf4qWv97MHo8y70fR1Ov13y9J3TKNXKw7iczsBUnL3T2R5+ea2V9IOijpLnc/P1r2VUl73f0r0Zfw69z9s3HWORFjbNOXJB1093+Ns7YTYWZzJM1x9yfMrFPSeknvk3SjErifxtmeDyi5+8gkdbj7QTNrk/SopE9L+jtJP3H3u83sDklPuvvt9d4vaS33kYt1u/uApMrFuhEjd39E5Xn8q10j6XvR/e+p/B8vMcbYpsRy953u/kR0v0/SJpWvfZzI/TTO9iSWlx2MHrZFN5d0maR7ouUN76OkhXuti3UneodGXNKvzGx9dJ3ZEMx2953R/ZclzY6zmCa62cyeirptEtGFMZqZLZR0kaTHFcB+GrU9UoL3kZllzGyDpF2SHpT0vKR97j4UrdJw5iUt3EP1dne/WNJVkm6KugSCEV1yMTn9f2O7XdLZki6UtFPS1+ItZ+LMLC/pXkmfcfcD1c8lcT/V2J5E7yN3H3b3C1W+VvUlkhaf6HslLdwbuVh34rj7jujvLkk/VXmnJt0rUb9opX90V8z1nDR3fyX6z1eS9O9K2H6K+nHvlfR9d/9JtDix+6nW9iR9H1W4+z5JD0t6q6QZZla5al7DmZe0cG/kYt2JYmYd0QEhmVmHpHdJemb8VyVC9UXTPyLp/hhraYpKCEauVYL2U3Sw7juSNrn716ueSuR+Gmt7Er6PusxsRnR/isonjmxSOeRXRas1vI8SdbaMJEWnNt2q1y7W/eWYSzopZnaWyq11qXxN2x8kbZvM7IeSVqo8g90rkr4o6T5JP5a0QOXZPz/g7ok5QDnGNq1U+ee+S3pB0ser+qtPa2b2dkm/lfS0pFK0+PMq91Mnbj+Nsz03KLn76AKVD5hmVG54/9jdb4ky4m5JMyX9QdKH3L2/7vslLdwBAPUlrVsGANAAwh0AAkS4A0CACHcACBDhDgABItwBIECEOwAEiHAHgAD9PzJMsEJl0U64AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}