Validation on NMT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "64plA8IuYdeS",
    "outputId": "20f42072-dd57-421a-f32e-38bea641f259"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow 2.x selected.\n",
      "Code is built using tensorflow:2.0.0\n"
     ]
    }
   ],
   "source": [
    "%tensorflow_version 2.x\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from model_transformer import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "gw4JM4ynZMdc",
    "outputId": "22d58d7d-ba7a-4ad4-f53b-dd2cf638f350"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Code is built using tensorflow:2.0.0\n"
     ]
    }
   ],
   "source": [
    "%run model_transformer.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g1tjRBXDZmA2"
   },
   "outputs": [],
   "source": [
    "x_train, x_valid = data['train'], data['validation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hhnyJIkiZnul"
   },
   "outputs": [],
   "source": [
    "en_token=tfds.features.text.SubwordTextEncoder.build_from_corpus((j.numpy() for i,j in x_train),target_vocab_size=10000)\n",
    "pt_token=tfds.features.text.SubwordTextEncoder.build_from_corpus((i.numpy() for i,j in x_train),target_vocab_size=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D6lk4754Zrq4"
   },
   "outputs": [],
   "source": [
    "def add_start_end_token(pt,en):\n",
    "    return [pt_token.vocab_size]+pt_token.encode(pt.numpy())+[pt_token.vocab_size+1],[en_token.vocab_size]+en_token.encode(en.numpy())+[en_token.vocab_size+1]\n",
    "           \n",
    "def encode(pt,en):\n",
    "    return tf.py_function(add_start_end_token,[pt,en],Tout=[tf.int64,tf.int64])\n",
    "\n",
    "def filter_sequences_length(pt,en,max_sequence_length=30):\n",
    "    return tf.math.logical_and(tf.shape(pt)[0]<=max_sequence_length,tf.shape(en)[0]<=max_sequence_length)\n",
    "\n",
    "batch_size=32\n",
    "x_train=x_train.map(encode).filter(filter_sequences_length).cache().shuffle(30000).padded_batch(batch_size,padded_shapes=([-1], [-1])).prefetch(tf.data.experimental.AUTOTUNE)\n",
    "x_valid=x_valid.map(encode).filter(filter_sequences_length).padded_batch(batch_size,padded_shapes=([-1], [-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3UTYoO9fZtF1"
   },
   "outputs": [],
   "source": [
    "path=\"./nmt/train/\"\n",
    "ckpt=tf.train.Checkpoint(transformer=transformer,optimizer=optimizer)\n",
    "ckpt_manage=tf.train.CheckpointManager(checkpoint=ckpt,directory=path,max_to_keep=4)\n",
    "if ckpt_manage.latest_checkpoint:\n",
    "    ckpt.restore(ckpt_manage.latest_checkpoint)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "cAo2at8iZvda",
    "outputId": "c0bd63ee-7166-483b-b6a0-eb2a6c7104ce"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"transformer_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "encoder_nx_1 (encoder_nx)    multiple                  5731748   \n",
      "_________________________________________________________________\n",
      "decoder_nx_1 (decoder_nx)    multiple                  7173848   \n",
      "_________________________________________________________________\n",
      "dense_129 (Dense)            multiple                  3055150   \n",
      "=================================================================\n",
      "Total params: 15,960,746\n",
      "Trainable params: 15,960,746\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Model: \"transformer_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "encoder_nx_1 (encoder_nx)    multiple                  5731748   \n",
      "_________________________________________________________________\n",
      "decoder_nx_1 (decoder_nx)    multiple                  7173848   \n",
      "_________________________________________________________________\n",
      "dense_129 (Dense)            multiple                  3055150   \n",
      "=================================================================\n",
      "Total params: 15,960,746\n",
      "Trainable params: 15,960,746\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      " For epoch 1, batch 0  the loss is  4.9803 Accuracy 0.0000\n",
      " For epoch 1, batch 200  the loss is  4.6433 Accuracy 0.0304\n",
      " For epoch 1, batch 400  the loss is  4.2019 Accuracy 0.0388\n",
      " For epoch 1, batch 600  the loss is  3.9323 Accuracy 0.0509\n",
      " For epoch 1, batch 800  the loss is  3.7644 Accuracy 0.0611\n",
      " For epoch 1, batch 1000  the loss is  3.6288 Accuracy 0.0707\n",
      " For epoch 1, batch 1200  the loss is  3.5197 Accuracy 0.0788\n",
      "Epoch 1 Loss 3.5053 Accuracy 0.0798\n",
      " For epoch 2, batch 0  the loss is  2.6314 Accuracy 0.1172\n",
      " For epoch 2, batch 200  the loss is  2.8590 Accuracy 0.1287\n",
      " For epoch 2, batch 400  the loss is  2.8423 Accuracy 0.1300\n",
      " For epoch 2, batch 600  the loss is  2.8199 Accuracy 0.1319\n",
      " For epoch 2, batch 800  the loss is  2.7991 Accuracy 0.1339\n",
      " For epoch 2, batch 1000  the loss is  2.7794 Accuracy 0.1363\n",
      " For epoch 2, batch 1200  the loss is  2.7602 Accuracy 0.1383\n",
      "Saving checkpoint for epoch 2 at ./nmt/train/ckpt-1\n",
      "Epoch 2 Loss 2.7585 Accuracy 0.1386\n",
      " For epoch 3, batch 0  the loss is  2.7898 Accuracy 0.1713\n",
      " For epoch 3, batch 200  the loss is  2.6060 Accuracy 0.1538\n",
      " For epoch 3, batch 400  the loss is  2.5936 Accuracy 0.1541\n",
      " For epoch 3, batch 600  the loss is  2.5800 Accuracy 0.1554\n",
      " For epoch 3, batch 800  the loss is  2.5735 Accuracy 0.1556\n",
      " For epoch 3, batch 1000  the loss is  2.5666 Accuracy 0.1563\n",
      " For epoch 3, batch 1200  the loss is  2.5620 Accuracy 0.1570\n",
      "Epoch 3 Loss 2.5621 Accuracy 0.1570\n",
      " For epoch 4, batch 0  the loss is  2.8859 Accuracy 0.1935\n",
      " For epoch 4, batch 200  the loss is  2.4864 Accuracy 0.1619\n",
      " For epoch 4, batch 400  the loss is  2.4910 Accuracy 0.1623\n",
      " For epoch 4, batch 600  the loss is  2.4893 Accuracy 0.1617\n",
      " For epoch 4, batch 800  the loss is  2.4812 Accuracy 0.1623\n",
      " For epoch 4, batch 1000  the loss is  2.4779 Accuracy 0.1626\n",
      " For epoch 4, batch 1200  the loss is  2.4706 Accuracy 0.1635\n",
      "Saving checkpoint for epoch 4 at ./nmt/train/ckpt-2\n",
      "Epoch 4 Loss 2.4715 Accuracy 0.1636\n",
      " For epoch 5, batch 0  the loss is  2.1724 Accuracy 0.1678\n",
      " For epoch 5, batch 200  the loss is  2.3523 Accuracy 0.1692\n",
      " For epoch 5, batch 400  the loss is  2.3545 Accuracy 0.1681\n",
      " For epoch 5, batch 600  the loss is  2.3614 Accuracy 0.1691\n",
      " For epoch 5, batch 800  the loss is  2.3636 Accuracy 0.1698\n",
      " For epoch 5, batch 1000  the loss is  2.3646 Accuracy 0.1700\n",
      " For epoch 5, batch 1200  the loss is  2.3618 Accuracy 0.1706\n",
      "Epoch 5 Loss 2.3604 Accuracy 0.1707\n",
      " For epoch 6, batch 0  the loss is  1.8965 Accuracy 0.1649\n",
      " For epoch 6, batch 200  the loss is  2.2648 Accuracy 0.1791\n",
      " For epoch 6, batch 400  the loss is  2.2699 Accuracy 0.1770\n",
      " For epoch 6, batch 600  the loss is  2.2999 Accuracy 0.1744\n",
      " For epoch 6, batch 800  the loss is  2.2989 Accuracy 0.1750\n",
      " For epoch 6, batch 1000  the loss is  2.2892 Accuracy 0.1756\n",
      " For epoch 6, batch 1200  the loss is  2.2888 Accuracy 0.1761\n",
      "Saving checkpoint for epoch 6 at ./nmt/train/ckpt-3\n",
      "Epoch 6 Loss 2.2886 Accuracy 0.1762\n",
      " For epoch 7, batch 0  the loss is  2.1604 Accuracy 0.1968\n",
      " For epoch 7, batch 200  the loss is  2.1824 Accuracy 0.1812\n",
      " For epoch 7, batch 400  the loss is  2.1955 Accuracy 0.1820\n",
      " For epoch 7, batch 600  the loss is  2.1911 Accuracy 0.1826\n",
      " For epoch 7, batch 800  the loss is  2.1958 Accuracy 0.1822\n",
      " For epoch 7, batch 1000  the loss is  2.2009 Accuracy 0.1824\n",
      " For epoch 7, batch 1200  the loss is  2.2007 Accuracy 0.1820\n",
      "Epoch 7 Loss 2.2017 Accuracy 0.1819\n",
      " For epoch 8, batch 0  the loss is  1.8962 Accuracy 0.1767\n",
      " For epoch 8, batch 200  the loss is  2.1416 Accuracy 0.1865\n",
      " For epoch 8, batch 400  the loss is  2.1373 Accuracy 0.1856\n",
      " For epoch 8, batch 600  the loss is  2.1337 Accuracy 0.1857\n",
      " For epoch 8, batch 800  the loss is  2.1365 Accuracy 0.1860\n",
      " For epoch 8, batch 1000  the loss is  2.1364 Accuracy 0.1862\n",
      " For epoch 8, batch 1200  the loss is  2.1361 Accuracy 0.1857\n",
      "Saving checkpoint for epoch 8 at ./nmt/train/ckpt-4\n",
      "Epoch 8 Loss 2.1366 Accuracy 0.1858\n",
      " For epoch 9, batch 0  the loss is  2.0712 Accuracy 0.2060\n",
      " For epoch 9, batch 200  the loss is  2.0153 Accuracy 0.1886\n",
      " For epoch 9, batch 400  the loss is  2.0453 Accuracy 0.1892\n",
      " For epoch 9, batch 600  the loss is  2.0540 Accuracy 0.1894\n",
      " For epoch 9, batch 800  the loss is  2.0718 Accuracy 0.1891\n",
      " For epoch 9, batch 1000  the loss is  2.0759 Accuracy 0.1889\n",
      " For epoch 9, batch 1200  the loss is  2.0778 Accuracy 0.1888\n",
      "Epoch 9 Loss 2.0810 Accuracy 0.1889\n",
      " For epoch 10, batch 0  the loss is  2.1498 Accuracy 0.2067\n",
      " For epoch 10, batch 200  the loss is  1.9843 Accuracy 0.1931\n",
      " For epoch 10, batch 400  the loss is  1.9959 Accuracy 0.1929\n",
      " For epoch 10, batch 600  the loss is  2.0040 Accuracy 0.1921\n",
      " For epoch 10, batch 800  the loss is  2.0086 Accuracy 0.1920\n",
      " For epoch 10, batch 1000  the loss is  2.0181 Accuracy 0.1920\n",
      " For epoch 10, batch 1200  the loss is  2.0200 Accuracy 0.1917\n",
      "Saving checkpoint for epoch 10 at ./nmt/train/ckpt-5\n",
      "Epoch 10 Loss 2.0192 Accuracy 0.1916\n",
      " For epoch 11, batch 0  the loss is  1.7425 Accuracy 0.1746\n",
      " For epoch 11, batch 200  the loss is  1.9029 Accuracy 0.1952\n",
      " For epoch 11, batch 400  the loss is  1.9290 Accuracy 0.1942\n",
      " For epoch 11, batch 600  the loss is  1.9561 Accuracy 0.1931\n",
      " For epoch 11, batch 800  the loss is  1.9689 Accuracy 0.1930\n",
      " For epoch 11, batch 1000  the loss is  1.9774 Accuracy 0.1927\n",
      " For epoch 11, batch 1200  the loss is  1.9845 Accuracy 0.1931\n",
      "Epoch 11 Loss 1.9853 Accuracy 0.1931\n",
      " For epoch 12, batch 0  the loss is  1.9525 Accuracy 0.2118\n",
      " For epoch 12, batch 200  the loss is  1.9118 Accuracy 0.1988\n",
      " For epoch 12, batch 400  the loss is  1.9087 Accuracy 0.1985\n",
      " For epoch 12, batch 600  the loss is  1.9135 Accuracy 0.1979\n",
      " For epoch 12, batch 800  the loss is  1.9160 Accuracy 0.1969\n",
      " For epoch 12, batch 1000  the loss is  1.9228 Accuracy 0.1965\n",
      " For epoch 12, batch 1200  the loss is  1.9286 Accuracy 0.1965\n",
      "Saving checkpoint for epoch 12 at ./nmt/train/ckpt-6\n",
      "Epoch 12 Loss 1.9286 Accuracy 0.1965\n",
      " For epoch 13, batch 0  the loss is  1.8808 Accuracy 0.1940\n",
      " For epoch 13, batch 200  the loss is  1.8490 Accuracy 0.2046\n",
      " For epoch 13, batch 400  the loss is  1.8538 Accuracy 0.2029\n",
      " For epoch 13, batch 600  the loss is  1.8646 Accuracy 0.2019\n",
      " For epoch 13, batch 800  the loss is  1.8706 Accuracy 0.2006\n",
      " For epoch 13, batch 1000  the loss is  1.8735 Accuracy 0.1997\n",
      " For epoch 13, batch 1200  the loss is  1.8787 Accuracy 0.1996\n",
      "Epoch 13 Loss 1.8802 Accuracy 0.1996\n",
      " For epoch 14, batch 0  the loss is  1.6635 Accuracy 0.2025\n",
      " For epoch 14, batch 200  the loss is  1.7829 Accuracy 0.2063\n",
      " For epoch 14, batch 400  the loss is  1.8020 Accuracy 0.2037\n",
      " For epoch 14, batch 600  the loss is  1.8197 Accuracy 0.2032\n",
      " For epoch 14, batch 800  the loss is  1.8311 Accuracy 0.2026\n",
      " For epoch 14, batch 1000  the loss is  1.8384 Accuracy 0.2021\n",
      " For epoch 14, batch 1200  the loss is  1.8461 Accuracy 0.2013\n",
      "Saving checkpoint for epoch 14 at ./nmt/train/ckpt-7\n",
      "Epoch 14 Loss 1.8457 Accuracy 0.2014\n",
      " For epoch 15, batch 0  the loss is  1.6926 Accuracy 0.1920\n",
      " For epoch 15, batch 200  the loss is  1.7306 Accuracy 0.2079\n",
      " For epoch 15, batch 400  the loss is  1.7461 Accuracy 0.2064\n",
      " For epoch 15, batch 600  the loss is  1.7618 Accuracy 0.2057\n",
      " For epoch 15, batch 800  the loss is  1.7733 Accuracy 0.2051\n",
      " For epoch 15, batch 1000  the loss is  1.7870 Accuracy 0.2042\n",
      " For epoch 15, batch 1200  the loss is  1.7988 Accuracy 0.2039\n",
      "Epoch 15 Loss 1.7990 Accuracy 0.2038\n",
      " For epoch 16, batch 0  the loss is  1.6746 Accuracy 0.2222\n",
      " For epoch 16, batch 200  the loss is  1.7077 Accuracy 0.2097\n",
      " For epoch 16, batch 400  the loss is  1.7167 Accuracy 0.2094\n",
      " For epoch 16, batch 600  the loss is  1.7288 Accuracy 0.2086\n",
      " For epoch 16, batch 800  the loss is  1.7443 Accuracy 0.2076\n",
      " For epoch 16, batch 1000  the loss is  1.7543 Accuracy 0.2069\n",
      " For epoch 16, batch 1200  the loss is  1.7629 Accuracy 0.2063\n",
      "Saving checkpoint for epoch 16 at ./nmt/train/ckpt-8\n",
      "Epoch 16 Loss 1.7646 Accuracy 0.2062\n",
      " For epoch 17, batch 0  the loss is  1.8634 Accuracy 0.2132\n",
      " For epoch 17, batch 200  the loss is  1.6749 Accuracy 0.2129\n",
      " For epoch 17, batch 400  the loss is  1.6785 Accuracy 0.2109\n",
      " For epoch 17, batch 600  the loss is  1.6929 Accuracy 0.2101\n",
      " For epoch 17, batch 800  the loss is  1.7107 Accuracy 0.2092\n",
      " For epoch 17, batch 1000  the loss is  1.7266 Accuracy 0.2077\n",
      " For epoch 17, batch 1200  the loss is  1.7356 Accuracy 0.2076\n",
      "Epoch 17 Loss 1.7377 Accuracy 0.2075\n",
      " For epoch 18, batch 0  the loss is  1.3413 Accuracy 0.2144\n",
      " For epoch 18, batch 200  the loss is  1.6730 Accuracy 0.2148\n",
      " For epoch 18, batch 400  the loss is  1.6730 Accuracy 0.2141\n",
      " For epoch 18, batch 600  the loss is  1.6783 Accuracy 0.2128\n",
      " For epoch 18, batch 800  the loss is  1.6897 Accuracy 0.2123\n",
      " For epoch 18, batch 1000  the loss is  1.7003 Accuracy 0.2118\n",
      " For epoch 18, batch 1200  the loss is  1.7049 Accuracy 0.2115\n",
      "Saving checkpoint for epoch 18 at ./nmt/train/ckpt-9\n",
      "Epoch 18 Loss 1.7053 Accuracy 0.2113\n",
      " For epoch 19, batch 0  the loss is  1.7961 Accuracy 0.1950\n",
      " For epoch 19, batch 200  the loss is  1.6059 Accuracy 0.2191\n",
      " For epoch 19, batch 400  the loss is  1.6152 Accuracy 0.2179\n",
      " For epoch 19, batch 600  the loss is  1.6296 Accuracy 0.2168\n",
      " For epoch 19, batch 800  the loss is  1.6392 Accuracy 0.2155\n",
      " For epoch 19, batch 1000  the loss is  1.6493 Accuracy 0.2143\n",
      " For epoch 19, batch 1200  the loss is  1.6566 Accuracy 0.2139\n",
      "Epoch 19 Loss 1.6580 Accuracy 0.2139\n",
      " For epoch 20, batch 0  the loss is  1.7175 Accuracy 0.2198\n",
      " For epoch 20, batch 200  the loss is  1.6069 Accuracy 0.2213\n",
      " For epoch 20, batch 400  the loss is  1.6083 Accuracy 0.2205\n",
      " For epoch 20, batch 600  the loss is  1.6077 Accuracy 0.2189\n",
      " For epoch 20, batch 800  the loss is  1.6164 Accuracy 0.2181\n",
      " For epoch 20, batch 1000  the loss is  1.6248 Accuracy 0.2178\n",
      " For epoch 20, batch 1200  the loss is  1.6361 Accuracy 0.2170\n",
      "Saving checkpoint for epoch 20 at ./nmt/train/ckpt-10\n",
      "Epoch 20 Loss 1.6385 Accuracy 0.2170\n",
      " For epoch 21, batch 0  the loss is  1.7177 Accuracy 0.2612\n",
      " For epoch 21, batch 200  the loss is  1.5435 Accuracy 0.2250\n",
      " For epoch 21, batch 400  the loss is  1.5643 Accuracy 0.2243\n",
      " For epoch 21, batch 600  the loss is  1.5696 Accuracy 0.2234\n",
      " For epoch 21, batch 800  the loss is  1.5822 Accuracy 0.2218\n",
      " For epoch 21, batch 1000  the loss is  1.5981 Accuracy 0.2204\n",
      " For epoch 21, batch 1200  the loss is  1.6071 Accuracy 0.2197\n",
      "Epoch 21 Loss 1.6084 Accuracy 0.2196\n",
      " For epoch 22, batch 0  the loss is  1.5998 Accuracy 0.2188\n",
      " For epoch 22, batch 200  the loss is  1.5234 Accuracy 0.2291\n",
      " For epoch 22, batch 400  the loss is  1.5479 Accuracy 0.2276\n",
      " For epoch 22, batch 600  the loss is  1.5646 Accuracy 0.2260\n",
      " For epoch 22, batch 800  the loss is  1.5717 Accuracy 0.2248\n",
      " For epoch 22, batch 1000  the loss is  1.5795 Accuracy 0.2234\n",
      " For epoch 22, batch 1200  the loss is  1.5836 Accuracy 0.2224\n",
      "Saving checkpoint for epoch 22 at ./nmt/train/ckpt-11\n",
      "Epoch 22 Loss 1.5851 Accuracy 0.2224\n",
      " For epoch 23, batch 0  the loss is  1.2910 Accuracy 0.2500\n",
      " For epoch 23, batch 200  the loss is  1.5046 Accuracy 0.2321\n",
      " For epoch 23, batch 400  the loss is  1.5081 Accuracy 0.2292\n",
      " For epoch 23, batch 600  the loss is  1.5223 Accuracy 0.2282\n",
      " For epoch 23, batch 800  the loss is  1.5372 Accuracy 0.2273\n",
      " For epoch 23, batch 1000  the loss is  1.5481 Accuracy 0.2260\n",
      " For epoch 23, batch 1200  the loss is  1.5563 Accuracy 0.2248\n",
      "Epoch 23 Loss 1.5583 Accuracy 0.2248\n",
      " For epoch 24, batch 0  the loss is  1.6093 Accuracy 0.2723\n",
      " For epoch 24, batch 200  the loss is  1.4928 Accuracy 0.2378\n",
      " For epoch 24, batch 400  the loss is  1.5015 Accuracy 0.2341\n",
      " For epoch 24, batch 600  the loss is  1.5091 Accuracy 0.2323\n",
      " For epoch 24, batch 800  the loss is  1.5185 Accuracy 0.2310\n",
      " For epoch 24, batch 1000  the loss is  1.5245 Accuracy 0.2299\n",
      " For epoch 24, batch 1200  the loss is  1.5330 Accuracy 0.2285\n",
      "Saving checkpoint for epoch 24 at ./nmt/train/ckpt-12\n",
      "Epoch 24 Loss 1.5336 Accuracy 0.2283\n",
      " For epoch 25, batch 0  the loss is  1.4303 Accuracy 0.2254\n",
      " For epoch 25, batch 200  the loss is  1.4682 Accuracy 0.2387\n",
      " For epoch 25, batch 400  the loss is  1.4751 Accuracy 0.2369\n",
      " For epoch 25, batch 600  the loss is  1.4896 Accuracy 0.2344\n",
      " For epoch 25, batch 800  the loss is  1.5007 Accuracy 0.2331\n",
      " For epoch 25, batch 1000  the loss is  1.5117 Accuracy 0.2318\n",
      " For epoch 25, batch 1200  the loss is  1.5140 Accuracy 0.2308\n",
      "Epoch 25 Loss 1.5152 Accuracy 0.2307\n",
      " For epoch 26, batch 0  the loss is  1.4437 Accuracy 0.2476\n",
      " For epoch 26, batch 200  the loss is  1.4378 Accuracy 0.2414\n",
      " For epoch 26, batch 400  the loss is  1.4740 Accuracy 0.2373\n",
      " For epoch 26, batch 600  the loss is  1.4758 Accuracy 0.2366\n",
      " For epoch 26, batch 800  the loss is  1.4843 Accuracy 0.2355\n",
      " For epoch 26, batch 1000  the loss is  1.4894 Accuracy 0.2343\n",
      " For epoch 26, batch 1200  the loss is  1.4956 Accuracy 0.2336\n",
      "Saving checkpoint for epoch 26 at ./nmt/train/ckpt-13\n",
      "Epoch 26 Loss 1.4961 Accuracy 0.2335\n",
      " For epoch 27, batch 0  the loss is  1.4354 Accuracy 0.2690\n",
      " For epoch 27, batch 200  the loss is  1.4016 Accuracy 0.2462\n",
      " For epoch 27, batch 400  the loss is  1.4248 Accuracy 0.2442\n",
      " For epoch 27, batch 600  the loss is  1.4424 Accuracy 0.2421\n",
      " For epoch 27, batch 800  the loss is  1.4510 Accuracy 0.2395\n",
      " For epoch 27, batch 1000  the loss is  1.4610 Accuracy 0.2381\n",
      " For epoch 27, batch 1200  the loss is  1.4703 Accuracy 0.2371\n",
      "Epoch 27 Loss 1.4708 Accuracy 0.2368\n",
      " For epoch 28, batch 0  the loss is  1.2415 Accuracy 0.2349\n",
      " For epoch 28, batch 200  the loss is  1.3864 Accuracy 0.2445\n",
      " For epoch 28, batch 400  the loss is  1.4017 Accuracy 0.2432\n",
      " For epoch 28, batch 600  the loss is  1.4180 Accuracy 0.2423\n",
      " For epoch 28, batch 800  the loss is  1.4348 Accuracy 0.2403\n",
      " For epoch 28, batch 1000  the loss is  1.4443 Accuracy 0.2386\n",
      " For epoch 28, batch 1200  the loss is  1.4521 Accuracy 0.2371\n",
      "Saving checkpoint for epoch 28 at ./nmt/train/ckpt-14\n",
      "Epoch 28 Loss 1.4526 Accuracy 0.2370\n",
      " For epoch 29, batch 0  the loss is  1.1041 Accuracy 0.2263\n",
      " For epoch 29, batch 200  the loss is  1.3734 Accuracy 0.2492\n",
      " For epoch 29, batch 400  the loss is  1.3897 Accuracy 0.2469\n",
      " For epoch 29, batch 600  the loss is  1.4060 Accuracy 0.2451\n",
      " For epoch 29, batch 800  the loss is  1.4132 Accuracy 0.2438\n",
      " For epoch 29, batch 1000  the loss is  1.4214 Accuracy 0.2422\n",
      " For epoch 29, batch 1200  the loss is  1.4306 Accuracy 0.2409\n",
      "Epoch 29 Loss 1.4318 Accuracy 0.2407\n",
      " For epoch 30, batch 0  the loss is  1.2735 Accuracy 0.2894\n",
      " For epoch 30, batch 200  the loss is  1.3565 Accuracy 0.2509\n",
      " For epoch 30, batch 400  the loss is  1.3772 Accuracy 0.2494\n",
      " For epoch 30, batch 600  the loss is  1.3837 Accuracy 0.2477\n",
      " For epoch 30, batch 800  the loss is  1.3923 Accuracy 0.2458\n",
      " For epoch 30, batch 1000  the loss is  1.4041 Accuracy 0.2445\n",
      " For epoch 30, batch 1200  the loss is  1.4115 Accuracy 0.2430\n",
      "Saving checkpoint for epoch 30 at ./nmt/train/ckpt-15\n",
      "Epoch 30 Loss 1.4110 Accuracy 0.2428\n"
     ]
    }
   ],
   "source": [
    "epochs=30 #should be at-least 100 epochs for getting good results because of less learning rate\n",
    "\n",
    "for ii in range(epochs):\n",
    "    train_loss.reset_states();train_accuracy.reset_states()\n",
    "    \n",
    "    for (batch,(inp,target)) in enumerate(x_train):\n",
    "        \n",
    "        \n",
    "        if batch ==0 and ii==0:\n",
    "          with tf.device(\"/gpu:0\"):\n",
    "            train(inp,target,True)\n",
    "        else:\n",
    "          with tf.device(\"/gpu:0\"):\n",
    "            train(inp,target,False)\n",
    "        \n",
    "        if batch % 200 == 0:\n",
    "              print (' For epoch {}, batch {}  the loss is  {:.4f} Accuracy {:.4f}'.format(ii + 1, batch, train_loss.result(), train_accuracy.result()))\n",
    "    \n",
    "    if (ii + 1) % 2 == 0:\n",
    "        ckpt_save = ckpt_manage.save()\n",
    "        print ('Saving checkpoint for epoch {} at {}'.format(ii+1,\n",
    "                                                         ckpt_save))\n",
    "    \n",
    "    print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(ii + 1, \n",
    "                                                train_loss.result(), \n",
    "                                                train_accuracy.result()))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nICZEytO1BoM"
   },
   "outputs": [],
   "source": [
    "def get_output(encoder_input):\n",
    "    start_token =[pt_token.vocab_size]\n",
    "    end_token=[pt_token.vocab_size+1]\n",
    "    encoder_input=start_token+pt_token.encode(encoder_input)+end_token\n",
    "    encoder_input=tf.expand_dims(encoder_input,0) \n",
    "    decoder_input=tf.expand_dims([en_token.vocab_size],0)\n",
    "    \n",
    "    for _ in range(30): #30 is the max-length till which we trained the decoder and want the<end> token to be predicted in those 30 sequences\n",
    "        encoder_mask,decoder_look_ahead=create_mask(encoder_input,decoder_input)\n",
    "        output,attn_dict=transformer(encoder_input,decoder_input,encoder_mask,decoder_look_ahead,True)\n",
    "        output=tf.cast(tf.argmax(output[:,-1,:],axis=-1),tf.int32)\n",
    "        if output==en_token.vocab_size+1:\n",
    "            return tf.squeeze(decoder_input,axis=0),attn_dict\n",
    "        \n",
    "        \n",
    "        decoder_input=tf.concat([decoder_input,tf.expand_dims(output,0)],axis=-1)\n",
    "    return tf.squeeze(decoder_input,axis=0),attn_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "rpE-WYLNPF_p",
    "outputId": "796be173-09ac-4d3a-bf96-f28dc57eb749"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'how  about  it  ? '"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample='como vai você ?'\n",
    "eng,attn_dict_sample=get_output(sample)\n",
    "out=''\n",
    "for i in eng:\n",
    "    if i<en_token.vocab_size:\n",
    "        out+=en_token.decode([i])\n",
    "        out+=' '\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "PoFLsslYPMQI",
    "outputId": "74d67e28-a9bf-432d-a0f3-2d8b2dca048a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i  am  a  friday  . '"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample='eu sou do'\n",
    "eng,attn_dict_sample=get_output(sample)\n",
    "out=''\n",
    "for i in eng:\n",
    "    if i<en_token.vocab_size:\n",
    "        out+=en_token.decode([i])\n",
    "        out+=' '\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 365
    },
    "colab_type": "code",
    "id": "IJkOsQ4qPmJH",
    "outputId": "dce86e5c-757e-4e20-f322-2e7cdf088257"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABrgAAAFICAYAAAALY/4pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de7jld10f+vdnLpnc78PNQKJAkYsh\nSLCABpXyaKXVHguWauEI2NLaQ1ExrXhpPY9HkaqtrXoOPrFA7dGCN7BwqhU5R+4gBhISQGw5Eqgo\nJOSeCUlmJt/+sdfITppJ9p7Ze33Xd39fr+eZZ9b67Zn9+3zz2/m915r3+q1VrbUAAAAAAADAKHb1\nHgAAAAAAAAA2Q8EFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQ\nFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBtUm15rer6rG9ZwGA+yOzABiFzAJgFDILYHUo\nuDbvG5I8Jcnf7z0IADwAmQXAKGQWAKOQWQArQsG1ed+VtQD75qra03sYALgfMguAUcgsAEYhswBW\nhIJrE6rq3CSPb639bpK3JflfOo8EAPdJZgEwCpkFwChkFsBqUXBtzguSvH5x+3VxKTKww1XVt1bV\nqb3n4JjILGAqMmtoMguYjtwalswCprPKmaXg2pwXZy280lr7oyQPraqH9x0JYHtU1SOT/HqS5/ee\nhWMis4BpyKzhySxgKnJraDILmMqqZ5aCa4Oq6swkv9Ba+8y6zZcmObfTSADb7UVJ/mXWHsAzEJkF\nTEhmDUpmAZOSWwOSWcCkVjqzFFwb1Fq7KclH7rXt95Oc3GcigO1TVbuTfFvWAuzmqnpi55HYBJkF\nzERmjU1mAbORW+OSWcBsRsgsBdfm/PwGtwGM7tlJ3t9auzXJa5N8V+d52DyZBcxCZo1PZgEzkVtj\nk1nATFY+s/b0HmAEVfW0JE9Psr+qXr7uS6cn2d1nKoBt9V1J/vXi9puS/HhVXdpau6vjTGyAzAIm\nJLMGJbOAScmtAcksYFIrn1mu4NqYE5KcmrVC8LR1v25J8tyOcwFsucX7ip/ZWntnkrTW7kjym0me\n2XUwNkpmAdOQWcOTWcBU5NbQZBYwlVEyq1prvWcYwuL9Jn+9tfac3rP0UFXfkuQZi7vvaK29pec8\nABydzJJZAKOQWTILYBSzZ1Yit4DV4y0KN6i1driqHtZ7jh6q6ieTfFWSX11sellVPa219kMdxwK2\nQVV95f19vbX2oWXNwrGTWTILZiCzdgaZJbNgFnJrfDNnViK3YCYjZZYruDahql6d5EuS/EaSA0e2\nt9be2G2oJaiqq5Jc1Fq7e3F/d5IrWmsX9p0M2GpV9QeLmycmuTjJh5NUkguTXN5ae1qv2dgcmSWz\nYKeTWTuHzJJZMAO5tTPMmlmJ3IKZjJRZruDanBOTXJ97vs9kS7LjQyzJmUluWNw+o+cgwPZprX19\nklTVG5N8ZWvt6sX9JyT53zuOxubJrDUyC3YombWjyKw1Mgt2MLm1Y8ycWYncgimMlFkKrk1orb2o\n9wyd/GSSKxbNbWXtvXZf0XckYJs95kh4JUlr7SNV9dieA7E5MktmwURk1uBklsyCycitgU2cWYnc\nghmtfGZ5i8JNqKoTk3xXksdn7RUbSZLW2ou7DbUkVfXQJE9Z3P1Aa+2zPedhOarqryR5dZIHt9ae\nUFUXJvmW1tqPdx6NbVZVr8/a2y38ymLT30tyamvt2/tNxWbILJk1G5k1L5k1Ppkls2Yjs+Ymt8Y2\nc2YlcmtWcmteI2TWrt4DDOb/TvKQJN+Y5B1Jzktya9eJlmf/4vc9SZ5eVX+75zBsn6raV1U/vbj7\nS0l+MMnBJGmtXZXk7/aajaV6UZKPJvmexa+PLbYxDpkls3Y8mcWCzBqfzJJZO57MYh25NbaZMyuR\nW9OQWyysfGa5gmsTquqK1tqTquqq1tqFVbU3ybtaa0/tPdt2qqrXZu0D5D6a5O7F5jbLq1NmU1Wv\nSHJVa+13quqPWmtPOfKzv/j6la21izqPCTwAmSWzZiCzYGeQWTJrBjILdoZZMyuRW7ORW4zCZ3Bt\nzsHF7zctPlDts0ke1HGeZXlqa+1xvYdgaX4hyauS/E6Sz1fVI7P2gampqucm+YuOs7EkVfXVWfvQ\nyPOzLitaa1/WayY2TWYxA5mFzNoZZBYzkFkkkVs7wKyZlcit2cgthsgsBdfmXFZVZyX5kSRvTnJq\nkn/ed6SleF9VPa619rHeg7D9Wmu3JXnp4u7/luSyJF9eVZ9J8skkz+81G0v1miTfl+SDSQ53noVj\nI7PY8WQWCzJrfDKLHU9msY7cGtusmZXIranILRZWPrO8ReEmVNWXttY++UDbdpqq+tqshfZnk9yZ\npLJ2CfKFXQdjaarqlCS7Wmszva/01KrqD1trf7X3HBw7mSWzZiWz5iOzxiezZNasZNac5NbYZs2s\nRG4ht2Y0QmYpuDahqj7UWvvKe237YGvtyb1mWoaq+kSSlye5Ol98j9201j7VbSiWoqpemeSnWms3\nLe6fleT7W2s/0ncytltVvSrJ7iRvzNoD1yRJa+1D3YZiU2SWzJqNzJqXzBqfzJJZs5FZc5NbY5s1\nsxK5NTO5Na8RMstbFG5AVX15kscnOaOq/va6L52e5MQ+Uy3Vda21N/cegi6+qbX2Q0futNZurKpn\nZ+1SfHa2I6/OuHjdtpbkmR1mYRNklsyamMyal8walMySWROTWXOTWwOSWUnk1szk1rxWPrMUXBvz\nmCR/M8mZSb553fZbk/yDLhMt1xVV9R+TvCX3bGrf2G+k7VVV726tfU1V3ZrFByge+VLWLr8+vdNo\ny7a7qva11u5Mkqo6Kcm+zjOxBK21r+89A8dMZsmsv/xSZJbMmoDMGprMmiyzErm1ILMmJreGNXtm\nJRPmlsz6S3JrUiNklrco3ISqelpr7X2951i2qnrdfWxurbUXL30YlqqqfiBrD9yO/Ay8KMmbW2s/\n1W8qlqGqHpzklUke1lr7pqp6XJKntdZe03k0Nkhm3YPMmoDMmpfMGp/MugeZNQGZNTe5NbZZMyuR\nWzOTW/MaIbMUXJtQVT+V5MeTfCHJf0lyYZLva639StfB2HZV9aCsu+S8tfbpjuMsVVX99STPWtz9\n/dba7/Wch+Woqt/N2gOXH26tPbGq9iS5orX2FZ1HY4Nk1rxklsyajcwan8ya26y5JbPmJbfGJrPm\nNmtmJXJrViNk1q7eAwzmG1prt2TtkuRrkjwqyT/tOtESVNV5VfWmqrp28eu3quq83nMtQ1V9S1X9\ntySfTPKOrB333+061PJdkbW1v31xmzmc21r79Sw+OLa1dijJ4b4jsUkyS2ZdE5nFHGTW+GTWZJmV\nyK3IrJnJrbFNmVnJ3Lkls5LIrVmtfGYpuDZn7+L3v5HkN1prN/ccZolel+TNSR62+PWWfPGS1J3u\n/0jy1CT/tbX2pUn+WpL39x1pearq7yT5QJLnJvk7Sf6wqp7bdyqW5EBVnZPFe0xX1VOTzHLO2ylk\nlsySWTJrFjJrfDJrvsxKJs4tmTU9uTW2WTMrmTu3ps2sRG5NbuUza0/vAQbzlqr6eNYuQ/7uqtqf\n5I7OMy3D/tba+sD691X1vd2mWa6DrbXrq2pXVe1qrf1BVf2b3kMt0Q8neUpr7dokWfzMvy3Jb3ad\nimV4edYeuD6yqt6TZH/WHsgwDpm1RmbNQ2bNS2aNT2atmSmzkrlzS2bNTW6NbdbMSubOrZkzK5Fb\nM1v5zFJwbUJr7RWL99q9ubV2uKpuT/K3es+1BNdX1fOTvH5x/9uTXN9xnmW6qapOTfLOJL9aVdcm\nOdB5pmXadSS8Fq7PDr/ys6q+LMkPZu3B6s/M9H7K67XWPlRVX5vkMUkqyZ+01g52HotNkFkyS2bJ\nrFnIrPHJrCkzK5k7t2TWpJmVyK3RTZxZydy5NXNmJXJr2twaIbOqtdZ7hiFU1clJHt1a+/C6bY9I\ncri19pl+k22/qjo/yc8neVrWLkd8b5KXzfA/dlWdkrUT2a4kfy/JGUl+tbU2RYAvHrQ9MV988PK8\nJFe11n6g31Tbq6o+kOTfJdmX5HuSfGdr7T19p1qumc93O8XMx1BmyazILJk1yflup5j5GM6cWcnc\nuSWz5sysZO5z3k4w+/GbObdmzqxEbmXS3BrlnKfg2qCq2pvk40kubK0dWGx7a5Ifaq1d3nU4tk1V\nvTzJr63S/7TLVFWXJvlckosWm97dWntTx5G2XVVd1Vq7cHH7oiSvSfLIJC9O8vLW2tf0nG8ZnO/G\n5xjOSWbJrMgs57sBOYbzmjm3ZNacmZU4543O8ZvXzJmVyK1Zc2uUc96OvpRwKy0uvXtT1j5I70hb\nuX+VDuZ2qapfrqoz190/q6pe23OmJTotyVur6l1V9dKqenDvgZbslCSvSPJVST6ZtVfn7HSfq6oL\nk6S1dmVr7cmttTNba2+cIbySuc93O8XMx1BmyazILJk1yflup5j5GE6eWcncuSWzJsysZO5z3k4w\n+/GbPLdmzqxEbk2ZW6Oc81zBtQlV9eVJLmutPaOqfiTJLa21n+s913arqitaa096oG072eKE9rwk\nz0nyZ621Z3UeaalmWn+tfVDmntbaX/SepadZz3c7yazHUGbNdc6+LzOtX2atmfV8t5PMegxl1pqZ\nztv3NtPaZdYXzXrO2ylmPn5ya67z9n2Zaf1ya80I57w9vQcYSWvt47XmryT5u0ku6T3TkuyqqrNa\nazcmSVWdnfl+dq5N8tmsfYjigzrP0sM062+tXdd7hlUw8flux5j4GMqsic7ZRzHN+mXWmonPdzvG\nxMdQZq2Z5rx9H6ZZu8z6oonPeTvC5MdPbk103j6KadYvt9aMcM6b7SS0FV6TtQ+Yu/rICX0C/yrJ\n+6rqNxb3vy3JT3ScZ2mq6h9n7TLM/Ul+I8k/aK19rO9UyzP7+pnyfLfTzHgMZdak5+zZ18+U57ud\nZsZjOG1mJXOft2deO39pxnPeTjLr8Zs2t2Y/b8++flb7nOctCjepqk5O8hdJntNae1vveZalqh6X\n5JmLu//fLCexqvrJrH2I5JW9Z+lh9vXPbtbz3U4y6zGUWXOes2df/+xmPd/tJLMew1kzK5n7vD3z\n2lkz6zlvp5j5+M2aW7Oft2df/+xW/Zyn4AIAAAAAAGAou3oPAAAAAAAAAJuh4AIAAAAAAGAoCq5j\nVFUv6T1DL9Y+p5nXnsy9/pnXvlPMfAytfV4zr9/aGdnMx3DmtSdzr9/a5zX7+kc3+/Gbef3WPq+Z\n17+qa1dwHbuVPKBLYu1zmnntydzrn3ntO8XMx9Da5zXz+q2dkc18DGdeezL3+q19XrOvf3SzH7+Z\n12/t85p5/Su5dgUXAAAAAAAAQ6nWWu8Zjsnu005pe845q9v+D996ILtPO6XLvvfcVl32e8ShOw5k\nz4l91p4kh07r9zPb87gnye4D/Y79oS8cyJ6T+q191+Fuu06SHLzjQPZ2/LnfdeOBbvs+2O7M3trX\nbf+3ths/31rb322ALbDnjJPbvgef0W3/h26+PXvOOLnLvg8f7vtamsO3HMju0/v9v9sO9lv/4dsO\nZPep/dZ+4nWHuu07Se46fHtO2N3n5/7ufbu77PeIg3cdyN4TOv7c7+73eKV3Xh+44c+Gz6wk2bvv\nlLbvlLO77PvgnQeyd1/H5xondtt1Dt9+ILtP7rf2JGknzPtc64Sbuu26+3n7rn4PU7s/XkmSPT2f\nZ3f895U7b7shh+7ouPgtcvbZu9qXnNfnsdcNN9yds8/u93j/k3/2kG77Tvpn9t17uu26+7+RVcd/\nzu+99rv7PtXK4S8cyO6O6z//Qdd22/dN19+dM8/pd877k6vvus/nWh1PBcdnzzln5SE//LLeY3Sx\n/73DHrYtcd3XHuw9QjdnfWBv7xG6OfGmMcv4rXL6b17ee4Rufv/gGz7Ve4bjte/BZ+TxP/fC3mN0\ncdNtJ/UeoauDn+n7DzY9Peay63uP0M2BR/V7EdYqOHjKvG8S8Yevv3T4zEqSfaecna/4hu/tPUYX\nNzx23p/fJLnjvHmfaz3izcP/O/8x+/Tf6D1BX/vf1/lfSzv547f8bO8RtsSXnLc7b/zP5/Yeo4sX\n/LM5s/qIO8+c97y9a964zh3nzHvck+QXv/sXeo/QzTO+9E/v87nW3I/eAQAAAAAAGI6CCwAAAAAA\ngKEouAAAAAAAABiKggsAAAAAAIChKLgAAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAA\ngKEouAAAAAAAABiKggsAAAAAAIChKLgAAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAA\ngKEouAAAAAAAABiKggsAAAAAAIChKLgAAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAA\ngKEouAAAAAAAABiKggsAAAAAAIChKLgAAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAA\ngKEcV8FVVRdU1Ue2ahgA2E5yC4BRyCwARiGzAOjFFVwAAAAAAAAMZSsKrt1V9UtV9dGqemtVnZQk\nVXVRVb2/qq6qqjdV1VlV9aCq+uDi60+sqlZVj1jc//+r6uQtmAcA7o/cAmAUMguAUcgsAJZuKwqu\nRyf5P1trj09yU5LnLLb/hyQ/0Fq7MMnVSX60tXZtkhOr6vQklyS5PMklVXV+kmtba7ff346q6iVV\ndXlVXX741gNbMDoAE1pKbq3PrEM332+8AcDRdHmudfBOz7UA2LQumXXDDXdv13oAGMBWFFyfbK1d\nubj9wSQXVNUZSc5srb1jsf2Xkzxjcfu9Sb56cf+Vi98vSfKuB9pRa+2y1trFrbWLd592yhaMDsCE\nlpJb6zNrzxlegAjAMenyXGvvPs+1ANi0Lpl19tk+fQVgZluRAneuu304yZ4H+PPvzFpgnZ/kPyV5\nYpKvyQYCDAC2gNwCYBQyC4BRyCwAlm5bXubQWrs5yY1Vdcli0wuSHHm1xruSPD/Jf2ut3Z3khiTP\nTvLu7ZgFAB6I3AJgFDILgFHILAC22wO9muJ4fGeSX1x8MOSfJnlRkrTWrqmqytorNZK14DqvtXbj\nNs4CAA9EbgEwCpkFwChkFgDb5rgKrtbaNUmesO7+z6y7fWWSpx7l7z183e1XZu29dgFgW8ktAEYh\nswAYhcwCoBefxAgAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAU\nXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAU\nXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAU\nXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAU\nXAAAAAAAAAxFwQUAAAAAAMBQ9vQe4FjtPeFQHn7B53uP0cUJbzir9whd3fLfT+w9QjcP/X8+1XuE\nbj72Yw/tPUJXZ73nIb1H6OfTvQc4fifsOpzzTrup9xhdnH3p3K+lueErT+k9Qjc3XXhO7xG6efYP\nv733CF294588rfcIHKfDJyS3PGLO8/cFv3VD7xG6uvar532uecv51XuEbs7/sj/vPUJXn712zuea\nh/f1nmBr7KvdeeTeU3uP0cVZ7/nvvUfo6lPPP7/3CN289H/9T71H6OZt1z+29whd/cQ3f3vvETr6\nifvcOuezFgAAAAAAAIal4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABgKAouAAAAAAAAhqLgAgAA\nAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABgKAouAAAAAAAAhqLgAgAA\nAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABgKAouAAAAAAAAhqLgAgAA\nAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABgKAouAAAAAAAAhqLgAgAA\nAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABgKMdccFXVbVsxQFVdVFXP\n3orvBQBHI7cAGIXMAmAUMguAnlbhCq6LkggwAEYhtwAYhcwCYBQyC4BNe8CCq6p+u6o+WFUfraqX\n3OtrP7vY/v9W1f7Ftouq6v1VdVVVvamqzlpsf3tVXby4fW5VXVNVJyT5sSTPq6orq+p5W79EAGYi\ntwAYhcwCYBQyC4BVtJEruF7cWntykouTvKyqzllsPyXJ5a21xyd5R5IfXWz/D0l+oLV2YZKr123/\nn7TW7kryL5L8Wmvtotbarx3jOgDgCLkFwChkFgCjkFkArJyNFFwvq6oPJ3l/kocnefRi+91JjgTO\nryT5mqo6I8mZrbV3LLb/cpJnbNWwVfWSqrq8qi4/dPMXturbArCzrERurc+sO2+SWQDcp5XIrOSe\nuXX4Cwe26tsCsHOsZGZdd/3hrfq2AAzofguuqvq6JM9K8rTW2hOTXJHkxKP88fYA+zq0bn9H+x73\nq7V2WWvt4tbaxXvOOOlYvgUAO9gq5db6zNp3pswC4J5WKbOSe+bW7pNOOZZvAcAOtcqZtf+c3cfy\nLQDYIR7oCq4zktzYWru9qr48yVPv9Xefu7j9HUne3Vq7OcmNVXXJYvsLsnZ5cpJck+TJi9tH/l6S\n3JrktGMbHwDuQW4BMAqZBcAoZBYAK+mBCq7/kmRPVf1xkldl7TLkIw4k+aqq+kiSZ2btwyCT5DuT\n/HRVXZXkonXbfybJd1fVFUnOXfd9/iDJ43yIJABbQG4BMAqZBcAoZBYAK2nP/X2xtXZnkm86ytdO\nPcr2K3PPV3Ic2f7xJBeu2/Qji+03JHnKBucFgKOSWwCMQmYBMAqZBcCqeqAruAAAAAAAAGClKLgA\nAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAAgKEouAAAAAAAABiKggsAAAAAAIChKLgA\nAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAAgKEouAAAAAAAABiKggsAAAAAAIChKLgA\nAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAAgKEouAAAAAAAABiKggsAAAAAAIChKLgA\nAAAAAAAYioILAAAAAACAoSi4AAAAAAAAGIqCCwAAAAAAgKEouAAAAAAAABiKggsAAAAAAIChKLgA\nAAAAAAAYyp7eAxyrQ7fvzWc/9JDeY3TxqD+/rvcIXe297cTeI3Rz8Pz9vUfo5qxzb+09Qle3XPwl\nvUfo59O9Bzh+t39hXz704Uf2HqOLdunh3iN0dfpV876W6KHvmfe8/YZPPLn3CF097K67e4/Acdp1\nKDn5c633GF3U567vPUJf7azeE3Rz8ufmfczy9if8du8Rurrw9/5x7xG6qB3yI3/L3ZW33r639xhd\n3HDJw3uP0NWBR9/Ve4Ru/u1Hn9l7hG7uvPbk3iN0tf/J8/4bQz5y35sn/i8CAAAAAADAiBRcAAAA\nAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAA\nAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAA\nAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAA\nAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADAUBRcAAAA\nAAAADEXBBQAAAAAAwFBWouCqqvcufr+gqr6j9zwAcH/kFgCjkFkAjEJmAbBZK1Fwtdaevrh5QRIB\nBsBKk1sAjEJmATAKmQXAZq1EwVVVty1uvirJJVV1ZVV9X8+ZAOBo5BYAo5BZAIxCZgGwWXt6D3Av\nr0hyaWvtb/YeBAA2QG4BMAqZBcAoZBYAG7ISV3BtVFW9pKour6rLDx840HscADiqe2TWbbc98F8A\ngI7W59ahOzzXAmB1rc+sm2841HscADoaquBqrV3WWru4tXbx7lNO6T0OABzVPTLr1FN7jwMA92t9\nbu050XMtAFbX+sw64+xVe3MqAJZp1QquW5Oc1nsIANgguQXAKGQWAKOQWQBsyKoVXFclOVxVH/Yh\nkgAMQG4BMAqZBcAoZBYAG7IS1/G21k5d/H4wyTM7jwMA90tuATAKmQXAKGQWAJu1aldwAQAAAAAA\nwP1ScAEAAAAAADAUBRcAAAAAAABDUXABAAAAAAAwFAUXAAAAAAAAQ1FwAQAAAAAAMBQFFwAAAAAA\nAENRcAEAAAAAADAUBRcAAAAAAABDUXABAAAAAAAwFAUXAAAAAAAAQ1FwAQAAAAAAMBQFFwAAAAAA\nAENRcAEAAAAAADAUBRcAAAAAAABDUXABAAAAAAAwFAUXAAAAAAAAQ1FwAQAAAAAAMBQFFwAAAAAA\nAENRcAEAAAAAADAUBRcAAAAAAABDUXABAAAAAAAwFAUXAAAAAAAAQ1FwAQAAAAAAMBQFFwAAAAAA\nAENRcAEAAAAAADCUPb0HOFZPOOe6fOA7X917jC6+8Qcv6j1CV2d+4pzeI3Sz5+Of7j1CN7fe9oje\nI3T1oGtu6z0Cx2HvvoM579HX9h6ji5O+8ZO9R+jq7q99Uu8Ruvmvzz+l9wjd/OlTf7H3CF1900cu\n6T0Cx2nXoZaTbjjUe4wuDl93Xe8Rujr3qof2HqGbeuX1vUfo5t/ceEHvEbo6fELvCTqp3gNsjZN2\nHc5F+27qPUYXp7/+/b1H6OqWC57ee4Ru9n/953uP0M0Lv+L3eo/Q1S+9/Vt7j7ByXMEFAAAAAADA\nUBRcAAAAAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAAAAAMRcEFAAAAAADA\nUBRcAAAAAAAADEXBBQAAAGfJgecAAAejSURBVAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAA\nAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAA\nAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFAUXAAAAAAAAAxFwQUAAAAAAMBQFFwAAAAA\nAAAMRcEFAAAAAADAUBRcAAAAAAAADEXBBQAAAAAAwFBWpuCqqgdV1duq6uqquryqHtV7JgC4LzIL\ngFHILABGIbMA2KyVKbiS7ElyaWvtK5L8UpJXdJ4HAI5GZgEwCpkFwChkFgCbsqf3AEe01v48yZ8v\n7u5LckfHcQDgqGQWAKOQWQCMQmYBsFkrU3AdUVUXJfneJM/sPQsA3B+ZBcAoZBYAo5BZAGzUKr1F\n4RGvTfLC1to19/5CVb1k8R68l193/eHlTwYA97ShzDp08xeWPxkA3NNRMyu5Z24dvOvAcicDgHva\ncGZdf/3dy50MgJWyigXXo1pr77yvL7TWLmutXdxau3j/ObuXPRcA3NuGMmvPGSctey4AuLejZlZy\nz9zae8Ipy5wLAO5tw5l1zjmr+E+bACzLKqbAi3oPAAAbJLMAGIXMAmAUMguADVnFguv7ew8AABsk\nswAYhcwCYBQyC4ANWbmCq7X29N4zAMBGyCwARiGzABiFzAJgo1au4AIAAAAAAID7o+ACAAAAAABg\nKAouAAAAAAAAhqLgAgAAAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABg\nKAouAAAAAAAAhqLgAgAAAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABg\nKAouAAAAAAAAhqLgAgAAAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABg\nKAouAAAAAAAAhqLgAgAAAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACGouACAAAAAABg\nKAouAAAAAAAAhlKttd4zHJOqui7JpzqOcG6Sz3fcf0/WPqeZ157Mvf7eaz+/tba/4/6Pm8zqytrn\nNfP6rb2f4TMr6Z5bvY9hTzOvPZl7/dY+r57rl1nHz8/vvOu39nnNvP7ea7/P3Bq24Oqtqi5vrV3c\ne44erN3aZzTz+mde+04x8zG09jnXnsy9fmufc+07xczHcOa1J3Ov39rnXHti/aOb/fjNvH5rn3Pt\nydzrX9W1e4tCAAAAAAAAhqLgAgAAAAAAYCgKrmN3We8BOrL2Y1BVt93r/gur6heOf6Skqt5eVf/T\nJaJV9dKq+kRVtao69zh3M/NxT+Ze/8xr3ylmPobWfgw6ZdavVtWfVNVHquq1VbX3OHfl2M9p5rXv\nFDMfw5nXnoyXW6+pqg9X1VVV9ZtVdepx7GbmYz/z2hPrH93sx2/m9Q+VWeu+/nP33v8xmPm4J3Ov\nfyXX7jO4YEmq6rbW2qnr7r8wycWttZduwfd+e5JLW2uX32v7k5LcmOTti33N+iGIAGxCp8x6dpLf\nXdz9j0ne2Vp79fHuD4Cdr1Nund5au2Vx+18nuba19qrj3R8AO1uPzFp87eIk35PkW9fvH0bnCi5Y\nAVW1v6p+q6r+aPHrqxfbv6qq3ldVV1TVe6vqMYvtJ1XVG6rqj6vqTUlOuq/v21q7orV2zfJWAsBO\nt42Z9TttIckHkpy3tEUBsGNtY24dKbdq8We8ehiA47JdmVVVu5P8dJJ/trTFwJLs6T0ATOSkqrpy\n3f2zk7x5cfvfJvnZ1tq7q+oRSX4vyWOTfDzJJa21Q1X1rCSvTPKcJN+d5PbW2mOr6sIkH1raKgCY\nQbfMWrw14Quy9upCANiILrlVVa9L8uwkH0vy/Vu9KAB2pB6Z9dIkb26t/cXa6zJg51BwwfJ8obV2\n0ZE7Ry5BXtx9VpLHrQuZ0xfv4X5Gkl+uqkdn7RWBRz6P5BlJfi5JWmtXVdVV2z8+ABPpmVn/V9be\nnvBdW7EQAKbQJbdaay9avCr+55M8L8nrtmxFAOxUS82sqnpYkm9L8nVbvhJYAQouWA27kjy1tXbH\n+o2LD5n8g9bat1bVBVn7LC0A6GnbMquqfjTJ/iT/8PjHBIAk2/xcq7V2uKrekLW3fVJwAXA8tiOz\nnpTkUUk+sSjOTq6qT7TWHrUlE0NnPoMLVsNbk/yTI3eq6sgrOc5I8pnF7Reu+/PvTPIdiz/7hCQX\nbv+IAJBkmzKrqv5+km9M8u2ttbu3dmQAJrbluVVrHnXkdpJvydrbRwHA8djyzGqt/efW2kNaaxe0\n1i7I2lsaKrfYMRRcsBpeluTiqrqqqj6W5B8ttv9Ukp+sqityzysuX53k1Kr64yQ/luSD9/VNq+pl\nVfVnSc5LclVV/bttWwEAs9iWzEryi0kenOR9VXVlVf2L7RkfgMlsR25V1t4q6uokVyd56OLPAsDx\n2K7nWrBjVWut9wwAAAAAAACwYa7gAgAAAAAAYCgKLgAAAAAAAIai4AIAAAAAAGAoCi4AAAAAAACG\nouACAAAAAABgKAouAAAAAAAAhqLgAgAAAAAAYCgKLgAAAAAAAIbyPwAFgOu33qUDSAAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<Figure size 1728x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JMM-78nUQeN4"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Untitled22.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}