Improved-InferSent.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "InferSent.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u67iWj5E2G14",
        "colab_type": "text"
      },
      "source": [
        "# Taha Samavati - 98722134\n",
        "## NLP Final Project / Fall-Winter 1398 / Dr.Minaei\n",
        "\n",
        "*Supervised Learning of Sentence Representation based on stanford natural language inference data (InferSent)*\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wxNRhguhDSx7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Imports\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import gensim\n",
        "import nltk\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from keras.layers import LSTM, GRU, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D, \\\n",
        "      concatenate, Conv1D, Lambda, multiply, Dense,subtract,TimeDistributed\n",
        "import keras.backend as K\n",
        "from keras import Model\n",
        "import os\n",
        "import math\n",
        "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "from keras.layers import Input, Embedding, concatenate\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "from keras import backend as K, initializers, regularizers, constraints\n",
        "from keras.engine.topology import Layer\n",
        "from keras.utils import plot_model\n",
        "from keras.models import load_model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xviqaWT4EBMG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8gt-Hvr53Tjy",
        "colab_type": "text"
      },
      "source": [
        "# 1. Load and Preprocessing of data set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0COxMmMSJ7k7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# download dataset and unzip it\n",
        "!wget https://nlp.stanford.edu/projects/snli/snli_1.0.zip -P /content/gdrive/My\\ Drive/NLP/\n",
        "!wget https://www.nyu.edu/projects/bowman/multinli/multinli_1.0.zip -P /content/gdrive/My\\ Drive/NLP/\n",
        "\n",
        "!unzip /content/gdrive/My\\ Drive/NLP/snli_1.0.zip -d /content/gdrive/My\\ Drive/NLP/\n",
        "!unzip /content/gdrive/My\\ Drive/NLP/multinli_1.0.zip -d /content/gdrive/My\\ Drive/NLP/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FDjR68kvb2Nl",
        "colab_type": "code",
        "outputId": "b134697c-45c2-445e-956e-ed79d895d846",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "!wget https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip -P /content/gdrive/My\\ Drive/NLP/\n",
        "!unzip /content/gdrive/My\\ Drive/NLP/wiki-news-300d-1M.vec.zip -d /content/gdrive/My\\ Drive/NLP/\n",
        "\n",
        "! wget \"http://nlp.stanford.edu/data/glove.6B.zip\" -O glove.6B.zip && unzip glove.6B.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/gdrive/My Drive/NLP/wiki-news-300d-1M.vec.zip\n",
            "replace /content/gdrive/My Drive/NLP/wiki-news-300d-1M.vec? [y]es, [n]o, [A]ll, [N]one, [r]ename: y\n",
            "  inflating: /content/gdrive/My Drive/NLP/wiki-news-300d-1M.vec  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SAntSet3OLJR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def onehot(labels):\n",
        "\n",
        "  # integer encode\n",
        "  label_encoder = LabelEncoder()\n",
        "  integer_encoded = label_encoder.fit_transform(labels)\n",
        "\n",
        "  # binary encode\n",
        "  onehot_encoder = OneHotEncoder(sparse=False)\n",
        "  integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
        "  onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n",
        "  return onehot_encoded\n",
        "\n",
        "# read dataset\n",
        "snli_train = pd.read_csv('/content/gdrive/My Drive/NLP/snli_1.0/snli_1.0_train.txt',sep=\"\\t\")\n",
        "snli_dev = pd.read_csv('/content/gdrive/My Drive/NLP/snli_1.0/snli_1.0_dev.txt', sep=\"\\t\")\n",
        "snli_test = pd.read_csv('/content/gdrive/My Drive/NLP/snli_1.0/snli_1.0_test.txt', sep=\"\\t\")\n",
        "\n",
        "# preprocess drop nan entries and ignore entries with no label\n",
        "snli_train = snli_train.dropna(subset = ['sentence2'])\n",
        "snli_train = snli_train[snli_train[\"gold_label\"] != \"-\"]\n",
        "snli_dev = snli_dev[snli_dev[\"gold_label\"] != \"-\"]\n",
        "snli_test = snli_test[snli_test[\"gold_label\"] != \"-\"]\n",
        "\n",
        "# construct label arrays\n",
        "y_train = snli_train['gold_label']\n",
        "y_val = snli_dev['gold_label']\n",
        "y_test = snli_test['gold_label']\n",
        "\n",
        "# constrauct X_train arrays\n",
        "x_train_raw = snli_train[['sentence1','sentence2']]\n",
        "x_val_raw = snli_dev[['sentence1','sentence2']]\n",
        "x_test_raw = snli_test[['sentence1','sentence2']]\n",
        "\n",
        "# one hot encode labels\n",
        "y_train_e = onehot(y_train)\n",
        "y_val_e = onehot(y_val)\n",
        "y_test_e = onehot(y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DntWL_wV7A3w",
        "colab_type": "text"
      },
      "source": [
        "* FastText was used as The Word Embedding Model. (300 dim - wiki-news)\n",
        "* in the preceeding cells :\n",
        "\n",
        "\n",
        "1.   Load and read fasttext model\n",
        "2.   Create corpus out of all sentences in dataset\n",
        "3.   Fit Tokenizer on whole corpus\n",
        "4.   Convert text to sequence\n",
        "5.   Zero padd sequences\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQ8sWthGatTm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 1.Load and read fasttext model\n",
        "\n",
        "embeddings_index = {}\n",
        "with open('/content/gdrive/My Drive/NLP/wiki-news-300d-1M.vec') as f:\n",
        "    for line in f:\n",
        "        word, coefs = line.split(maxsplit=1)\n",
        "        coefs = np.fromstring(coefs, 'f', sep=' ')\n",
        "        embeddings_index[word] = coefs\n",
        "'''\n",
        "# 1.Load and read glove model\n",
        "embeddings_index = {}\n",
        "with open('glove.6B.300d.txt') as f:\n",
        "    for line in f:\n",
        "        word, coefs = line.split(maxsplit=1)\n",
        "        coefs = np.fromstring(coefs, 'f', sep=' ')\n",
        "        embeddings_index[word] = coefs\n",
        "'''"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6HwLZjExTO0e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 2.Create corpus out of all sentences in dataset\n",
        "main_train, hypo_train, main_val, hypo_val, main_test, hypo_test=[], [], [], [], [], []\n",
        "\n",
        "for entry in x_train_raw['sentence1']:\n",
        "  main_train.append(entry)\n",
        "for entry in x_train_raw['sentence2']:\n",
        "  hypo_train.append(entry)\n",
        "for entry in x_val_raw['sentence1']:\n",
        "  main_val.append(entry)\n",
        "for entry in x_val_raw['sentence2']:\n",
        "  hypo_val.append(entry)\n",
        "for entry in x_test_raw['sentence1']:\n",
        "  main_test.append(entry)\n",
        "for entry in x_test_raw['sentence2']:\n",
        "  hypo_test.append(entry)\n",
        "\n",
        "corpus = main_train + hypo_train + main_val + hypo_val\n",
        "# 3.Fit Tokenizer on whole corpus\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(corpus)\n",
        "del corpus # delete to free memmory"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V9xc__ZVkMXs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 4.Convert text to sequence\n",
        "main_sequences = tokenizer.texts_to_sequences(main_train)\n",
        "hypo_sequences = tokenizer.texts_to_sequences(hypo_train)\n",
        "main_sequences_val = tokenizer.texts_to_sequences(main_val)\n",
        "hypo_sequences_val = tokenizer.texts_to_sequences(hypo_val)\n",
        "main_sequences_test = tokenizer.texts_to_sequences(main_test)\n",
        "hypo_sequences_test = tokenizer.texts_to_sequences(hypo_test)\n",
        "# 5.Zero pad sequences\n",
        "main_seq_padded = pad_sequences(main_sequences, maxlen=82)\n",
        "hypo_seq_padded = pad_sequences(hypo_sequences, maxlen=82)\n",
        "main_seq_padded_val = pad_sequences(main_sequences_val, maxlen=82)\n",
        "hypo_seq_padded_val = pad_sequences(hypo_sequences_val, maxlen=82)\n",
        "main_seq_padded_test = pad_sequences(main_sequences_test, maxlen=82)\n",
        "hypo_seq_padded_test = pad_sequences(hypo_sequences_test, maxlen=82)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gjgz82ovA9fg",
        "colab_type": "text"
      },
      "source": [
        "* Construct the embedding matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O-koTqShlnCu",
        "colab_type": "code",
        "outputId": "1dd14d13-de4b-48af-b977-ef41ffd0f236",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "EMBEDDING_DIM=300\n",
        "word_index = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(word_index))\n",
        "\n",
        "# reserve zero for padding\n",
        "num_words = len(word_index) + 1\n",
        "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
        "\n",
        "for word, i in word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None :\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix[i] = embedding_vector"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 34606 unique tokens.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9fkAyYNM9V1P",
        "colab_type": "text"
      },
      "source": [
        "# 2. Original Model implementation (discussed in paper)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofBV2Lq1LbWs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sentence_encoder(premise, hypothesis, encoder_type):\n",
        "\trnn_units=512\n",
        "\t# Construct encoder part of the model with specified structure #\n",
        "\tif encoder_type == 'lstm':\n",
        "\t\tlstm = LSTM(units=rnn_units)\n",
        "\t\treturn lstm(premise), lstm(hypothesis)\n",
        "\telif encoder_type == 'gru':\n",
        "\t\tgru = GRU(units=rnn_units)\n",
        "\t\treturn gru(premise), gru(hypothesis)\n",
        "\telif encoder_type == 'bilstm':\n",
        "\t\tbilstm = Bidirectional(LSTM(units=rnn_units))\n",
        "\t\treturn bilstm(premise), bilstm(hypothesis)\n",
        "\telif encoder_type == 'bigru':\n",
        "\t\tbigru = Bidirectional(GRU(units=rnn_units))\n",
        "\t\treturn bigru(premise), bigru(hypothesis)\n",
        "\telif encoder_type == 'bilstm_max_pool':\n",
        "\t\tbilstm = Bidirectional(LSTM(units=rnn_units, return_sequences=True))\n",
        "\t\tglobal_max_pooling = Lambda(lambda x: K.max(x, axis=1)) # GlobalMaxPooling1D didn't support masking\n",
        "\t\treturn global_max_pooling(bilstm(premise)), global_max_pooling(bilstm(hypothesis))\n",
        "\telif encoder_type == 'bilstm_mean_pool':\n",
        "\t\tbilstm = Bidirectional(LSTM(units=rnn_units, return_sequences=True))\n",
        "\t\treturn GlobalAveragePooling1D()(bilstm(premise)), GlobalAveragePooling1D()(bilstm(hypothesis))\n",
        "\telse:\n",
        "\t\traise ValueError('Encoder Type Not Understood : {}'.format(encoder_type))\n",
        "\n",
        "def build_input(mask_zero=True): # Build Input of the model--------------------------\n",
        "\n",
        "  input_premise = Input(shape=(max_len,))\n",
        "  input_hypothesis = Input(shape=(max_len,))\n",
        "  inputs = [input_premise, input_hypothesis]\n",
        "    \n",
        "  embedding = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n",
        "                        weights=[embedding_matrix], trainable=word_embed_trainable,\n",
        "                        mask_zero=mask_zero)\n",
        "    \n",
        "  premise_embed = embedding(input_premise)\n",
        "  hypothesis_embed = embedding(input_hypothesis)\n",
        "\n",
        "  return premise_embed, hypothesis_embed , inputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "niVmeZt5J75G",
        "colab_type": "code",
        "outputId": "b98e9259-fa31-4b1d-b40e-ca2990536e35",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 675
        }
      },
      "source": [
        "max_len = 82\n",
        "word_embed_type = 'fasttext'\n",
        "#word_embed_type = 'GloVe'\n",
        "word_embed_dim = 300\n",
        "word_embed_trainable = False\n",
        "mask_zero =  True\n",
        "\n",
        "premise_embed, hypothesis_embed, inputs = build_input(mask_zero=mask_zero)\n",
        "\n",
        "premise_encoded, hypothesis_encoded = sentence_encoder(premise_embed, hypothesis_embed, 'bilstm_max_pool')\n",
        "p_sub_h = Lambda(lambda x: K.abs(x[0] - x[1]))([premise_encoded, hypothesis_encoded])\n",
        "p_mul_h = multiply([premise_encoded, hypothesis_encoded])\n",
        "p_concat_h = concatenate([premise_encoded, hypothesis_encoded, p_sub_h, p_mul_h])\n",
        "\n",
        "dense = Dense(units=128, activation='relu')(p_concat_h)\n",
        "output = Dense(3, activation='softmax')(dense)\n",
        "\n",
        "model = Model(inputs, output)\n",
        "model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='Adam')\n",
        "\n",
        "print(model.summary())"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_3 (InputLayer)            (None, 82)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_4 (InputLayer)            (None, 82)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_2 (Embedding)         (None, 82, 300)      664500      input_3[0][0]                    \n",
            "                                                                 input_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_2 (Bidirectional) (None, 82, 1024)     3330048     embedding_2[0][0]                \n",
            "                                                                 embedding_2[1][0]                \n",
            "__________________________________________________________________________________________________\n",
            "lambda_3 (Lambda)               (None, 1024)         0           bidirectional_2[0][0]            \n",
            "                                                                 bidirectional_2[1][0]            \n",
            "__________________________________________________________________________________________________\n",
            "lambda_4 (Lambda)               (None, 1024)         0           lambda_3[0][0]                   \n",
            "                                                                 lambda_3[1][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "multiply_2 (Multiply)           (None, 1024)         0           lambda_3[0][0]                   \n",
            "                                                                 lambda_3[1][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)     (None, 4096)         0           lambda_3[0][0]                   \n",
            "                                                                 lambda_3[1][0]                   \n",
            "                                                                 lambda_4[0][0]                   \n",
            "                                                                 multiply_2[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 128)          524416      concatenate_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 3)            387         dense_3[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 4,519,351\n",
            "Trainable params: 3,854,851\n",
            "Non-trainable params: 664,500\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZVkIOeKL93_O",
        "colab_type": "text"
      },
      "source": [
        "# 3. Modified Model (Esim) -- Implementation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UzT_Ge8qnAIt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DotProductAttention(Layer):\n",
        "    \"\"\"\n",
        "    dot-product-attention mechanism, supporting masking\n",
        "    \"\"\"\n",
        "    def __init__(self, return_attend_weight=False, keep_mask=True, **kwargs):\n",
        "        self.return_attend_weight = return_attend_weight\n",
        "        self.keep_mask = keep_mask\n",
        "        self.supports_masking = True\n",
        "        super(DotProductAttention, self).__init__(**kwargs)\n",
        "\n",
        "    def build(self, input_shape): # validate input data dimension\n",
        "        assert isinstance(input_shape, list)\n",
        "        input_shape_a, input_shape_b = input_shape\n",
        "\n",
        "        if len(input_shape_a) != 3 or len(input_shape_b) != 3:\n",
        "            raise ValueError('Inputs into DotProductAttention should be 3D tensors')\n",
        "\n",
        "        if input_shape_a[-1] != input_shape_b[-1]:\n",
        "            raise ValueError('Inputs into DotProductAttention should have the same dimensionality at the last axis')\n",
        "\n",
        "    def call(self, inputs, mask=None):\n",
        "        assert isinstance(inputs, list)\n",
        "        inputs_a, inputs_b = inputs\n",
        "\n",
        "        if mask is not None:\n",
        "            mask_a, mask_b = mask\n",
        "        else:\n",
        "            mask_a, mask_b = None, None\n",
        "\n",
        "        e = K.exp(K.batch_dot(inputs_a, inputs_b, axes=2))  # similarity between a & b\n",
        "\n",
        "        # apply mask before normalization (softmax)\n",
        "        if mask_a is not None:\n",
        "            e *= K.expand_dims(K.cast(mask_a, K.floatx()), 2)\n",
        "        if mask_b is not None:\n",
        "            e *= K.expand_dims(K.cast(mask_b, K.floatx()), 1)\n",
        "\n",
        "        e_b = e / K.cast(K.sum(e, axis=2, keepdims=True) + K.epsilon(), K.floatx())    # attention weight over b\n",
        "        e_a = e / K.cast(K.sum(e, axis=1, keepdims=True) + K.epsilon(), K.floatx())     # attention weight over a\n",
        "\n",
        "        if self.return_attend_weight:\n",
        "            return [e_b, e_a]\n",
        "\n",
        "        a_attend = K.batch_dot(e_b, inputs_b, axes=(2, 1))  # a attend to b\n",
        "        b_attend = K.batch_dot(e_a, inputs_a, axes=(1, 1))  # b attend to a\n",
        "        return [a_attend, b_attend]\n",
        "\n",
        "    def compute_mask(self, inputs, mask=None):\n",
        "        if self.keep_mask:\n",
        "            return mask\n",
        "        else:\n",
        "            return [None, None]\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        if self.return_attend_weight:\n",
        "            input_shape_a, input_shape_b = input_shape\n",
        "            return [(input_shape_a[0], input_shape_a[1], input_shape_b[1]),\n",
        "                    (input_shape_a[0], input_shape_a[1], input_shape_b[1])]\n",
        "        return input_shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mvu8NeydlQMl",
        "colab_type": "code",
        "outputId": "25f1baeb-4b64-4156-ecac-71ab6791e5fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def build_esim(mask_zero=True):\n",
        "  premise_embed, hypothesis_embed, inputs = build_input(mask_zero=True)\n",
        "  rnn_units=512\n",
        "  # input encoding\n",
        "  bilstm_1 = Bidirectional(LSTM(units=rnn_units, return_sequences=True))\n",
        "  premise_hidden = bilstm_1(premise_embed)\n",
        "  hypothesis_hidden = bilstm_1(hypothesis_embed)\n",
        "\n",
        "  # local inference collected over sequences\n",
        "  premise_attend, hypothesis_attend = DotProductAttention()([premise_hidden, hypothesis_hidden])\n",
        "\n",
        "  # enhancement of local inference information\n",
        "  premise_enhance = concatenate([premise_hidden, premise_attend, subtract([premise_hidden, premise_attend]),\n",
        "                                  multiply([premise_hidden, premise_attend])])\n",
        "  hypothesis_enhance = concatenate([hypothesis_hidden, hypothesis_attend,\n",
        "                                    subtract([hypothesis_hidden, hypothesis_attend]),\n",
        "                                    multiply([hypothesis_hidden, hypothesis_attend])])\n",
        "\n",
        "  # inference composition\n",
        "  feed_forward = TimeDistributed(Dense(units=rnn_units, activation='relu'))\n",
        "  bilstm_2 = Bidirectional(LSTM(units=rnn_units, return_sequences=True))\n",
        "  premise_compose = bilstm_2(feed_forward(premise_enhance))\n",
        "  hypothesis_compose = bilstm_2(feed_forward(hypothesis_enhance))\n",
        "\n",
        "  global_max_pooling = Lambda(lambda x: K.max(x, axis=1))  # GlobalMaxPooling1D didn't support masking\n",
        "  premise_avg = GlobalAveragePooling1D()(premise_compose)\n",
        "  premise_max = global_max_pooling(premise_compose)\n",
        "  hypothesis_avg = GlobalAveragePooling1D()(hypothesis_compose)\n",
        "  hypothesis_max = global_max_pooling(hypothesis_compose)\n",
        "\n",
        "  inference_compose = concatenate([premise_avg, premise_max, hypothesis_avg, hypothesis_max])\n",
        "\n",
        "  dense = Dense(units=256, activation='tanh')(inference_compose)\n",
        "  output = Dense(3, activation='softmax')(dense)\n",
        "\n",
        "  model = Model(inputs, output)\n",
        "  model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='adam')\n",
        "  return model\n",
        "\n",
        "esim_model = build_esim()\n",
        "print(esim_model.summary())"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_3 (InputLayer)            (None, 82)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_4 (InputLayer)            (None, 82)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_2 (Embedding)         (None, 82, 300)      10382100    input_3[0][0]                    \n",
            "                                                                 input_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_3 (Bidirectional) (None, 82, 1024)     3330048     embedding_2[0][0]                \n",
            "                                                                 embedding_2[1][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dot_product_attention_2 (DotPro [(None, 82, 1024), ( 0           bidirectional_3[0][0]            \n",
            "                                                                 bidirectional_3[1][0]            \n",
            "__________________________________________________________________________________________________\n",
            "subtract_3 (Subtract)           (None, 82, 1024)     0           bidirectional_3[0][0]            \n",
            "                                                                 dot_product_attention_2[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "multiply_3 (Multiply)           (None, 82, 1024)     0           bidirectional_3[0][0]            \n",
            "                                                                 dot_product_attention_2[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "subtract_4 (Subtract)           (None, 82, 1024)     0           bidirectional_3[1][0]            \n",
            "                                                                 dot_product_attention_2[0][1]    \n",
            "__________________________________________________________________________________________________\n",
            "multiply_4 (Multiply)           (None, 82, 1024)     0           bidirectional_3[1][0]            \n",
            "                                                                 dot_product_attention_2[0][1]    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_4 (Concatenate)     (None, 82, 4096)     0           bidirectional_3[0][0]            \n",
            "                                                                 dot_product_attention_2[0][0]    \n",
            "                                                                 subtract_3[0][0]                 \n",
            "                                                                 multiply_3[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_5 (Concatenate)     (None, 82, 4096)     0           bidirectional_3[1][0]            \n",
            "                                                                 dot_product_attention_2[0][1]    \n",
            "                                                                 subtract_4[0][0]                 \n",
            "                                                                 multiply_4[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed_2 (TimeDistrib (None, 82, 512)      2097664     concatenate_4[0][0]              \n",
            "                                                                 concatenate_5[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_4 (Bidirectional) (None, 82, 1024)     4198400     time_distributed_2[0][0]         \n",
            "                                                                 time_distributed_2[1][0]         \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_3 (Glo (None, 1024)         0           bidirectional_4[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "lambda_2 (Lambda)               (None, 1024)         0           bidirectional_4[0][0]            \n",
            "                                                                 bidirectional_4[1][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_4 (Glo (None, 1024)         0           bidirectional_4[1][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_6 (Concatenate)     (None, 4096)         0           global_average_pooling1d_3[0][0] \n",
            "                                                                 lambda_2[0][0]                   \n",
            "                                                                 global_average_pooling1d_4[0][0] \n",
            "                                                                 lambda_2[1][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense_5 (Dense)                 (None, 256)          1048832     concatenate_6[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_6 (Dense)                 (None, 3)            771         dense_5[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 21,057,815\n",
            "Trainable params: 10,675,715\n",
            "Non-trainable params: 10,382,100\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lMLmdHmDsGWF",
        "colab_type": "text"
      },
      "source": [
        "# 4. Training of Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPjKME_aop9g",
        "colab_type": "code",
        "outputId": "aa608950-fef3-4c6c-9766-85dd35f11a2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        }
      },
      "source": [
        "# train ESIM model with FastText\n",
        "batch_size = 256\n",
        "n_epoch = 4\n",
        "model_name='ESIM'\n",
        "model_save_path = '/content/gdrive/My Drive/NLP/'+model_name+'-{epoch:02d}-{val_acc:.4f}.hdf5'\n",
        "checkpoint = ModelCheckpoint(model_save_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
        "callback_list=[checkpoint]\n",
        "hist = esim_model.fit([main_seq_padded,hypo_seq_padded], y_train_e,\n",
        "          batch_size=batch_size,\n",
        "          epochs=n_epoch,\n",
        "          validation_data=([main_seq_padded_val,hypo_seq_padded_val], y_val_e),\n",
        "          callbacks=callback_list)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 549361 samples, validate on 9842 samples\n",
            "Epoch 1/4\n",
            "549361/549361 [==============================] - 2954s 5ms/step - loss: 0.5627 - acc: 0.7694 - val_loss: 0.4164 - val_acc: 0.8403\n",
            "\n",
            "Epoch 00001: val_acc improved from -inf to 0.84028, saving model to /content/gdrive/My Drive/NLP/bilstm-max-ESIM-01-0.8403.hdf5\n",
            "Epoch 2/4\n",
            "549361/549361 [==============================] - 2976s 5ms/step - loss: 0.4057 - acc: 0.8445 - val_loss: 0.3596 - val_acc: 0.8631\n",
            "\n",
            "Epoch 00002: val_acc improved from 0.84028 to 0.86314, saving model to /content/gdrive/My Drive/NLP/bilstm-max-ESIM-02-0.8631.hdf5\n",
            "Epoch 3/4\n",
            "549361/549361 [==============================] - 2967s 5ms/step - loss: 0.3520 - acc: 0.8673 - val_loss: 0.3401 - val_acc: 0.8706\n",
            "\n",
            "Epoch 00003: val_acc improved from 0.86314 to 0.87055, saving model to /content/gdrive/My Drive/NLP/bilstm-max-ESIM-03-0.8706.hdf5\n",
            "Epoch 4/4\n",
            "549361/549361 [==============================] - 2959s 5ms/step - loss: 0.3073 - acc: 0.8862 - val_loss: 0.3398 - val_acc: 0.8724\n",
            "\n",
            "Epoch 00004: val_acc improved from 0.87055 to 0.87238, saving model to /content/gdrive/My Drive/NLP/bilstm-max-ESIM-04-0.8724.hdf5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yA4HGEe2spEo",
        "colab_type": "code",
        "outputId": "1a9aa55c-e457-44b2-c33b-fa53f4c8224c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "source": [
        "# train Esim model with Glove\n",
        "batch_size = 256\n",
        "n_epoch = 4\n",
        "model_name='ESIM_Glove'\n",
        "model_save_path = '/content/gdrive/My Drive/NLP/'+model_name+'-{epoch:02d}-{val_acc:.4f}.hdf5'\n",
        "checkpoint = ModelCheckpoint(model_save_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
        "callback_list=[checkpoint]\n",
        "hist = esim_model.fit([main_seq_padded,hypo_seq_padded], y_train_e,\n",
        "          batch_size=batch_size,\n",
        "          epochs=n_epoch,\n",
        "          validation_data=([main_seq_padded_val,hypo_seq_padded_val], y_val_e),\n",
        "          callbacks=callback_list)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "Train on 549361 samples, validate on 9842 samples\n",
            "Epoch 1/4\n",
            "549361/549361 [==============================] - 3453s 6ms/step - loss: 0.5147 - acc: 0.7933 - val_loss: 0.3912 - val_acc: 0.8524\n",
            "\n",
            "Epoch 00001: val_acc improved from -inf to 0.85237, saving model to /content/gdrive/My Drive/NLP/ESIM_Glove-01-0.8524.hdf5\n",
            "Epoch 2/4\n",
            "549361/549361 [==============================] - 3436s 6ms/step - loss: 0.3834 - acc: 0.8542 - val_loss: 0.3570 - val_acc: 0.8654\n",
            "\n",
            "Epoch 00002: val_acc improved from 0.85237 to 0.86537, saving model to /content/gdrive/My Drive/NLP/ESIM_Glove-02-0.8654.hdf5\n",
            "Epoch 3/4\n",
            "549361/549361 [==============================] - 3482s 6ms/step - loss: 0.3209 - acc: 0.8810 - val_loss: 0.3474 - val_acc: 0.8728\n",
            "\n",
            "Epoch 00003: val_acc improved from 0.86537 to 0.87279, saving model to /content/gdrive/My Drive/NLP/ESIM_Glove-03-0.8728.hdf5\n",
            "Epoch 4/4\n",
            "549361/549361 [==============================] - 3422s 6ms/step - loss: 0.2616 - acc: 0.9051 - val_loss: 0.3555 - val_acc: 0.8705\n",
            "\n",
            "Epoch 00004: val_acc did not improve from 0.87279\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qX2b9xqL-zUO",
        "colab_type": "text"
      },
      "source": [
        "# 5.**===== Evaluation on Test set ======**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPEmzvjOhaTN",
        "colab_type": "code",
        "outputId": "1edca43a-13a7-4d1f-9312-0e0bbcc4686f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# evaluate on test set\n",
        "model = load_model('/content/gdrive/My Drive/NLP/bilstm-max-01-0.8542.hdf5')\n",
        "metrics = model.evaluate([main_seq_padded_test,hypo_seq_padded_test],y_test_e,batch_size=256,verbose=0)\n",
        "accs.append(round(metrics[1]*100,2))\n",
        "print(\"Original Paper Model Accuracy on test-set : \"+str(round(metrics[1]*100,2))+\" %\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original Paper Model Accuracy on test-set : 84.71 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mz04OvdtIlSL",
        "colab_type": "code",
        "outputId": "40e209d9-7957-4076-e892-435064b572a8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "source": [
        "# evaluate on test set\n",
        "accs = []\n",
        "model = build_esim()\n",
        "model.load_weights('/content/gdrive/My Drive/NLP/bilstm-max-ESIM-04-0.8724.hdf5')\n",
        "metrics = model.evaluate([main_seq_padded_test,hypo_seq_padded_test],y_test_e,batch_size=256)\n",
        "accs.append(round(metrics[1]*100,2))\n",
        "print(\"ESIM-FastText Accuracy on test-set : \"+str(round(metrics[1]*100,2))+\" %\")\n",
        "print()\n",
        "\n",
        "#model.load_weights('/content/gdrive/My Drive/NLP/ESIM_Glove-03-0.8728.hdf5')\n",
        "metrics = esim_model.evaluate([main_seq_padded_test,hypo_seq_padded_test],y_test_e,batch_size=256)\n",
        "accs.append(round(metrics[1]*100,2))\n",
        "print(\"ESIM-GloVe Accuracy on test-set : \"+str(round(metrics[1]*100,2))+\" %\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9824/9824 [==============================] - 25s 3ms/step\n",
            "ESIM-FastText Accuracy on test-set : 86.95 %\n",
            "\n",
            "9824/9824 [==============================] - 24s 2ms/step\n",
            "ESIM-GloVe Accuracy on test-set : 86.63 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8uTdlHO3sTZ0",
        "colab_type": "text"
      },
      "source": [
        "* Plot Accuracies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ddyjh73Qsqyb",
        "colab_type": "code",
        "outputId": "11b488cb-62a9-4065-d131-478a02b2cdb5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 558
        }
      },
      "source": [
        "def autolabel(rects):\n",
        "    \"\"\"Attach a text label above each bar \"\"\"\n",
        "    for rect in rects:\n",
        "        height = rect.get_height()\n",
        "        ax.annotate('{}'.format(height),\n",
        "                    xy=(rect.get_x() + rect.get_width() / 2, height),\n",
        "                    xytext=(0, 3),  # 3 points vertical offset\n",
        "                    textcoords=\"offset points\",\n",
        "                    ha='center', va='bottom')\n",
        "\n",
        "y_preds = [accs[2],accs[1],accs[0]]\n",
        "\n",
        "\n",
        "method_names=['Original Model','ESIM_GloVe','ESIM_FastText']\n",
        "accuracy=['Accuracy']\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(8,6))\n",
        "labels = method_names\n",
        "\n",
        "data = np.round(y_preds,2)\n",
        "\n",
        "x = np.arange(len(labels))  # the label locations\n",
        "width = 0.4  # the width of the bars\n",
        "\n",
        "\n",
        "rects1 = ax.bar(x, data, width)\n",
        "\n",
        "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
        "ax.set_ylabel('Accuracy')\n",
        "ax.set_title('Accuracy of each model')\n",
        "ax.set_xticks(x)\n",
        "ax.set_xticklabels(labels)\n",
        "ax.legend()\n",
        "\n",
        "autolabel(rects1)\n",
        "\n",
        "fig.tight_layout()\n",
        "\n",
        "plt.show()\n",
        "print()\n",
        "print(\"Best Performing model is ESIM-FastText\")\n",
        "print(\"There is no significant difference between using Glove or FastText Embeddings\")\n",
        "print(\"The Proposed ESIM-FastText outperforms original model by 2.24%\")\n",
        "print(\"---------------------------------------------------------------------------------\")\n",
        "print(\"Taha Samavati - 98722134 - Analysis of final results\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No handles with labels found to put in legend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGoCAYAAABL+58oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de7xVdZ3/8dcHDgmE5o0URRNDE0XB\nwjTTpl8mTU4XYswwJy85Of1+TRfN20Rp/eo3k2mRTqVjlHcTy0admixvZU1lIJGZWN5QUFRUyBBQ\nwc/vj7UObQ7nwObAPuf4Pa/n47EfZ6/vun3WOXvv897f9d1rR2YiSZJUkgG9XYAkSdKmZsCRJEnF\nMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQcA46kPisitouI2yLiLxHx5R7a55sjYkFP7Gt9IuKzEXF5\nk8v+NCL+sdU1SS8Vbb1dgKSuRcRPgXHA9pn5XC+X0xtOAJ4Etkgv2iVpA9iDI/VREbELcDCQwLt6\neN995c3Pq4C7DTeSNpQBR+q7jgZ+DVwMHNM4IyKGRMSXI+KhiPhzRPwiIobU8w6KiF9GxJKImB8R\nx9bta5zCiIhjI+IXDdMZER+JiHuBe+u2c+ttPBMRd0TEwQ3LD4yIT0XE/fUppDsiYqeI+HrH00kR\ncX1EnNjZQUbEgRExsz6OmRFxYN3eftynRsTSiHhrJ+tuFhHnRMTDEfF4RFzQ8HvYKiJ+EBGLImJx\nfX9kw7pbR8RFEfFoPf/aDtv+ZEQ8ERELI+K4rv5I9e/1C/XvfGlE/FdEbBMRV9S/t5l1WF3n8dbz\nRkXEz+rf543Ath32dUDD3/Z3EfHmruqS+jsDjtR3HQ1cUd/eFhHbNcw7B3gdcCCwNXAq8GJEvAr4\nEfDvwHBgPDBnA/Y5Cdgf2LOenllvY2vgSuC7ETG4nncScCRwGLAF8EFgGXAJcGREDACIiG2Bt9br\nryEitgZ+CJwHbAN8BfhhRGyTmcfWx/6lzByWmTd1Uu8Xgd3rGkcDOwJn1PMGABdR9QLtDCwHvtaw\n7mXAUGAv4JXAtIZ52wOvqLd3PPD1iNiqi98ZwBTgA/XyrwZ+Ve97a2AucOb6jrfezpXAHVTB5vM0\nBNuI2LFe9wv1dk8GromI4euoS+q/MtObN2997AYcBLwAbFtP3wOcWN8fQPXPelwn6/0L8J9dbPOn\nwD82TB8L/KJhOoG3rKeuxe37Bf4IvLuL5eYCh9b3/xn47y6W+wDwmw5tvwKOre9fDHyhi3UDeBZ4\ndUPbG4AHu1h+PLC4vj8CeBHYqpPl3lz/ftsa2p4ADljH73Vqw/SXgR81TL8TmLO+46UKYSuBlzfM\nuxK4vL5/GnBZh3V/DBzT2d/Xm7f+frMHR+qbjgF+kplP1tNX8td389sCg4H7O1lvpy7amzW/cSIi\nTo6IufXplCVUvRrtp03Wta9LgH+o7/8DVW9JZ3YAHurQ9hBVT8j6DKfqgbmjPmWzBLihbicihkbE\nf9Sn8Z4BbgO2jIiBde1PZ+biLrb9VGaubJheBgxbRy2PN9xf3sl0+7rrOt4dqALYsx3mtXsV8N72\nY62P9yCqsCapg74ykFBSrR5DcgQwMCIeq5s3o/rnPA74PbCC6lTI7zqsPh94fRebfpYqELTbvpNl\nVg/mrcfbnAocAvwhM1+MiMVUPSft+3o1cFcn27kcuKuudwxwbSfLADxK9Y+70c5UQWV9nqQKD3tl\n5iOdzP8k8Bpg/8x8LCLGA7+t658PbB0RW2bmkib2tams63gXAltFxMsbQs7O/PVvMp+qB+dDPVKp\n9BJnD47U90wCVlGNgxlf38YAPweOzswXgW8DX4mIHerBvm+IiM2oxqy8NSKOiIi2erDr+Hq7c4DJ\ndc/GaKqxJeuyOdUpk0VAW0ScQTXWpt104PMRsVtU9mkfS5KZC6jG71wGXJOZy7vYx38Du0fE++t6\n31cf9w/W90uqfw/fBKZFxCuhGqcSEW9rqH85sKQe+3Jmw7oLqcYqfaMejDwoIt60vn1uAl0eb2Y+\nBMwCPhcRL4uIg6hOb7W7HHhnRLyt/psPjuqaPSPX3o0kA47U9xwDXJSZD2fmY+03qgGyR0X1Ee6T\nqXpyZgJPA2cBAzLzYapBv5+s2+dQXUcHqkG0z1OdPrmEKgyty4+pehb+RHWqZAVrnsL6CnA18BPg\nGeBbwJCG+ZcAe9P16Sky8yngHXW9T1H1GL2j4dTc+pwG3Af8uj4NdRNVrw3AV+t6nqT6NFrHXqEP\nUI1zuodqjM0nmtxntzVxvO+nGuT9NFUgu7Rh3fnAu4FPUYXO+cAp+DoudSoyvbyEpE2v7hG5HHhV\n+kIjqYeZ/CVtchExCPg4MN1wI6k3GHAkbVIRMQZYQvXpnq/2cjmS+ilPUUmSpOLYgyNJkorzkrgO\nzrbbbpu77LJLb5chSZL6mDvuuOPJzFzrK0teEgFnl112YdasWb1dhiRJ6mMiouPVwQFPUUmSpAIZ\ncCRJUnEMOJIkqTgviTE4kiSpfC+88AILFixgxYoVa80bPHgwI0eOZNCgQU1tyx4cSZL6kGnTprHX\nXnsxduxYjjzySFasWEFmMnXqVHbffXfGjBnDeeed1+m6p512GmPHjmXs2LHMmDFjdfuxxx7LqFGj\nGD9+POPHj2fOnDk9dTgbZMGCBWy++ebssccejBkzZvVtjz32YPPNN2fBggVNb8uAI6lXbcyL+cMP\nP8zEiRMZM2YMe+65J/PmzQPg+OOPZ9y4ceyzzz4cfvjhLF26tAePSOq+Rx55hPPOO49Zs2Zx1113\nsWrVKq666iouvvhi5s+fzz333MPcuXOZMmXKWuv+8Ic/ZPbs2cyZM4fbb7+dc845h2eeeWb1/LPP\nPps5c+YwZ84cxo8f35OH1bQVK1awzTbbEBFrtEcE22yzTac9O13xFJWkXtP+Yn733XczZMgQjjji\nCK666ioyc/WL+YABA3jiiSc6Xf/oo49m6tSpHHrooSxdupQBA6r3bNOmTWOLLbYA4KSTTuJrX/sa\np59+eo8dl7QxVq5cyfLlyxk0aBDLli1jhx124NOf/jRXXnnl6sf4K1/5yrXWu/vuu3nTm95EW1sb\nbW1t7LPPPtxwww0cccQRPX0IG6VjuFlfe1fswZHUq9pfzFeuXLn6xfz888/njDPOWO+L+cqVKzn0\n0EMBGDZsGEOHDgVYHW4yk+XLl2/wC6PUW3bccUdOPvlkdt55Z0aMGMErXvEKJk6cyP3338+MGTOY\nMGECb3/727n33nvXWnfcuHHccMMNLFu2jCeffJJbb72V+fPnr54/depU9tlnH0488USee+65njys\nXmHAkdRrNubF/E9/+hNbbrklkydPZt999+WUU05h1apVq+cfd9xxbL/99txzzz189KMf7cnDkrpt\n8eLFXHfddTz44IM8+uijPPvss1x++eU899xzDB48mFmzZvGhD32ID37wg2utO3HiRA477DAOPPBA\njjzySN7whjcwcOBAAP7t3/6Ne+65h5kzZ/L0009z1lln9fSh9TgDjqReszEv5itXruTnP/8555xz\nDjNnzuSBBx7g4osvXj3/oosu4tFHH2XMmDFrDLaU+rKbbrqJUaNGMXz4cAYNGsTkyZP55S9/yciR\nI5k8eTIA73nPe7jzzjs7XX/q1KnMmTOHG2+8kcxk9913B2DEiBFEBJttthnHHXccv/nNb3rsmDZU\nV18CvqFfDm7AkdRrNubFfOTIkYwfP55dd92VtrY2Jk2axOzZs9dYZuDAgUyZMoVrrrmmR45H2lg7\n77wzv/71r1m2bBmZyc0338yYMWOYNGkSt956KwA/+9nPVgeXRqtWreKpp54C4M477+TOO+9k4sSJ\nACxcuBCoQsK1117L2LFje+iINszgwYN56qmn1gozmclTTz3F4MGDm96Wg4wl9ZrGF/MhQ4Zw8803\nM2HCBLbYYgtuvfVWRo0a1eWL+X777ceSJUtYtGgRw4cP55ZbbmHChAlkJvfffz+jR48mM7n++uvZ\nY489euHopA23//77c/jhh/Pa176WtrY29t13X0444QSWL1/OUUcdxbRp0xg2bBjTp08HYNasWVxw\nwQVMnz6dF154gYMPPhioxqFdfvnltLVV/+aPOuooFi1aRGYyfvx4Lrjggl47xnUZOXIkCxYsYNGi\nRWvNa78OTrNiQ7t8esOECRPSL9uUynTmmWcyY8aM1S/m06dPX/1i/vDDDzNs2DAuuOACxo0bt8aL\nOcCNN97IJz/5STKT173udVx44YW0tbVx8MEH88wzz5CZjBs3jvPPP3/1wGNJZYmIOzJzwlrtBpye\nM23aNKZPn05EsPfee3PRRRet7m772Mc+xre//e1Or9dxxRVXcPbZZ6+evvPOO5k9ezbjx49n6tSp\nXHrppSxevNhrfUiS+p2uAo5jcHpIVxdvgqqLcfHixV2ue9RRR62+ONNll122+mqUAO985zv79GAx\nSZJ6gwGnB3V2vY9Vq1Zxyimn8KUvfampbXznO99Z4wqWBxxwACNGjGhVyZIkvSQ5yLiHNF7vY8iQ\nIUycOJGJEydy7rnn8q53vavpkDJjxgyuu+66FlcrSWq0y+k/7O0SXrLmffHvemW/9uD0kM6u93Hp\npZfy3e9+t+mLkN1+++0MHTq0z368T5KkvsIenB7SeL0PgMmTJ3PmmWeyfPlyRo8eDcCyZcsYPXo0\n9913X6fbuOqqqzjyyCN7rGb1X75b7b7eercqaU324PSQzi7edNJJJ/HYY48xb9485s2bx9ChQ7sM\nNy+++CJXX311p98gK0mS1mTA6SGNF2/ae++9efHFFznhhBO6XP7666/njDPOWD192223sdNOO7Hr\nrruusdypp57KyJEjWbZsGSNHjuSzn/1sqw5BkqSXDK+DI2ktnqLqPk9RlcnnRPe1+jnhdXAkSVK/\nYcCRJEnFMeBIkqTiGHAkSVJx+v11cBw41n0OppQk9VX24EiSpOIYcCRJUnEMOJIkqTgGHEmSVBwD\njiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQV\nx4AjSZKK09KAExEnRsQfIuKuiPhORAyOiFERcXtE3BcRMyLiZa2sQZIk9T8tCzgRsSPwMWBCZo4F\nBgJTgLOAaZk5GlgMHN+qGiRJUv/U6lNUbcCQiGgDhgILgbcA36vnXwJManENkiSpn2lZwMnMR4Bz\ngIepgs2fgTuAJZm5sl5sAbBjZ+tHxAkRMSsiZi1atKhVZUqSpAK18hTVVsC7gVHADsDLgb9tdv3M\nvDAzJ2TmhOHDh7eoSkmSVKJWnqJ6K/BgZi7KzBeA7wNvBLasT1kBjAQeaWENkiSpH2plwHkYOCAi\nhkZEAIcAdwO3AofXyxwDXNfCGiRJUj/UyjE4t1MNJp4N/L7e14XAacBJEXEfsA3wrVbVIEmS+qe2\n9S/SfZl5JnBmh+YHgNe3cr+SJKl/80rGkiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnF\nMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJ\nUnEMOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYc\nSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqO\nAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEMOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mS\nimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBI\nkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEM\nOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOC0NOBGx\nZUR8LyLuiYi5EfGGiNg6Im6MiHvrn1u1sgZJktT/tLoH51zghszcAxgHzAVOB27OzN2Am+tpSZKk\nTaZlASciXgG8CfgWQGY+n5lLgHcDl9SLXQJMalUNkiSpf2plD84oYBFwUUT8NiKmR8TLge0yc2G9\nzGPAdp2tHBEnRMSsiJi1aNGiFpYpSZJK08qA0wa8Fjg/M/cFnqXD6ajMTCA7WzkzL8zMCZk5Yfjw\n4S0sU5IklaaVAWcBsCAzb6+nv0cVeB6PiBEA9c8nWliDJEnqh1oWcDLzMWB+RLymbjoEuBu4Hjim\nbjsGuK5VNUiSpP6prcXb/yhwRUS8DHgAOI4qVF0dEccDDwFHtLgGSZLUz7Q04GTmHGBCJ7MOaeV+\nJUlS/+aVjCVJUnEMOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElS\ncQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBIkqTiGHAkSVJxDDiSJKk4BhxJ\nklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEMOJIkqTgGHEmSVBwDjiRJKo4B\nR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScdYbcCLioxGxVU8UI0mStCk004OzHTAzIq6OiL+N\niGh1UZIkSRtjvQEnMz8N7AZ8CzgWuDci/jUiXt3i2iRJkrqlqTE4mZnAY/VtJbAV8L2I+FILa5Mk\nSeqWtvUtEBEfB44GngSmA6dk5gsRMQC4Fzi1tSVKkiRtmPUGHGBrYHJmPtTYmJkvRsQ7WlOWJElS\n9zVziupHwNPtExGxRUTsD5CZc1tVmCRJUnc1E3DOB5Y2TC+t2yRJkvqkZgJO1IOMgerUFM2d2pIk\nSeoVzQScByLiYxExqL59HHig1YVJkiR1VzMB58PAgcAjwAJgf+CEVhYlSZK0MdZ7qikznwCm9EAt\nkiRJm0Qz18EZDBwP7AUMbm/PzA+2sC5JkqRua+YU1WXA9sDbgJ8BI4G/tLIoSZKkjdFMwBmdmZ8B\nns3MS4C/oxqHI0mS1Cc1E3BeqH8uiYixwCuAV7auJEmSpI3TzPVsLoyIrYBPA9cDw4DPtLQqSZKk\njbDOgFN/oeYzmbkYuA3YtUeqkiRJ2gjrPEVVX7XYbwuXJEkvKc2MwbkpIk6OiJ0iYuv2W8srkyRJ\n6qZmxuC8r/75kYa2xNNVkiSpj2rmSsajeqIQSZKkTaWZKxkf3Vl7Zl666cuRJEnaeM2cotqv4f5g\n4BBgNmDAkSRJfVIzp6g+2jgdEVsCV7WsIkmSpI3UzKeoOnoWcFyOJEnqs5oZg/NfVJ+agioQ7Qlc\n3cqiJEmSNkYzY3DOabi/EngoMxe0qB5JkqSN1kzAeRhYmJkrACJiSETskpnzWlqZJElSNzUzBue7\nwIsN06vqNkmSpD6pmYDTlpnPt0/U91/WupIkSZI2TjMBZ1FEvKt9IiLeDTzZupIkSZI2TjNjcD4M\nXBERX6unFwCdXt1YkiSpL2jmQn/3AwdExLB6emnLq5IkSdoI6z1FFRH/GhFbZubSzFwaEVtFxBd6\nojhJkqTuaGYMztszc0n7RGYuBg5rXUmSJEkbp5mAMzAiNmufiIghwGbrWF6SJKlXNTPI+Arg5oi4\nCAjgWOCSVhYlSZK0MZoZZHxWRPwOeCvVd1L9GHhVqwuTJEnqrma/TfxxqnDzXuAtwNyWVSRJkrSR\nuuzBiYjdgSPr25PADCAy83/1UG2SJEndsq4enHuoemvekZkHZea/U30P1QaJiIER8duI+EE9PSoi\nbo+I+yJiRkT4tQ+SJGmTWlfAmQwsBG6NiG9GxCFUg4w31MdZ85TWWcC0zBwNLAaO78Y2JUmSutRl\nwMnMazNzCrAHcCvwCeCVEXF+RExsZuMRMRL4O2B6PR1UvULfqxe5BJjU/fIlSZLWtt5Bxpn5bGZe\nmZnvBEYCvwVOa3L7XwVOBV6sp7cBlmTmynp6AbBjZytGxAkRMSsiZi1atKjJ3UmSJDX/KSqguopx\nZl6YmYesb9mIeAfwRGbe0Z3C6v1MyMwJw4cP784mJElSP9XMhf66643AuyLiMGAwsAVwLrBlRLTV\nvTgjgUdaWIMkSeqHNqgHZ0Nk5r9k5sjM3AWYAtySmUdRjec5vF7sGOC6VtUgSZL6p5YFnHU4DTgp\nIu6jGpPzrV6oQZIkFayVp6hWy8yfAj+t7z8AvL4n9itJkvqn3ujBkSRJaikDjiRJKo4BR5IkFceA\nI0mSimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnF\nMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJ\nUnEMOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYc\nSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqO\nAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEMOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mS\nimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBI\nkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEM\nOJIkqTgGHEmSVJyWBZyI2Ckibo2IuyPiDxHx8bp964i4MSLurX9u1aoaJElS/9TKHpyVwCczc0/g\nAOAjEbEncDpwc2buBtxcT0uSJG0yLQs4mbkwM2fX9/8CzAV2BN4NXFIvdgkwqVU1SJKk/qlHxuBE\nxC7AvsDtwHaZubCe9RiwXRfrnBARsyJi1qJFi3qiTEmSVIiWB5yIGAZcA3wiM59pnJeZCWRn62Xm\nhZk5ITMnDB8+vNVlSpKkgrQ04ETEIKpwc0Vmfr9ufjwiRtTzRwBPtLIGSZLU/7TyU1QBfAuYm5lf\naZh1PXBMff8Y4LpW1SBJkvqnthZu+43AB4DfR8Scuu1TwBeBqyPieOAh4IgW1iBJkvqhlgWczPwF\nEF3MPqRV+5UkSfJKxpIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElS\ncQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBIkqTiGHAkSVJxDDiSJKk4BhxJ\nklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEMOJIkqTgGHEmSVBwDjiRJKo4B\nR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScQw4kiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKK\nY8CRJEnFMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQcA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiS\npOIYcCRJUnEMOJIkqTgGHEmSVBwDjiRJKo4BR5IkFceAI0mSimPAkSRJxTHgSJKk4hhwJElScQw4\nkiSpOAYcSZJUHAOOJEkqjgFHkiQVx4AjSZKKY8CRJEnFMeBIkqTiGHAkSVJxDDiSJKk4BhxJklQc\nA44kSSqOAUeSJBXHgCNJkopjwJEkScUx4EiSpOIYcCRJUnEMOJIkqTgGHEmSVJxeCTgR8bcR8ceI\nuC8iTu+NGiRJUrl6POBExEDg68DbgT2BIyNiz56uQ5Iklas3enBeD9yXmQ9k5vPAVcC7e6EOSZJU\nqLZe2OeOwPyG6QXA/h0XiogTgBPqyaUR8cceqK0v2hZ4sreL6Eyc1dsVqJ/qs88J8HmhXtHfnxOv\n6qyxNwJOUzLzQuDC3q6jt0XErMyc0Nt1SH2FzwlpTT4nOtcbp6geAXZqmB5Zt0mSJG0SvRFwZgK7\nRcSoiHgZMAW4vhfqkCRJherxU1SZuTIi/hn4MTAQ+HZm/qGn63gJ6fen6aQOfE5Ia/I50YnIzN6u\nQZIkaZPySsaSJKk4BhxJklQcA04nImJkRFwXEfdGxP0RcW49ILqzZXeIiO81sc3/jogtu1nPZyPi\n5C7aMyJGN7R9om5r+iODEXFsRHxtY5eRJKmvMOB0EBEBfB+4NjN3A3YHhgH/r5Nl2zLz0cw8fH3b\nzczDMnPJJi8Yfk/1SbR27wUctK0eExGrImJOw+30uv0dEfHbiPhdRNwdEf9Ut68O7BFxcUQsi4jN\nG7b31Tqkb7uOfW4XEVdGxAMRcUdE/Coi3lPPe3NE/GAd6+4SEQsiYkCH9jkRsdZFR6Wu9NJjv+M+\nd+lG3Z+qf27TsJ3HIuKRhulO39SvY5tbR8SHN7SWVjLgrO0twIrMvAggM1cBJwIfjIihdU/G9RFx\nC3Bz/WJ5F0A9/+r6Af2fEXF7e09KRMyLiG3r5edGxDcj4g8R8ZOIGFIv86GImFk/Ka6JiKFN1Hst\n9VddRMSrgT/TcEXLiDgyIn4fEXdF/PV6khFxXET8KSJ+A7yxoX14ve+Z9e2NSOu2PDPHN9y+GBGD\nqD7Z8c7MHAfsC/y0i/Xv46+P4QFUz8Eur41Vvwm5FrgtM3fNzNdRhfyRzRSbmfOAh4GDG7a5B7B5\nZt7ezDakWo8+9rvY57xu1P0pgMx8qn07wAXAtIbtPr+B29waMOD0cXsBdzQ2ZOYzVC+I7aeCXgsc\nnpl/02Hd/wMszsw9gc8Ar+tiH7sBX8/MvYAlwN/X7d/PzP3qJ8Vc4Pgm6n0GmB8RY6le5Ge0z4iI\nHYCzqJ4044H9ImJSRIwAPkcVbA6i+tLTdudSPcj3q+ua3kQNUkebU12G4imAzHwuM7v6upWrgPfV\n998M/A+wch3bfgvwfGZe0N6QmQ9l5r93XLB+V3ltRNwZEb+OiH3qWd9hzZ7PKXUdhnxtrFY+9jtV\nv3H+eUTMrm8H1u0jIuK2urR4JHkAAATBSURBVEfmrog4OCK+CAyp265Yz3aPiYjf1Mt+IyIGRHUN\nu3vr59bAiPhlRLwF+CLwmnrZL27oMbSCAad7bszMpztpP4j6RTIz7wLu7GL9BzNzTn3/DmCX+v7Y\n+kH6e+AoqrDVjKuoXqAnAf/Z0L4f8NPMXJSZK4ErgDdRffdXe/vzNIQi4K3A1yJiDtUFGLeIiGFN\n1qH+qf3Fsv32vvr5cT3wUER8JyKOig6nhBr8CRgeEVsBR1I/h9ZhL2B2k7V9DvhtZu5D9a710rr9\namBSRLRfC+x9VKEHDPlqXk8/9jvus/31/gng0Mx8LdVj+by6/f3Aj+semnHAnMw8nb/2Ah3V1U7q\nN83vAQ6s128DpmTmg8CXgW8Ap1I9v24BTgf+WG/39CaOo+X67HdR9aK7gTXG1ETEFsDOVN2JrwWe\n3ch9PNdwfxUwpL5/MTApM38XEcdSJfpm/AA4G5iVmc9UPfjdNgA4IDNXNDZu5DZVtuX1C+AaMvMf\nI2JvqtB8MnAocGwX2/g+VUjfH/inDdl5RHyd6s3F83UoaXQQdQ9pZt4S1ZiDLTLz8frU8iER8Tiw\nsn5TQl3vng2P+S0iYlhmLt2QutQv9MZjv7N9DqJ6Yzqe6n/K7nX7TODb9WmzaxveWDfjrVRvkmfV\nz4Uh1F+UnZkXRMR7geOoTsH1SfbgrO1mYGhEHA0QEQOp0urFmblsPev+D3BEvd6ewN4buO/NgYX1\ng7HLZN1RXddprD0Q+jfA39RjfwZSvUP4GXB73b5Nva/3NqzzE+Cj7RP1E0bqlsz8fWZOo3qB//t1\nLDoD+DxV7+iL69nsH6jeaLTv4yPAIcDwDSyv/TTVFP7aewN/DfntYxF2NNxoQ7Xosd+VE4HHqXpp\nJgAvq2u4jarX/hHg4vb/a00Kqm8aaH8evCYzPw9Q9+rvQPVtBH22h9+A00FWl3Z+D/DeiLiXqgtx\nBfWgrPX4BlV3493AF6heiP+8Abv/DFX4+B/gng2s+6rMnN2hbSFVt+GtwO+AOzLzurr9s8Cv6n3N\nbVjtY8CEeszC3fSxQWN6aYiIYRHx5oam8cBDXS2fmQ8BU6meQ+tzCzA4Iv53Q1tXA/J/Tv1moa7n\nyXpMHVTvnA+j6tJvPDVgyFe3tfix35VXAAvrgPQBquBBRLwKeDwzv0l1qrX9jcEL9ZvbdbkJOCLq\nT3TVb4h3ruedDVwE/F/gP+q2v1C9Se8z/KqGTajuJRmUmSui+kTTTcBrujEaXXrJiIhVVJcraHcD\nVW/iDODVwHKq07ofz8xZEfFZYGlmnhMRFwM/yMzvddjmPGBCZj5JJ+qB8tOouvUX1du/IDNn1P9c\nTs7Md0TE1sC3gV2BZcAJmXlnw3auBbbPzAMa2rYFvg6MoTqNf1tmGvS1ll567C/NzGEd2nYDrgGy\nruEjmTksIo4BTgFeAJYCR2fmg1F9ovZdwOz2cTiNtTVs9/1U42wG1Nv4MLAFVY/TwZm5KiKuB76b\nmZdFxNVUz5sf9oVxOAacTSiq6xncSnU+NIDTMvNHvVuVJEn9jwFHkiQVx09RSeqTImIbqkH/HR2S\nmU/1dD1ST/Gxv2nYgyNJkorjp6gkSVJxDDiSJKk4BhxJklQcA44kSSrO/wfc5kNR0MROgAAAAABJ\nRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 576x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Best Performing model is ESIM-FastText\n",
            "There is no significant defference between using Glove or FastText Embeddings\n",
            "The Proposed ESIM-FastText outperforms original model by 2.24%\n",
            "---------------------------------------------------------------------------------\n",
            "Taha Samavati - 98722134 - Analysis of final results\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aZggYLtnj2u5",
        "colab_type": "code",
        "outputId": "19eb17bd-c9c8-4f19-faf7-858509f4393d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 590
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "def plot_acc(history, title):\n",
        "\n",
        "  plt.plot(history.history['acc'])\n",
        "  print(history.history['acc'])\n",
        "  plt.plot(history.history['val_acc'])\n",
        "  plt.title('Model accuracy')\n",
        "  plt.ylabel('Accuracy')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Test'], loc='upper left')\n",
        "  plt.show()\n",
        "\n",
        "def plot_loss(history, title):\n",
        "\n",
        "  plt.plot(history.history['loss'])\n",
        "  plt.plot(history.history['val_loss'])\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Test'], loc='upper left')\n",
        "  plt.show()\n",
        "\n",
        "plot_acc(hist,\"Accuracy during training - ESIM GLOVE\")\n",
        "plot_loss(hist,\"Loss during training - ESIM GLOVE\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0.7933490000228443, 0.8542270019208953, 0.8810454327883241, 0.9051243171637063]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU1fn48c9DFhKWECDs2VhlkT2y\nWLe6Iu5VKyggCFL91qW2tqX92brUb2v9dbPVH4qIEECRulRatVpbQStrQBbZFGJCErYQSFhC9uf3\nx72RIQ4wQCY3M/O8X695Zebec2ee4+B95pxz7zmiqhhjjDF1NfE6AGOMMY2TJQhjjDF+WYIwxhjj\nlyUIY4wxflmCMMYY45clCGOMMX5ZgjART0TSRURFJDqAshNF5L8NEZcxXrMEYUKKiOSISIWIJNXZ\n/pl7kk/3JjJjwo8lCBOKvgLG1r4Qkf5AM+/CaRwCaQEZczosQZhQNBeY4PP6TiDTt4CItBKRTBEp\nFJFcEXlERJq4+6JE5Hcisk9EsoFr/Bz7kojsEpECEXlSRKICCUxE/ioiu0WkREQ+FpF+PvviReT3\nbjwlIvJfEYl3910gIktFpFhE8kRkort9sYhM8XmP47q43FbT90XkS+BLd9sz7nscFJHVInKhT/ko\nEfm5iGwXkUPu/hQReU5Efl+nLotE5KFA6m3CkyUIE4qWAwki0sc9cY8B5tUp8xegFdANuBgnoUxy\n990NXAsMBjKAW+ocOxuoAnq4Za4EphCY94CeQHtgDTDfZ9/vgKHA+UAb4CdAjYikucf9BWgHDALW\nBvh5ADcCw4G+7utV7nu0AV4B/ioice6+H+K0vkYDCcBdQCkwBxjrk0STgMvd402kUlV72CNkHkAO\nzonrEeA3wCjgX0A0oEA6EAVUAH19jvsesNh9/h/gHp99V7rHRgMdgHIg3mf/WOAj9/lE4L8Bxpro\nvm8rnB9jR4GBfsr9DHjrBO+xGJji8/q4z3ff/9JTxHGg9nOBrcANJyi3GbjCfX4f8K7X37c9vH1Y\nn6UJVXOBj4Gu1OleApKAGCDXZ1su0MV93hnIq7OvVpp77C4Rqd3WpE55v9zWzP8Ct+K0BGp84mkK\nxAHb/RyacoLtgTouNhF5GJiMU0/FaSnUDuqf7LPmAONwEu444JmziMmEAetiMiFJVXNxBqtHA2/W\n2b0PqMQ52ddKBQrc57twTpS++2rl4bQgklQ10X0kqGo/Tu124AacFk4rnNYMgLgxlQHd/RyXd4Lt\nAEc4fgC+o58yX0/J7I43/AT4LtBaVROBEjeGU33WPOAGERkI9AH+doJyJkJYgjChbDJO98oR342q\nWg0sBP5XRFq6ffw/5Ng4xULgARFJFpHWwDSfY3cBHwC/F5EEEWkiIt1F5OIA4mmJk1yKcE7qv/Z5\n3xpgFvAHEensDhaPFJGmOOMUl4vId0UkWkTaisgg99C1wHdEpJmI9HDrfKoYqoBCIFpEfonTgqg1\nE/iViPQUxwARaevGmI8zfjEXeENVjwZQZxPGLEGYkKWq21U16wS778f59Z0N/BdnsHWWu+9F4H1g\nHc5Act0WyAQgFtiE03//OtApgJAycbqrCtxjl9fZ/zCwAeckvB/4LdBEVXfgtIR+5G5fCwx0j/kj\nznjKHpwuoPmc3PvAP4Ev3FjKOL4L6g84CfID4CDwEhDvs38O0B8nSZgIJ6q2YJAxxiEiF+G0tNLU\nTg4Rz1oQxhgARCQGeBCYacnBgCUIYwwgIn2AYpyutD95HI5pJKyLyRhjjF/WgjDGGONX2Nwol5SU\npOnp6V6HYYwxIWX16tX7VLWdv31hkyDS09PJyjrRFY/GGGP8EZHcE+2zLiZjjDF+WYIwxhjjlyUI\nY4wxfoXNGIQ/lZWV5OfnU1ZW5nUoDSYuLo7k5GRiYmK8DsUYE+LCOkHk5+fTsmVL0tPT8Zm6OWyp\nKkVFReTn59O1a1evwzHGhLiw7mIqKyujbdu2EZEcAESEtm3bRlSLyRgTPGGdIICISQ61Iq2+xpjg\nCfsEYYwx4epIeRXzV+Ty3oZdQXn/sB6D8FpRURGXXXYZALt37yYqKop27ZwbFleuXElsbOwp32PS\npElMmzaNc845J6ixGmNCR3bhYeYuz+X1rHwOlVdx3cDOXN0/kCVLTo8liCBq27Yta9euBeCxxx6j\nRYsWPPzww8eVqV0cvEkT/425l19+OehxGmMav+oa5T9b9pK5LIdPvtxHTJQwun8nJoxMY0hq66B8\npnUxeWDbtm307duXO+64g379+rFr1y6mTp1KRkYG/fr144knnvi67AUXXMDatWupqqoiMTGRadOm\nMXDgQEaOHMnevXs9rIUxpiHsP1LB9MXbuejpj7g7M4sv9xzmR1f0Yum0y3hmzGCGprUJ2thjxLQg\nHv/7RjbtPFiv79m3cwKPXhfIWvbftGXLFjIzM8nIyADgqaeeok2bNlRVVfHtb3+bW265hb59+x53\nTElJCRdffDFPPfUUP/zhD5k1axbTpk3z9/bGmBC3Lq+YzGW5/H39TiqqahjZrS2PXNOHK/p2IDqq\nYX7bR0yCaGy6d+/+dXIAePXVV3nppZeoqqpi586dbNq06RsJIj4+nquvvhqAoUOH8sknnzRozMaY\n4CqrrOad9bvIXJ7LurximsdGcVtGCuNHptGrQ8sGjydiEsSZ/tIPlubNm3/9/Msvv+SZZ55h5cqV\nJCYmMm7cOL/3MvgOakdFRVFVVdUgsRpjgiv/QCnzV+zgtVV57D9SQfd2zXn8+n58Z0gXWsZ5NytC\nUBOEiIwCngGicNa5farO/jRgFtAO2A+MU9V8d9+dwCNu0SdVdU4wY/XSwYMHadmyJQkJCezatYv3\n33+fUaNGeR2WMSaIVJVPtxUxZ1kO/968B4Ar+nZgwsh0zu/eOG7wDVqCEJEo4DngCiAfWCUii1R1\nk0+x3wGZqjpHRC4FfgOMF5E2wKNABqDAavfYA8GK10tDhgyhb9++9O7dm7S0NL71rW95HZIxJkgO\nllXy5up8Mpfnkl14hLbNY7n3ku7cPjyNLonxXod3nKCtSS0iI4HHVPUq9/XPAFT1Nz5lNgKjVDVP\nnHRZoqoJIjIWuERVv+eWewFYrKqvnujzMjIytO6CQZs3b6ZPnz71XbVGL1LrbUxjtnX3ITKX5fDW\nZwWUVlQzKCWRO89PY3T/TjSNjvIsLhFZraoZ/vYFs4upC5Dn8zofGF6nzDrgOzjdUDcBLUWk7QmO\n7VL3A0RkKjAVIDU1td4CN8aY+lBZXcO/Nu1hztIcVny1n9joJlw/sDMTRqYxIDnR6/BOyetB6oeB\nZ0VkIvAxUABUB3qwqs4AZoDTgghGgMYYc7r2Hipjwco85q/IZc/BcpJbxzPt6t58NyOFNs1PPYNC\nYxHMBFEApPi8Tna3fU1Vd+K0IBCRFsDNqlosIgXAJXWOXRzEWI0x5qyoKqtzDzBnWS7//HwXldXK\nRb3a8b83pvHt3u2JauL9oPPpCmaCWAX0FJGuOIlhDHC7bwERSQL2q2oN8DOcK5oA3gd+LSK1949f\n6e43xphG5WhFNW+vLWDOslw27zpIy7hoxo9IZ9yIVLq1a+F1eGclaAlCVatE5D6ck30UMEtVN4rI\nE0CWqi7CaSX8RkQUp4vp++6x+0XkVzhJBuAJVd0frFiNMeZ05ew7wrzluSzMyuNgWRW9O7bk1zf1\n58bBnWkW63Xvff0Iai1U9V3g3Trbfunz/HXg9RMcO4tjLQpjjPFcdY2y5Iu9ZC7LZfHWQqKbCKPO\n7ciEkemcl966Udy7UJ/CI801UvUx3TfArFmzGD16NB07dgxarMaYEysurWBhVh5zl+eSt/8o7Vs2\n5QeX9+T2Yam0T4jzOrygsQQRRIFM9x2IWbNmMWTIEEsQxjSwzwtKyFyWw9trd1JeVcOwrm346aje\nXNWvIzENNGGelyxBeGTOnDk899xzVFRUcP755/Pss89SU1PDpEmTWLt2LarK1KlT6dChA2vXruW2\n224jPj7+tFoexpjTV15VzXsbdpO5LIc1O4qJj4ni5qHJTBiZRu+OCV6H16AiJ0G8Nw12b6jf9+zY\nH65+6tTl6vj888956623WLp0KdHR0UydOpUFCxbQvXt39u3bx4YNTpzFxcUkJibyl7/8hWeffZZB\ngwbVb/zGmK/tLD7KKyt2sGDVDvYdrqBrUnN+eW1fbh6aTKt47ybM81LkJIhG5MMPP2TVqlVfT/d9\n9OhRUlJSuOqqq9i6dSsPPPAA11xzDVdeeaXHkRoT3lSVZdlFZC7N5V+b91CjymW9OzBhZBoX9Eii\nSQjeu1CfIidBnMEv/WBRVe666y5+9atffWPf+vXree+993juued44403mDFjhgcRGhPeDpdX8eaa\nfOYuy+XLvYdp3SyGuy/sxh3DU0lp08zr8BqNyEkQjcjll1/OLbfcwoMPPkhSUhJFRUUcOXKE+Ph4\n4uLiuPXWW+nZsydTpkwBoGXLlhw6dMjjqI0Jfdv2HiJzWS5vringcHkVA5Jb8btbB3LtgE7ExXg3\nYV5jZQnCA/379+fRRx/l8ssvp6amhpiYGJ5//nmioqKYPHkyqoqI8Nvf/haASZMmMWXKFBukNuYM\nVFXX8OHmvWQuy2Hp9iJio5pw7YBOTDg/nUEpjX/CPC8FbbrvhmbTfR8TqfU2xte+w+W8tiqP+ctz\n2VlSRpfEeG4fnsqY81Jo26Kp1+E1Gl5N922MMQ1KVfksr5jMpTm8u2E3FdU1XNAjiUev78dlvdsT\nHQH3LtQnSxDGmJBXVlnNonU7yVyWw+cFB2nRNJrbh6cybkQaPdqH9oR5Xgr7BFHbnx8pwqXL0JhA\n5O0vZd7yXF7LyqO4tJJeHVrwqxvP5abBXWjRNOxPb0EX1v8F4+LiKCoqom3bxrEAeLCpKkVFRcTF\nhe/cMMbU1Cgff1lI5rJcPtq6lyYiXNWvA+NHpDOiW5uI+H+9oYR1gkhOTiY/P5/CwkKvQ2kwcXFx\nJCcnex2GMfWupLSSv67OY97yXHKKSklq0ZT7v92DscNT6dQq3uvwwlJYJ4iYmBi6du3qdRjGmLOw\naedB5i7P4a3PCiirrCEjrTUPXdGLq8/tRGy0DToHU1gnCGNMaKqoquGfG3eTuTSHrNwDxMU04cZB\nXRg/Mo1+nVt5HV7EsARhjGk09hwsY/6KHby6cgeFh8pJbdOMR67pw61DU2jVLDInzPOSJQhjjKdU\nlZVf7SdzWS7vb9xNtSqX9GrHhPPTubhnu4ifMM9LliCMMZ44Ul7FW58VMHdZLlv3HKJVfAyTvpXO\nuBFppLVt7nV4BksQxpgGtr3wMHOX5fLG6nwOlVfRr3MCT988gOsGdiY+1ibMa0wsQRhjgq66RvnP\nFmfCvE++3EdMlDC6fycmjExnSGqi3bvQSFmCMMYEzf4jFSxYtYP5y3dQUHyUjglx/OiKXowZlkq7\nljZhXmNnCcIYU+/W5RUzZ1kO/1i/i4qqGkZ2a8svru3D5X062IR5IcQShDGmXpRVVvPO+l1kLsth\nXX4JzWOjuC0jhfEj0+jVoaXX4ZkzYAnCGHNW8g+UMn/FDl5blcf+IxV0b9ecx6/vx3eGdKFlnN27\nEMosQRhjTltNjfLp9n3MWZrLf7bsAeCKvh2YMDKd87tHxuSYkcAShDEmYAfLKnljdT5zl+WSve8I\nbZvHcu8l3bl9eBpdEm3CvHAT1AQhIqOAZ4AoYKaqPlVnfyowB0h0y0xT1XdFJAaYCQxxY8xU1d8E\nM1ZjzIntLD7KzE++YsGqHZRWVDMoJZE/3jaQ0f070TTa7l0IV0FLECISBTwHXAHkA6tEZJGqbvIp\n9giwUFWni0hf4F0gHbgVaKqq/UWkGbBJRF5V1ZxgxWuM+aYv9xzi+SXZvL22AIDrB3Zm0re60j/Z\nJsyLBMFsQQwDtqlqNoCILABuAHwThAIJ7vNWwE6f7c1FJBqIByqAg0GM1RjjY3XuAaYv3s6Hm/cQ\nHxPFuBFpTLmwK8mtm3kdmmlAwUwQXYA8n9f5wPA6ZR4DPhCR+4HmwOXu9tdxkskuoBnwkKrur/sB\nIjIVmAqQmppan7EbE3FUlcVbC5m+ZDsrv9pPYrMYHrysJ3een06b5rFeh2c84PUg9Vhgtqr+XkRG\nAnNF5Fyc1kc10BloDXwiIh/WtkZqqeoMYAZARkaGLcZszBmoqq7hnQ27mL54O1t2H6Jzqzh+eW1f\nbjsvhea2rnNEC+a3XwCk+LxOdrf5mgyMAlDVZSISByQBtwP/VNVKYK+IfApkANkYY+rF0Ypq/ro6\njxkfZ5N/4Cg927fgd7cO5PqBnW2lNgMEN0GsAnqKSFecxDAG58TvawdwGTBbRPoAcUChu/1SnBZF\nc2AE8KcgxmpMxCgprSRzWQ6zl+ZQdKSCIamJPHpdPy7r3d7WXjDHCVqCUNUqEbkPeB/nEtZZqrpR\nRJ4AslR1EfAj4EUReQhnYHqiqqqIPAe8LCIbAQFeVtX1wYrVmEiwu6SMl/6bzSsrdnCkoppvn9OO\ney/pwXnpre3GNuOXqIZH131GRoZmZWV5HYYxjc62vYeZ8fF23vqsgBqFawd04nsXdadv54RTH2zC\nnoisVtUMf/tsBMqYMLU2r5jpi7fxwaY9xEY1YeywVO6+sBspbexSVRMYSxDGhBFV5ZMv9zF98XaW\nZReREBfNfd/uwZ3np5PUwtZfMKfHEoQxYaCquob3Pt/N80u2s3HnQTokNOX/jO7D2OGptLBLVc0Z\nsn85xoSwsspqXl+dz4yPs9mxv5Ru7Zrz9M0DuGFwZ5sjyZw1SxDGhKCDZZXMW57LrP/msO9wOQNT\nEvn56D5c2beDXapq6o0lCGNCyN6DZbz06VfMX76Dw+VVXNSrHfdc3I2R3YK4BoMqlBbBgVw4vBui\nYiEmHmKaOY9Y929MPETHQxO7yS5cWIIwJgR8te8IMz7ezhurC6iqqWF0/07cc3F3zu1ST7Oqlh+G\n4lwnCfj7W3E48PeKjvdJGs2OJZPY2ufNnb+x7l+/2/wkn9pt0TYvVEOxBGFMI7Yhv4Tnl2zn3c93\nERPVhFszkpl6UTfS2jY/vTeqqoCSvDon/5xjz0uLji8f0xxap0FiGnS90PnbOg1adoKaaqg8ApVH\nocL9W3nUz7ZS93EUKkqhdP+xbRWlx8pwmvdiNYkOIPn47q+7zWdf3W2xzawV5MMShDGNjKry6bYi\nnl+ynf9u20fLptHce3F3Jn4rnfYt4/wfVFPjdP8ccE/8dVsBh3aC1hwr3yQGWiVD63Toc92xBJCY\n7vxt1hYa4u5qVagqq5NYfBLM14nkyLFEU3mibaVw9MA3t1VXnH5c0XF1WjE+iea4ls4ZJqSo2Ib5\n73uWLEEY00hU1yjvb9zN9MXb2VBQQruWTZl2dW9uH55KQtNo5+RXsMl/F1BxHlSX+7ybOL/2W6dB\n+gXHWgO1fxM6Q5NGcJWTyLGTbrM2wfmM6iqfpOLbqqmbkPxsOy4hlUJZMRzcefy2iiOcditIouok\nH98k0vwkCclf11tzaNEOEut/yQNLEMZ4rLyqmjfXFDBnyWaq9+eQkVDCo0OqGdSihOhdefCymwTK\n66yZFd/aOdl36AfnjPZpAaRDYgpE241xAERFQ1QCxAVpahFVqCo/Pvl83frxl5BOse3gzuO3VZTW\nSf5+9LsJbp1d71WzBGFMQ6muhJL8r3/5l+/7ih3bN1NWuJ3La/YwVg5CU6AcZ93F6Phjv/jTRron\nfp9WQLBOeOb0iEBMnPMgSK2gmupvJg3flk7zpKB8rCUIY+qLKhze438Q+EAuHCwArf66eBRNiK1J\nojy+M9XJo9C0cxDfJNC8XUj0U5sG0CQKmrZwHg3IEoQxp+Nosf9B4OJcKN7hDLj6atHROdmnjqA4\nrjP/2RXHW7kxfFWVxKB+fZl6SS8GJCd6UhVjTsUShDG+Ko86J3rfVoBvEigrOb58XCvnF3+7c6Dn\nlU43UG0rIDEFYuLZuLOE55dk807WTqKbNOHmoV14/MJudGvXsL8GjTldliBMZKmucrp6/N4UluN0\nEfmKjjvW5ZMy/JtXA8X7//WvqizP3s/0JRv4+ItCWjSN5u4Lu3HXBV3pkHCCS1WNaWQsQZjwogpH\nCuuMA+QcPw5QU3WsvERBqy7Oyb7nFcfuA6hNAi06nNY4QE2N8sGmPUxfsp11ecUktYjlx1edw7gR\nabSKj6nv2hoTVJYgTOgpO+h/ELh2HKCy9Pjyzds7J/vk86D1LcdO/q3TIaELRJ39ibuiqoa/rS3g\n+SXbyS48QmqbZjx547ncMjSZuJhGcL+BMWfAEoRpnEryYe8WKM75ZhI4euD4sk0TnJN+2x7Q4/Lj\nu4ASU52bioLkcHkVC1buYOYnX7H7YBl9OyXw57GDGX1uR6KjbLoGE9osQRjvVVfC7vWQtxLyVjh/\nDxYc2x/V1DnRt06DLkP9jAO0bvDLQYsOlzN7aQ5zluZwsKyKEd3a8NtbBnBRz6TgzapqTAOzBGEa\n3pEiyPdJBgVroOqos69VKqSOdAaEO/Z3xwE6NprJ0/L2l/LiJ9kszMqjvKqGK/t24J6LuzM4tbXX\noRlT7yxBmOCqqYF9W48lg7wVULTN2dckBjoNhIy7IGWY80jo7G28J7B510FeWLKdv6/fRROBmwZ3\nYepF3enR3i5VNeHLEoSpX+WHoGC1T3fRKih37x1oluS0DAaPd/52HuRMNtZIqSqrcg4wffE2Ptpa\nSLPYKCadn87kC7vSqVXjjduY+mIJwpw5VWfQ+OtksAL2bHSnlRZo3wfOvclJBinDoU23kJg6oqZG\n+feWvUxfvI01O4pp0zyWH13Ri/Ej00hsZovVmMhhCcIErqocdq07lgzyVh67sSy2BSRnwEU/drqK\numSc8Cayxqqyuoa31+7khSXb+XLvYbokxvP49f34bkYK8bF2qaqJPJYgzIkd3nv8lUU7Pzs27XDr\ndOh2iTt2MBza920c6wucgdKKKhaszGPmJ9nsLCmjd8eW/Om2QVwzoBMxdqmqiWBBTRAiMgp4BogC\nZqrqU3X2pwJzgES3zDRVfdfdNwB4AUgAaoDzVLXOTGim3tRUw97Nxw8mH/jK2RcVC50Hw/CpTjJI\nHgYtO3gbbz3Yf6SCOUtzmLMsh+LSSoalt+F/b+rPJee0s0tVjSGICUJEooDngCuAfGCViCxS1U0+\nxR4BFqrqdBHpC7wLpItINDAPGK+q60SkLVAZrFgjUlkJ5GcdSwb5WVBxyNnXvD2kDofzJjsJodPA\nsFp8pqD4KC9+nM1rq/I4WlnN5X06cO8l3RiaFqS5/I0JUadMECJyPzBPVQ+cqmwdw4Btqprtvs8C\n4AacpVBqKU4LAaAVsNN9fiWwXlXXAahqnRXVzWlRhf3Zx3cX7d0EKEgTZ0Wygbe5g8nDnJvPwvAX\n9Bd7DvH8ku0sWuv8M7t+UGfuubg7vTq09DgyYxqnQFoQHXB+/a8BZgHvq2ogC7B2AfJ8XucDw+uU\neQz4wE1CzYHL3e29ABWR94F2wAJVfbruB4jIVGAqQGpq/a/HGrIqj8LOtcd3F5Xuc/Y1TXDmJOp7\ngzuYPDTsVyZbnbuf6Yu38+HmvcTHRDF+ZBpTLuxGl0S7VNWYkzllglDVR0TkFzi/6icBz4rIQuAl\nVd1+lp8/Fpitqr8XkZHAXBE5143rAuA8oBT4t4isVtV/14ltBjADICMj4zRXDQ8jB3cdnwx2rYMa\nt0euTXdnnYLaweR2vRvNXcnBpKp8tHUv0xdvZ1XOARKbxfCDy3ty58h0Wje3S1WNCURAYxCqqiKy\nG9gNVAGtgddF5F+q+pMTHFYApPi8Tna3+ZoMjHI/Y5mIxAFJOK2Nj1V1H4CIvAsMAf5NpKuugj2f\nH99dVLLD2RcdB52HwMjvH+suCtJatY1VVXUN/1i/i+eXbGfL7kN0bhXHL6/ty5hhKTSLtYv2jDkd\ngYxBPAhMAPYBM4Efq2qliDQBvgROlCBWAT1FpCtOYhgD3F6nzA7gMmC2iPQB4oBC4H3gJyLSDKgA\nLgb+eJp1Cw9HDzh3I9fee1CwxlmoHKBlJycRjLj32NxF0ZH56/hoRTULs/KY8XE2BcVH6dm+Bb+/\ndSDXD+psl6oac4YC+UnVBviOqub6blTVGhG59kQHqWqViNyHc7KPAmap6kYReQLIUtVFwI+AF0Xk\nIZwB64nu+MYBEfkDTpJR4F1VfedMKhhSVJ15inxvRCvc4uyTKCcBDB53rLuoVXJYDiafjuLSCjKX\n5TJ7aQ77j1QwJDWRx6/vx6W929OkSWT/tzHmbMmpxptFZASwUVUPua8TgD6quqIB4gtYRkaGZmVl\neR3G6akohZ1rjh8/qF3rIC7xWDdRynDoMgRim3sbbyOyq+QoL33yFa+s3EFpRTWX9m7PPRd357z0\n1nYPgzGnwR3fzfC3L5AWxHSc/v9ah/1sM4EoyT8+GezecGz5y6Re0PuaY/MWte0ZEYPJp2vb3kO8\nsCSbv60toEbhugGd+N7F3enTKbyvxDLGC4EkCPG9rNXtWrLRvlM52SI4Mc2cy0u/9aB7Z/J50Mxu\n0jqZz3YcYPri7XywaQ9No5tw+7BUplzYjZQ2wVstzphIF8iJPltEHsBpNQD8D5AdvJBC1EkXwUmB\n1BHHuow6nFsv6yCHO1VlyReFPL9kO8uz99MqPob7L+3Bneenk9QifO7sNqaxCiRB3AP8GWdaDMW5\n1HRqMINq9E66CE60uwjOJCcZJA+DVl28jTcEffxFIb95bwubdx2kY0Icj1zThzHDUmnR1BqvxjSU\nQG6U24tziWrkKj8MBb7zFq1y5jICaNbWXQRnnLsIzuBGvQhOKPhoy17uzswipU0znr5lADcO6kJs\ntI3HGNPQArkPIg7nhrZ+OPcpAKCqdwUxLu+oQvGOOovgfO4uggO06wP9Qm8RnFCxPLuIe+atpnen\nlrxy9wgS4qwrzhivBNJenwtsAa4CngDuADYHM6gGVVXhZxGc3c6+2BbOYPKFD7uDyaG3CE4oWZ9f\nzJQ5Tssh867hlhyM8VggCaKHqt4qIjeo6hwReQX4JNiBNZiCLHj5aud5Yhp0vej4RXCirM+7IWzd\nfYgJs1bSunkM8yYPp43Nl8F3qPYAABYkSURBVGSM5wI5+9Wuw1DsTqS3G2gfvJAaWOfB8N25TlJo\n2dHraCJSbtERxr20gtioJsyfPIKOreJOfZAxJugCSRAzRKQ1zlVMi4AWwC+CGlVDiomHvtd7HUXE\n2lVylDtmrqCquoaF3xtJalu7r8GYxuKkCcKdkO+gu1jQx0C3BonKRISiw+WMm7mC4tJKXr17BD1t\n4R5jGpWTXjuoqjWceLZWY87YwbJKJsxaSf6Bo7x0Zwb9k1t5HZIxpo5ALi7/UEQeFpEUEWlT+wh6\nZCZslVZUcdfLq/hizyFeGD+U4d3aeh2SMcaPQMYgbnP/ft9nm2LdTeYMlFdV8725q1mz4wDP3j6E\nS84Jn+sdjAk3gdxJ3bUhAjHhr6q6hgdfXcsnX+7j6VsGMLp/J69DMsacRCB3Uk/wt11VM+s/HBOu\namqUn76xgX9u3M0vr+3LdzNSTn2QMcZTgXQxnefzPA5nidA1gCUIExBV5fG/b+SNNfn88Ipe3HWB\nNUqNCQWBdDHd7/taRBKBBUGLyISd33/wBXOW5XL3hV25/9IeXodjjAnQmUyReQSwn4AmIC8s2c6z\nH21j7LAUfj66jy0HakwICWQM4u84Vy2Bk1D6AguDGZQJD/NX5PKb97Zw7YBOPHljf0sOxoSYQMYg\nfufzvArIVdX8IMVjwsTbawt45G+fc1nv9vzxtkFENbHkYEyoCSRB7AB2qWoZgIjEi0i6quYENTIT\nsv61aQ8/XLiOEV3b8twdQ4iJssV+jAlFgfyf+1egxud1tbvNmG/4dNs+vv/KGs7t0ooX78wgLibK\n65CMMWcokAQRraoVtS/c5zZZv/mGNTsOcHdmFl3bNmfOpPNs/WhjQlwgCaJQRL6eD1tEbgD2BS8k\nE4o27TzIxFkrad+yKXMnDyOxmf2GMCbUBfIT7x5gvog8677OB/zeXW0iU3bhYSbMWkHzptHMmzKc\n9gm24I8x4eCULQhV3a6qI3Aub+2rquer6rZA3lxERonIVhHZJiLT/OxPFZGPROQzEVkvIqP97D8s\nIg8HWiHTsAqKjzJu5gpUYd6U4SS3tgV/jAkXp0wQIvJrEUlU1cOqelhEWovIkwEcFwU8B1yNk1zG\nikjfOsUeARaq6mBgDPD/6uz/A/BeIBUxDa/wkLPgz6HyKjInD6N7uxZeh2SMqUeBjEFcrarFtS/c\n1eVGn6R8rWHANlXNdge2FwA31CmjQIL7vBWws3aHiNwIfAVsDOCzTAMrKa1k/Esr2F1SxuxJ59Gv\nsy34Y0y4CSRBRIlI09oXIhIPND1J+VpdgDyf1/nuNl+PAeNEJB94F7jf/YwWwE+Bx0/2ASIyVUSy\nRCSrsLAwgJBMfThSXsXE2SvJLjzCixMyGJpm60cZE44CSRDzgX+LyGQRmQL8C5hTT58/Fpitqsk4\nrZK57jrYjwF/VNXDJztYVWeoaoaqZrRr166eQjInU1ZZzd2ZWazPL+Evtw/mgp5JXodkjAmSQGZz\n/a2IrAMux+kSeh9IC+C9CwDfSf+T3W2+JgOj3M9ZJiJxQBIwHLhFRJ4GEoEaESlT1WcxnqmsruG+\nVz5j6fYi/vDdgVzVr6PXIRljgijQO5n24CSHW3HGBd4I4JhVQE8R6YqTGMYAt9cpswNnfYnZItIH\nZ72JQlW9sLaAiDwGHLbk4K2aGuXhv67jw817+NUN/fjOkGSvQzLGBNkJE4SI9MLpAhqLc2Pca4Co\n6rcDeWNVrRKR+3BaHFHALFXdKCJPAFmqugj4EfCiiDyEk4Amqqqe+F2NF1SVX7z9OW+v3clPRp3D\n+JHpXodkjGkAcqLzsYjUAJ8Ak2vvexCRbFXt1oDxBSwjI0OzsrK8DiPsqCpP/XMLLyzJ5t5LuvPT\nUb29DskYU49EZLWqZvjbd7JB6u8Au4CPRORFEbkMsDmbI8z/W7ydF5ZkM35EGj+56hyvwzHGNKAT\nJghV/ZuqjgF6Ax8BPwDai8h0EbmyoQI03pmzNIf/+/5WvjO4C49f388W/DEmwgQy1cYRVX1FVa/D\nuRLpM5x7FEwYe311Po8u2siVfTvw9C0DaGIL/hgTcU5rJRdVPeDee3BZsAIy3vvn57v4yevruKBH\nEn+5fTDRtuCPMRHJ/s83x/n4i0Luf/UzBqUkMmPCUJpG24I/xkQqSxDma6ty9jN1bhY92rfk5UnD\naBZrC/4YE8ksQRgAPi8o4a6XV9E5MZ65k4fRKj7G65CMMR6zBGHYtvcQE2atJCE+hnmTh5PUIpC5\nGI0x4c4SRITL21/KuJkraSLCvCnD6ZwY73VIxphGwhJEBNtzsIw7Zq7gaGU186YMo2tSc69DMsY0\nIpYgItSBIxWMf2kFRYfLmT3pPHp3TDj1QcaYiGKXqUSgQ2WV3PnySnKKSpkzaRiDU1t7HZIxphGy\nFkSEKausZvKcLDbtPMj0O4Ywsntbr0MyxjRS1oKIIBVVNdw7bzWrcvbzzJjBXNang9chGWMaMWtB\nRIjqGuWh19by0dZCfn1Tf64f2NnrkIwxjZwliAigqvz8zQ28s2EX/2d0H8YOS/U6JGNMCLAEEeZU\nlSff2cxrWXk8cGkP7r6oUa73ZIxphCxBhLln/v0lL/33Kyaen85DV/TyOhxjTAixBBHGZn6SzZ8+\n/JJbhybzy2v72oI/xpjTYgkiTL22agdPvrOZ0f078tTNtuCPMeb0WYIIQ/9Yv5Npb27g4l7t+NNt\ng4my5GCMOQOWIMLMR1v28oMFazkvrQ3PjxtKbLR9xcaYM2NnjzCyPLuIe+atpk+nBGZOzCA+1laD\nM8acOUsQYWJdXjGTZ68itU0z5tw1jIQ4W/DHGHN2LEGEga27D3Hnyytp0yKWeVOG06Z5rNchGWPC\ngCWIEJdbdIRxL62gaXQT5k8eQYeEOK9DMsaECUsQIWxXyVHumLmCquoa5k0eTmrbZl6HZIwJI0FN\nECIySkS2isg2EZnmZ3+qiHwkIp+JyHoRGe1uv0JEVovIBvfvpcGMMxQVHS5n3MwVlJRWknnXcHp2\naOl1SMaYMBO06b5FJAp4DrgCyAdWicgiVd3kU+wRYKGqTheRvsC7QDqwD7hOVXeKyLnA+0CXYMUa\nakqOVjJh1koKio+Seddw+ie38jokY0wYCmYLYhiwTVWzVbUCWADcUKeMArVrXbYCdgKo6mequtPd\nvhGIF5GmQYw1ZJRWVDF59iq+2HOI58cNZVjXNl6HZIwJU8FMEF2APJ/X+XyzFfAYME5E8nFaD/f7\neZ+bgTWqWl53h4hMFZEsEckqLCysn6gbsfKqar43dzVrdhzgmTGDueSc9l6HZIwJY14PUo8FZqtq\nMjAamCsiX8ckIv2A3wLf83ewqs5Q1QxVzWjXrl2DBOyVquoaHnj1Mz75ch+/vXkAo/t38jokY0yY\nC2aCKABSfF4nu9t8TQYWAqjqMiAOSAIQkWTgLWCCqm4PYpyNXk2N8pM31vP+xj08el1fbs1IOfVB\nxhhzloKZIFYBPUWkq4jEAmOARXXK7AAuAxCRPjgJolBEEoF3gGmq+mkQY2z0VJXH/76RN9cU8MMr\nejHpW129DskYEyGCliBUtQq4D+cKpM04VyttFJEnROR6t9iPgLtFZB3wKjBRVdU9rgfwSxFZ6z4i\nssP99x98wZxluUy9qBv3X9rD63CMMRFEnPNx6MvIyNCsrCyvw6hXzy/ZzlPvbWHssFR+fdO5tuCP\nMabeichqVc3wt8/rQWpzAvOW5/LUe1u4bmBnnrzRkoMxpuFZgmiE3l5bwC/e/pzLerfnD98daAv+\nGGM8YQmikflg425+uHAdI7q25bk7hhATZV+RMcYbdvZpRD7dto/7XvmM/l1a8eKdGcTF2II/xhjv\nWIJoJFbnHuDuzCy6tWvO7Enn0aJp0KbJMsaYgFiCaAQ27TzIpJdX0r5lUzInDyOxmS34Y4zxniUI\nj2UXHmbCrBU0bxrNvCnDad/SFvwxxjQOliA8VFB8lHEzV6AK86YMJ7m1LfhjjGk8rKPbI4WHnAV/\nDpVXsWDqCLq3a+F1SMYYcxxrQXigpLSS8S+tYHdJGbMnnUe/zrbgjzGm8bEE0cAOl1dx58sryS48\nwosTMhiaZgv+GGMaJ+tiakBlldVMzcxiQ0EJ0+8YwgU9k7wOyRhjTshaEA2ksrqG+15Zw9LtRfzu\n1gFc2a+j1yEZY8xJWYJoADU1ysN/XceHm/fyqxv6cdPgZK9DMsaYU7IEEWSqyi/e/py31+7kp6N6\nM35kutchGWNMQCxBBJGq8tR7W5i/Ygf/c0l37r2ku9chGWNMwCxBBNFzH23jhY+zmTAyjR9fdY7X\n4RhjzGmxBBEksz/9it998AXfGdyFx67rZwv+GGNCjiWIIHh9dT6P/X0TV/XrwNO3DKCJLfhjjAlB\nliDq2XsbdvGT19dxYc8k/jx2MNG24I8xJkTZ2aseLfmikAcWfMbg1Na8MH4oTaNtwR9jTOiyBFFP\nVuXs53tzs+jZviWzJp5Hs1i7Sd0YE9osQdSDzwtKuOvlVXROjCdz8jBaxcd4HZIxxpw1SxBnadve\nQ0yYtZKE+BjmTR5OUoumXodkjDH1whLEWcjbX8odM1cQ1USYP2U4nRPjvQ7JGGPqjSWIM7TnYBl3\nzFxBWWUNcycPIz2pudchGWNMvQpqghCRUSKyVUS2icg0P/tTReQjEflMRNaLyGiffT9zj9sqIlcF\nM87TdeBIBeNfWkHR4XLm3DWM3h0TvA7JGGPqXdAutRGRKOA54AogH1glIotUdZNPsUeAhao6XUT6\nAu8C6e7zMUA/oDPwoYj0UtXqYMUbqENlldz58kpyi0qZPWkYg1ISvQ7JGGOCIpgtiGHANlXNVtUK\nYAFwQ50yCtT+/G4F7HSf3wAsUNVyVf0K2Oa+n6eOVlQzeU4Wm3YeZPq4IYzs3tbrkIwxJmiCmSC6\nAHk+r/Pdbb4eA8aJSD5O6+H+0zgWEZkqIlkiklVYWFhfcftVUVXDvfNXsypnP3+8bRCX9u4Q1M8z\nxhiveT1IPRaYrarJwGhgrogEHJOqzlDVDFXNaNeuXdCCrK5RHnptLYu3FvLrm/pz3cDOQfssY4xp\nLIJ5u28BkOLzOtnd5msyMApAVZeJSByQFOCxDaKmRvnZm+t5Z8MuHrmmD2OHpXoRhjHGNLhgtiBW\nAT1FpKuIxOIMOi+qU2YHcBmAiPQB4oBCt9wYEWkqIl2BnsDKIMbql6ry5DubWZiVzwOX9WTKhd0a\nOgRjjPFM0FoQqlolIvcB7wNRwCxV3SgiTwBZqroI+BHwoog8hDNgPVFVFdgoIguBTUAV8H0vrmD6\n04dfMuvTr5j0rXQeurxnQ3+8McZ4SpzzcejLyMjQrKysenu/mZ9k8+Q7m7l1aDK/vdnWdDDGhCcR\nWa2qGf72eT1I3SgtWLmDJ9/ZzDX9O/GUJQdjTISyBFHH39ft5GdvbeCSc9rxx9sGEWXJwRgToSxB\n+PjPlj089Npazktrw/Q7hhIbbf95jDGRy86AruXZRdw7bw19OiXw0sQM4mNtNThjTGSzBAGszStm\n8uxVpLZpxpy7htEyzhb8McaYiE8QX+w5xMSXV9K2RVPmTRlOm+axXodkjDGNQsQniFbxMfTv0or5\nU4bTISHO63CMMabRCOZUGyGhQ0IccycP9zoMY4xpdCK+BWGMMcY/SxDGGGP8sgRhjDHGL0sQxhhj\n/LIEYYwxxi9LEMYYY/yyBGGMMcYvSxDGGGP8CpsFg0SkEMg9i7dIAvbVUzheCpd6gNWlMQqXeoDV\npVaaqrbztyNsEsTZEpGsE62qFErCpR5gdWmMwqUeYHUJhHUxGWOM8csShDHGGL8sQRwzw+sA6km4\n1AOsLo1RuNQDrC6nZGMQxhhj/LIWhDHGGL8sQRhjjPErohKEiIwSka0isk1EpvnZ31REXnP3rxCR\n9IaPMjAB1GWiiBSKyFr3McWLOE9FRGaJyF4R+fwE+0VE/uzWc72IDGnoGAMVQF0uEZESn+/klw0d\nYyBEJEVEPhKRTSKyUUQe9FMmJL6XAOsSKt9LnIisFJF1bl0e91Omfs9hqhoRDyAK2A50A2KBdUDf\nOmX+B3jefT4GeM3ruM+iLhOBZ72ONYC6XAQMAT4/wf7RwHuAACOAFV7HfBZ1uQT4h9dxBlCPTsAQ\n93lL4As//75C4nsJsC6h8r0I0MJ9HgOsAEbUKVOv57BIakEMA7aparaqVgALgBvqlLkBmOM+fx24\nTESkAWMMVCB1CQmq+jGw/yRFbgAy1bEcSBSRTg0T3ekJoC4hQVV3qeoa9/khYDPQpU6xkPheAqxL\nSHD/Wx92X8a4j7pXGdXrOSySEkQXIM/ndT7f/IfydRlVrQJKgLYNEt3pCaQuADe7zf/XRSSlYUKr\nd4HWNVSMdLsI3hORfl4HcypuF8VgnF+rvkLuezlJXSBEvhcRiRKRtcBe4F+qesLvpT7OYZGUICLN\n34F0VR0A/ItjvyqMd9bgzHszEPgL8DeP4zkpEWkBvAH8QFUPeh3P2ThFXULme1HValUdBCQDw0Tk\n3GB+XiQliALA91d0srvNbxkRiQZaAUUNEt3pOWVdVLVIVcvdlzOBoQ0UW30L5HsLCap6sLaLQFXf\nBWJEJMnjsPwSkRicE+p8VX3TT5GQ+V5OVZdQ+l5qqWox8BEwqs6uej2HRVKCWAX0FJGuIhKLM4Cz\nqE6ZRcCd7vNbgP+oO9rTyJyyLnX6g6/H6XsNRYuACe5VMyOAElXd5XVQZ0JEOtb2B4vIMJz//xrd\nDxA3xpeAzar6hxMUC4nvJZC6hND30k5EEt3n8cAVwJY6xer1HBZ9pgeGGlWtEpH7gPdxrgKapaob\nReQJIEtVF+H8Q5orIttwBhvHeBfxiQVYlwdE5HqgCqcuEz0L+CRE5FWcq0iSRCQfeBRn8A1VfR54\nF+eKmW1AKTDJm0hPLYC63ALcKyJVwFFgTCP9AfItYDywwe3vBvg5kAoh970EUpdQ+V46AXNEJAon\niS1U1X8E8xxmU20YY4zxK5K6mIwxxpwGSxDGGGP8sgRhjDHGL0sQxhhj/LIEYYwxxi9LEMacBhGp\n9pn1c634mUn3LN47/UQzwRrjhYi5D8KYenLUnerAmLBnLQhj6oGI5IjI0yKywZ2zv4e7PV1E/uNO\nmvhvEUl1t3cQkbfcCeLWicj57ltFiciL7nz/H7h3zBrjCUsQxpye+DpdTLf57CtR1f7As8Cf3G1/\nAea4kybOB/7sbv8zsMSdIG4IsNHd3hN4TlX7AcXAzUGujzEnZHdSG3MaROSwqrbwsz0HuFRVs93J\n4XaralsR2Qd0UtVKd/suVU0SkUIg2WdCxdrpqP+lqj3d1z8FYlT1yeDXzJhvshaEMfVHT/D8dJT7\nPK/GxgmNhyxBGFN/bvP5u8x9vpRjE6bdAXziPv83cC98vQhMq4YK0phA2a8TY05PvM+soAD/VNXa\nS11bi8h6nFbAWHfb/cDLIvJjoJBjs54+CMwQkck4LYV7gUY3XbaJbDYGYUw9cMcgMlR1n9exGFNf\nrIvJGGOMX9aCMMYY45e1IIwxxvhlCcIYY4xfliCMMcb4ZQnCGGOMX5YgjDHG+PX/AZBo+c/fM1cH\nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU9b3/8dcnk30jZGFNIAug4sJi\nZFHZvC5YW21/agXFBYGolWtvvZv93d5rr/b3u9reX29boVUE3BGt3eiiVis7soTFBRANIUAQJAtr\nIGSZz++POYEhTCAhM5nt83w85sGcbeZzOva88z3fc75HVBVjjDGmtZhgF2CMMSY0WUAYY4zxyQLC\nGGOMTxYQxhhjfLKAMMYY45MFhDHGGJ8sIIzpBBHJFxEVkdh2rHufiKzo7OcY01UsIEzUEJEKEWkQ\nkexW8zc6B+f84FRmTGiygDDRZgcwuWVCRC4FkoNXjjGhywLCRJtXgHu8pu8FXvZeQUS6icjLIlIl\nIjtF5AciEuMsc4nIf4tItYiUAzf52HaeiOwVkT0i8iMRcXW0SBHpIyKLRKRWRMpEZIbXshEiUioi\nh0XkKxH5qTM/UUReFZEaETkoIutEpGdHv9uYFhYQJtqsBtJF5CLnwD0JeLXVOs8A3YBCYByeQJnq\nLJsBfB0YBhQDt7Xa9kWgCRjgrHM9MP086lwIVAJ9nO/4vyJyjbPs58DPVTUdKALedObf69SdB2QB\nDwLHz+O7jQEsIEx0amlFXAdsBfa0LPAKje+r6hFVrQD+H3C3s8q3gZ+p6m5VrQX+y2vbnsDXgH9Q\n1TpV3Q/8j/N57SYiecBVwL+qar2qbgLmcqrl0wgMEJFsVT2qqqu95mcBA1S1WVXXq+rhjny3Md4s\nIEw0egW4E7iPVqeXgGwgDtjpNW8n0Nd53wfY3WpZi/7OtnudUzwHgeeAHh2srw9Qq6pH2qhhGjAI\n+Mw5jfR1r/16F1goIl+KyI9FJK6D323MSRYQJuqo6k48ndVfA37banE1nr/E+3vN68epVsZePKdw\nvJe12A2cALJVNcN5pavqxR0s8UsgU0TSfNWgql+o6mQ8wfM08JaIpKhqo6r+p6oOBq7EcyrsHow5\nTxYQJlpNA65R1TrvmarajOec/v8RkTQR6Q88yql+ijeBR0QkV0S6A495bbsX+Cvw/0QkXURiRKRI\nRMZ1pDBV3Q2sAv7L6Xi+zKn3VQARmSIiOarqBg46m7lFZIKIXOqcJjuMJ+jcHfluY7xZQJiopKrb\nVbW0jcV/D9QB5cAKYAEw31n2PJ7TOB8BGzizBXIPEA9sAQ4AbwG9z6PEyUA+ntbE74DHVfV9Z9lE\nYLOIHMXTYT1JVY8DvZzvO4ynb2UpntNOxpwXsQcGGWOM8cVaEMYYY3yygDDGGOOTBYQxxhifLCCM\nMcb4FDFDC2dnZ2t+fn6wyzDGmLCyfv36alXN8bUsYgIiPz+f0tK2rlo0xhjji4jsbGuZnWIyxhjj\nkwWEMcYYnywgjDHG+BQxfRC+NDY2UllZSX19fbBL6TKJiYnk5uYSF2eDeBpjOieiA6KyspK0tDTy\n8/MRkWCXE3CqSk1NDZWVlRQUFAS7HGNMmIvoU0z19fVkZWVFRTgAiAhZWVlR1WIyxgRORAcEEDXh\n0CLa9tcYEzgRHxDnoqrsPXSchqbmYJdijDEhJeoDoqHJTW1dA+XVdTQ0+ffZKjU1NQwdOpShQ4fS\nq1cv+vbte3K6oaGhXZ8xdepUtm3b5te6jDGmPSK6k7o9EuJcFGSnsKOqjvLqoxRmpxIf65/czMrK\nYtOmTQD88Ic/JDU1lX/6p386bR1VRVWJifH9nS+88IJfajHGmI6K+hYEQHJ8LAXZKTQ3Kzuq62j0\nc0uitbKyMgYPHsxdd93FxRdfzN69eykpKaG4uJiLL76YJ5544uS6V199NZs2baKpqYmMjAwee+wx\nhgwZwujRo9m/f39A6zTGRLeoaUH85x83s+XLw2ddx63K8cZmYhAS412cq7t3cJ90Hv9GR59H7/HZ\nZ5/x8ssvU1xcDMBTTz1FZmYmTU1NTJgwgdtuu43Bgwefts2hQ4cYN24cTz31FI8++ijz58/nscce\n8/XxxhjTadaC8BIjQlKcCzdKfUMzgXwYa1FR0clwAHj99dcZPnw4w4cPZ+vWrWzZsuWMbZKSkrjx\nxhsBuPzyy6moqAhghcaYaBc1LYiO/KVfd6KJHdV1xLliKMxJIc7l/xxNSUk5+f6LL77g5z//OWvX\nriUjI4MpU6b4vJchPj7+5HuXy0VTU5Pf6zLGmBbWgvAhJSGW/OwUGpvdnj6J5sD2SRw+fJi0tDTS\n09PZu3cv7777bkC/zxhj2iNqWhAdlZoQS35WMhU1x9hRXUdhdgqxAWhJAAwfPpzBgwdz4YUX0r9/\nf6666qqAfI8xxnSEqAbyTHvXKS4u1tYPDNq6dSsXXXRRpz73aH0jFTXHiI+NCWhI+JM/9tsYEx1E\nZL2qFvtaFtCjnYhMFJFtIlImImdcbiMi94lIlYhscl7TvZbdKyJfOK97A1nn2aQmxtE/K5kTTZ7T\nTU0BPt1kjDGhImCnmETEBcwGrgMqgXUiskhVW1+e84aqzmy1bSbwOFAMKLDe2fZAoOo9m7TEuNNO\nNxWESUvCGGM6I5BHuRFAmaqWq2oDsBC4pZ3b3gC8p6q1Tii8B0wMUJ3tkua0JOpbWhJua0kYYyJb\nIAOiL7Dba7rSmdfarSLysYi8JSJ5HdlWREpEpFRESquqqvxVd5vSE+Pon+kJiYrqOpotJIwxESzY\n50n+COSr6mV4WgkvdWRjVZ2jqsWqWpyTkxOQAltLT/KExPFGNzuqj1lIGGMiViADYg+Q5zWd68w7\nSVVrVPWEMzkXuLy92wZTelIc/TKTOd7Q7IREZFwJZowx3gIZEOuAgSJSICLxwCRgkfcKItLba/Jm\nYKvz/l3gehHpLiLdgeudeSGjW1Ic/bKSON7Q7JxuOjMk/DHcN8D8+fPZt2+fP8s3xphzCthVTKra\nJCIz8RzYXcB8Vd0sIk8Apaq6CHhERG4GmoBa4D5n21oReRJPyAA8oaq1gar1fHVLiqdfJuyqPU5F\ndR352Sm4Yk4N8dee4b7bY/78+QwfPpxevXr5rXZjjDmXgN5Jrap/Af7Sat5/eL3/PvD9NradD8wP\nZH3+0C05njxgd+0xKmrqyM86PSTa8tJLLzF79mwaGhq48sormTVrFm63m6lTp7Jp0yZUlZKSEnr2\n7MmmTZu44447SEpKYu3ataeNyWSMMYESPUNtvP0Y7PvEv5/Z61K48Skykj0H7N21x9jphETMWULi\n008/5Xe/+x2rVq0iNjaWkpISFi5cSFFREdXV1XzyiafOgwcPkpGRwTPPPMOsWbMYOnSof+s3xpiz\niJ6ACLCM5HiU01sSbYXE+++/z7p1604O9338+HHy8vK44YYb2LZtG4888gg33XQT119/fRfugTHG\nnC56AuLGpwL+Fd2T40Fh94Gzh4Sqcv/99/Pkk0+esezjjz/m7bffZvbs2fzmN79hzpw5Aa/bGGN8\nCfZ9EBGne0o8ud2TOXqiiZ21x3D7uLrp2muv5c0336S6uhrwXO20a9cuqqqqUFVuv/12nnjiCTZs\n2ABAWloaR44c6dL9MMaY6GlBdKHMlHhAqTxwnJ21x+iflXza8ksvvZTHH3+ca6+9FrfbTVxcHM8+\n+ywul4tp06ahqogITz/9NABTp05l+vTp1kltjOlSNtx3ANXUnWDPgeOkJ8bRLyuZGDn31U3+EOz9\nNsaEj6AN9x3tslIS6JuRxOH6RnbVHMMdIWFsjIkOFhABlpWaQB8nJHbXWkgYY8JHxAdEKJxCy3ZC\n4tDxwIdEKOyvMSYyRHRAJCYmUlNTExIHzezUBHp3OxUSgahJVampqSExMdHvn22MiT4RfRVTbm4u\nlZWVdMWzItqrvr6Jr443UhnvontyHOLnjuvExERyc3P9+pnGmOgU0QERFxdHQUFBsMs4w3NLt/Nf\niz7jlqF9+Om3h7Zr7CZjjOlqER0QoeqBcUU0q/Ljd7YRI8J/3z7EQsIYE3IsIILkO+MH4HYr//3X\nz4kR4ce3XWYhYYwJKRYQQTTzmoE0u+F/3v+cGIGnb73srKPAGmNMV7KACLLvXjuQZlV+8bcvcMUI\n//dbl1pIGGNCggVECPjetQNxu5VZi8sQEf7PNy+xkDDGBJ0FRAgQEf7x+kG4Vfnlku24YuDJWy7x\n+yWwxhjTERYQIUJE+OcbLqBZleeWlhMjwn/efLGFhDEmaCwgQoiI8NjEC3G7leeX7yBGhMe/MdhC\nwhgTFBYQIUZE+N9fu4hmN8xfuQNXjPCDmy6ykDDGdDkLiBAkIvz71y/Crcq8FZ6Q+P6NF1pIGGO6\nlAVEiBLn9JJblTnLyhGBxyZaSBhjuo4FRAgTp6O62e3puHY5HdkWEsaYrmABEeJEhCdvuQS34lwC\nKzx63SALCWNMwFlAhIGYGM/Nc6rKMx+UESPC964bFOyyjDERzgIiTMQ4w3A0u5WfO8NyPPJ3A4Nd\nljEmgllAhJGYGOGpWy+jWZWfvvc5rhjh4QkDgl2WMSZCWUCEGVeM8JPbhqAKP3l3GyKeocONMcbf\nLCDCkCvG85ChZrfnoUMuER4YVxTssowxESYmkB8uIhNFZJuIlInIY2dZ71YRUREpdqbzReS4iGxy\nXs8Gss5w5IoRfvrtIXz9st7819ufMXd5ebBLMsZEmIC1IETEBcwGrgMqgXUiskhVt7RaLw34LrCm\n1UdsV9WhgaovEsS6YvjZHUNRhR/9eSsiwrSrQ+8Z3MaY8BTIFsQIoExVy1W1AVgI3OJjvSeBp4H6\nANYSsWJdMfxs0lBuvKQXT/5pCy+u3BHskowxESKQAdEX2O01XenMO0lEhgN5qvpnH9sXiMhGEVkq\nImN8fYGIlIhIqYiUVlVV+a3wcBPniuEXk4dx/eCe/PCPW3jlw4pgl2SMiQAB7YM4GxGJAX4K/KOP\nxXuBfqo6DHgUWCAi6a1XUtU5qlqsqsU5OTmBLTjExblimHXncK69qCf//ofNvLZmZ7BLMsaEuUAG\nxB4gz2s615nXIg24BFgiIhXAKGCRiBSr6glVrQFQ1fXAdsBuHT6H+NgYZt81jGsu7MG//e5TXl+7\nK9glGWPCWCADYh0wUEQKRCQemAQsalmoqodUNVtV81U1H1gN3KyqpSKS43RyIyKFwEDALtNph4RY\nF7+aMpzxF+Tw/d9+whvrLCSMMecnYAGhqk3ATOBdYCvwpqpuFpEnROTmc2w+FvhYRDYBbwEPqmpt\noGqNNAmxLp6dcjljB+Xw2G8/4delu8+9kTHGtCKqGuwa/KK4uFhLS0uDXUZIqW9sZsbLpawoq+a/\nbxvCrZfnBrskY0yIEZH1qlrsa1nQOqlN4CXGuXj+nmKuLMrin976iN9v3HPujYwxxmEBEeES41zM\nvecKRhVk8eibm/jDJgsJY0z7WEBEgaR4F/PuK+aK/Ey+98Ym/vjRl8EuyRgTBiwgokRyfCzz77uC\n4v6Z/MMbm/jzx3uDXZIxJsRZQESRlIRY5k+9gmF5GTyycCPvfGohYYxpmwVElElNiOWFqVcwJLcb\nMxds5N3N+4JdkjEmRFlARKG0xDheun8El/TtxswFG3h/y1fBLskYE4IsIKJUWmIcL08bweDe6Tz0\n2no++MxCwhhzOguIKJaeGMfL00ZyYa90HnxlA4u37Q92ScaYEGIBEeW6JcXxyrQRDOyZygOvrGfp\n59E7bLox5nQWEIaM5HhenTaSopxUZrxcyvIvLCSMMRYQxtE9JZ7Xpo+kMDuF6S+VsrKsOtglGWOC\nzALCnJTphER+VgrTXlrHqu0WEsZEMwsIc5qs1ARemzGSvO7JTHuxlNXlNcEuyRgTJBYQ5gzZqQks\nmDGKvt2TuP/FdazdYY/iMCYaWUAYn3LSElgwYyS9uiUy9YW1lFZYSBgTbSwgTJt6pCXy+oxR9EhP\n5N75a1m/80CwSzLGdCELCHNWPdM9IZGTlsC989eycZeFhDHRwgLCnFOvbom8XjKKrNR47pm3lo92\nHwx2ScaYLmABYdqld7ckXp8xioyUOO6et4ZPKg8FuyRjTIBZQJh265PhCYn0pDimzFvDp3ssJIyJ\nZBYQpkNyuyfz+oxRpCbEMmXeGjZ/aSFhTKSygDAdlpfpCYnkOBdT5q5h697DwS7JGBMAFhDmvPTL\nSub1klEkxLq4a+4atu07EuySjDF+ZgFhzlv/rBReLxlFnEu48/nVfP6VhYQxkcQCwnRKQXYKC2aM\nwhXjCYmy/RYSxkQKCwjTaUU5qSyYMQoQJj+/hu1VR4NdkjHGDywgjF8M6JHKwpKRqCqT56ym3ELC\nmLBnAWH8ZkCPNBbMGEWzW5n8/Gp2VNcFuyRjTCdYQBi/GtQzjddmjKSx2dOS2FljIWFMuApoQIjI\nRBHZJiJlIvLYWda7VURURIq95n3f2W6biNwQyDqNf13YK51Xp42kvqmZyXNWs6vmWLBLMsach4AF\nhIi4gNnAjcBgYLKIDPaxXhrwXWCN17zBwCTgYmAi8Evn80yYGNwnndemj6SuoZnJz69md62FhDHh\nJpAtiBFAmaqWq2oDsBC4xcd6TwJPA/Ve824BFqrqCVXdAZQ5n2fCyMV9uvHa9JEcqW9k8vOrqTxg\nIWFMOAlkQPQFdntNVzrzThKR4UCeqv65o9s625eISKmIlFZVVfmnauNXl/TtxmvTR3HouCckvjx4\nPNglGWPaqV0BISJFIpLgvB8vIo+ISEZnvlhEYoCfAv94vp+hqnNUtVhVi3NycjpTjgmgS3O78eq0\nkRys84TE3kMWEsaEg/a2IH4DNIvIAGAOkAcsOMc2e5z1WuQ681qkAZcAS0SkAhgFLHI6qs+1rQkz\nQ/IyeHnaCGqONjB5zmr2Hao/90bGmKBqb0C4VbUJ+BbwjKr+M9D7HNusAwaKSIGIxOPpdF7UslBV\nD6lqtqrmq2o+sBq4WVVLnfUmiUiCiBQAA4G1HdozE3KG9evOS/dfQdWRE9z5/Gr2H7aQMCaUtTcg\nGkVkMnAv8CdnXtzZNnACZSbwLrAVeFNVN4vIEyJy8zm23Qy8CWwB3gEeVtXmdtZqQtjl/TN58f4R\n7Dtcz6TnV7P/iIWEMaFKVPXcK3kuO30Q+FBVX3f+qv+2qj4d6ALbq7i4WEtLS89v4yNfQVpP/xZk\nzmrtjlrue2HtyafU5aQlBLskY6KSiKxX1WKfy9oTEK0+rDueK48+9kdx/nLeAXGsFn5cCJkFUDAO\nCsdB/lhIyfJ/keY0q8trmPrCOvIyk1gwYxTZqRYSxnS1TgeEiCwBbgZigfXAfmClqj7qxzo75bwD\n4vhB+GghlC+BihXQcAQQ6HWpJywKx0O/0RCf4t+CDQCrtldz/4vryM/yDBuemRIf7JKMiSr+CIiN\nqjpMRKbjaT08LiIfq+pl/i72fHXqFFOL5ib4cgOUL/UERuVaaG6AmDjIG+kJjIJx0Hc4uM7aBWM6\nYGWZJyQKslN4fcYoultIGNNl/BEQnwDXAy8B/6aq6yIyIFprOAa7PvSExY6lsPdjQCE+DfKv8rQu\nCsZBj4tAxL/fHWWWfV7F9JdLGZCTyoIZI8lItpAwpiv4IyBuB/4dz2mlh0SkEPiJqt7q31LPX0AC\norVjtbBjmScsypdAbblnfkqPU62LwvGQkXeWDzFtWbJtPyUvr2dgz1QWTB9Ft2RrpRkTaH7tpA5V\nXRIQrR3c5TkdtWOp59+6/Z75mYWnwqJgLCRndm1dYWzxZ/t54JX1XNArjVenj6RbkoWEMYHkjxZE\nLvAMcJUzaznwXVWt9FuVnRSUgPCmCvu3ngoL7w7v3pedukKq35UQnxy8OsPA37Z+xYOvrqdfZjLf\nu24QN17SG1eMncIzJhD8ERDv4Rla4xVn1hTgLlW9zm9VdlLQA6K15kb4cqPnVFT5Uti9BtyN4IqH\n3BGe1kXhOOgzHFyxQS429Cz7vIof/nEz5VV1FOak8NC4Ir45rC9xLnvGlTH+5I+A2KSqQ881L5hC\nLiBaa6hzOryd/ot9n3Cqw/vqU5fU5lxoHd6OZrfy9qd7mb14O1v3HqZvRhIPjivk9uI8EuPs8SDG\n+IM/AuJvwAvA686sycBUVf07v1XZSSEfEK21dHi3XCHV0uGd2tPTb1E43nNayjq8UVU++Gw/sxaX\nsXHXQXLSEpgxpoC7RvYnJcFaX8Z0hj8Coj+ePojRgAKrgL9X1d1n3bALhV1AtHZah/cSqHOeb5FZ\ndOoKqSjv8FZVPtxew6zFZazaXkNGchxTryzgvivz7YonY85TQK5iEpF/UNWfdaoyPwr7gPDW0uHd\n0rqoWHl6h3fheE9g9BsdtR3eG3YdYPYHZfzts/2kJsQyZVR/po8psOE6jOmgQAXELlXt16nK/Cii\nAqK15kbYs+FU62L32lMd3nkjT10hFYUd3lu+PMzsJWX85ZO9xLtimDyiHyVjC+mTkRTs0owJC4EK\niN2qGjInyCM6IFo72eG9xHNaap8zbmJCOvS/6tQVUlHU4b296ii/WrKd32/cgwj8r2G5PDS+iPxs\nG0PLmLOxFkSkq6uBimWnrpA6sMMzP7XnqdZFlHR4Vx44xnNLy3mjdDdNzW6+flkfHp4wgAt6pQW7\nNGNC0nkHhIgcwdMpfcYiIElVQ+Z8RlQHRGsHdp66YW/H0jM7vAvHQ/6YiO7w3n+4nrkrdvDq6p0c\na2jmusE9mTlhAEPyOvUodWMijg21Ec1UYf+WU62LnSuh4SieDu8hp1oXEdrhfaCugRdWVfDiyh0c\nrm9izMBsZk4YwMhCe96HMWABYby1dHi3XCHVusO7cBwUjIc+wyKqw/tIfSOvrt7FvBXlVB9t4Ir8\n7nxnwgDGD8pBoqSfxhhfLCBM2xrqYOeHsGOJ1x3eeDq8868+NehgzgUR0eFd39jMwrW7eG5ZOXsP\n1XNJ33QeHj+AGy7uRYyN92SikAWEab+6aq8hzZd6dXj3OnWHd+E46JYbzCo7raHJze82VvKrJdup\nqDnGgB6pPDyhiG9c1odYG+/JRBELCHP+TnZ4L/EEx2kd3uOdZ3iHb4d3U7ObP3+yl18u3s62r47Q\nLzOZB8cVcevlfUmItfGeTOSzgDD+cbLDe4mndXFGh/d4Z0jz0RAXXjequd3K+1u/YvbiMj6qPESv\n9ERmjC1k8og8kuMjpy/GmNYsIExgNDfCnvVez/BeF/Yd3qrKirJqZn1QxpodtWSmxDPt6gLuHt2f\n9EQb78lEHgsI0zVaOrzLF3tOS7Xu8C4c7+n0DpMO79KKWmYtLmPJtirSEmO5d3Q+919dQGaKPS/b\nRA4LCBMcLR3eLZfUHqjwzE/t5fUM79Dv8P50zyFmLy7jnc37SIx1cedIz3hPPdMTg12aMZ1mAWFC\nw4GK05/hfazaMz9rgNczvMdAUvcgFtm2sv1H+OXi7fzhoy9xiXBbcS4PjSsiLzPybjA00cMCwoQe\nt9vT4d1yhVTFSmisAwT6DPUERt/LITnLc4VUUqYnOGKDf3pnV80xnl22nbdKK2lW5ZYhffjOhCIG\n9LDxnkz4sYAwoa+pwdPh3dK6qFwL7qYz14tPg+TunsBIdkLj5Hvvf73mJ6QHpM9j36F6nl9ezoI1\nu6hvambixb14eMIALunbze/fZcwZ3M1Qf8jzUjdkFZ3Xx1hAmPBz4ijUbvc8mvV4rfPvgVbTXv/W\nH2r7s2JifQRJW8HS8dZKbV0D81fs4KVVFRw50cT4C3KYOWEAxfnheW+I6SLeB/jzeTUcOfVZuVfA\n9PfPqwwLCBP5mpug/uCZwXFGqLSabj7R9mfGp7YvTJzlh2PSeWVDLfNWVlBb18DIgkxmXjOAqwdk\n23hPkai5CU4c9s8B3ieBxHRI7Oa8Mrzet3ql9/Vc8HEeghYQIjIR+DngAuaq6lOtlj8IPAw0A0eB\nElXdIiL5wFZgm7PqalV98GzfZQFhOkwVGo+10So5/9aKJnbnoKSx81gC+5uSkeRMBuT3o39uHjHJ\nbZwaC4G+lajTJQd4Xwd0Xwf69DPnxadBTOCHfQlKQIiIC/gcuA6oBNYBk1V1i9c66ap62Hl/M/Ad\nVZ3oBMSfVPWS9n6fBYTpMu5mOH7Qd3h4/es+Vsuh2q9oOlpDuvsICdLY9md2sLUSyL6VsHHyAH/w\nPA/wR8/xBR05wPt4xad2yQG+s84WEIG8vXUEUKaq5U4RC4FbgJMB0RIOjhR8P5zImNAS44KULM/r\nbKsB3fGM9/THj/Ywf/EWaqr2MTijiSlDUrm6j4vYEwe8WisHToXMwZ2ef+sPnuULztK3ktS97ZAJ\nldZKMA7wmYURd4APpEAGRF9gt9d0JTCy9Uoi8jDwKBAPXOO1qEBENgKHgR+o6nIf25YAJQD9+oXM\n00+NOU2sK4ZvDc/jlqG5/HXLPmYtLuO+pYfp0y2RkrGXM+nKfiTGtTEwYDtbKxw/4LnP5MsNfu9b\nabO1EugDvMR4vtcO8EETyFNMtwETVXW6M303MFJVZ7ax/p3ADap6r4gkAKmqWiMilwO/By5u1eI4\njZ1iMuFCVVn6eRWzF5exruIA2anxTLu6kCmj+pHmj/GeTutbOdAqTFpPe/1bf4g2G/EtrZXEbtB4\n/PwP8BF4iibcBesU0x4gz2s615nXloXArwBU9QRwwnm/XkS2A4MASwAT9kSE8Rf0YPwFPVhTXsOs\nxWU8/c5n/GpJGfddVcDUK/Pp3pnxnkQgPsXzysg79/ot2tNaqT8EcSl2gI8SgQyIdcBAESnAEwyT\ngDu9VxCRgar6hTN5E/CFMz8HqFXVZhEpBAYC5QGs1ZigGFmYxcjCLD7afZDZi8v4xd++YO7ycqaM\n6s/0MQX0SOvC8Z7a2bdiokfAAkJVm0RkJvAunstc56vqZhF5AihV1UXATBG5FmgEDgD3OpuPBZ4Q\nkUbADTyoqrWBqtWYYBuSl8Gce4rZtu8Iv1xSxtzl5by4qoI7ivN4YFwhud1tvCfT9exGOWNCUEV1\nHc8u3c5vNlSiCt8c1pfvjC+iMCc12KWZCGN3UhsTpr48eJw5y8pZuG4XJ5rcfO3S3jw8fgCD+6QH\nuzQTISwgjAlz1UdPMG/FDlcqouIAABC/SURBVF75cCdHTzTxdxf24OFrBjC8X2gOjW7ChwWEMRHi\n0LFGXvqwgvkrd3DwWCNXFmUxc8IARhdl2XhP5rxYQBgTYepONLFgzS7mLC+n6sgJhvXL4O+vGcCE\nC3pYUJgOsYAwJkLVNzbz6/WVPLtkO3sOHuei3uk8PKGIGy/pjSvGgsKcmwWEMRGusdnNHzZ9yS+X\nlFFeVUdhTgoPjSvim8P6Eueym9VM2ywgjIkSzW7lnU894z1t3XuYvhlJPDiukNuL89oe78lENQsI\nY6KMqrJ4235mfVDGhl0HyUlLoGRMIXeO7EdKQiAHUDDhxgLCmCilqnxYXsPsxWWsLKshIzmO+68q\n4N7R+XRL9sPAgCbsWUAYY9i46wCzF5fx/tb9pCbEcvfo/ky7uoDs1IRgl2aCyALCGHPS1r2Hmb24\njD9/spd4VwyTR/SjZGwhfTKSgl2aCQILCGPMGcqrjvKrJdv53cY9iMCtw3N5cFwR+dkpwS7NdCEL\nCGNMmyoPHHPGe9pNU7Obbwzpw3fGD+CCXmnBLs10AQsIY8w57T9Sz7zlO3h19U7qGpq5fnBPZl4z\ngMtyM4JdmgkgCwhjTLsdPNbACysreHFVBYeONzJmYDYzJwxgZKE9SCgSWUAYYzrs6IkmXl29k7nL\ny6k+2sAV+d35zoQBjB+UY+M9RRALCGPMeatvbOaNdbt5bul2vjxUz0W90ykZW8DXL+tjw3hEAAsI\nY0ynNTS5+cOmPTy/vJzPvzpKn26J3H91AZNG9CPV7s4OWxYQxhi/cbuVpZ9X8dyy7awuryUtMZa7\nRvZn6lX59ExPDHZ5poMsIIwxAfFx5UGeW1bO25/sxRUj3DK0LyVjCxnU0y6RDRcWEMaYgNpVc4x5\nK8p5s7SS443NTLggh5KxRYwqzLQO7RBnAWGM6RIH6hp4dfVOXvqwguqjDVyW242SsYVMvLgXsdah\nHZIsIIwxXaq+sZnfbtjD3OXllFfXkZeZxPSrC7m9OJfkeOvQDiUWEMaYoHC7lfe2fsWcZeWs33mA\njOQ47h7Vn3uvzLdRZEOEBYQxJujW76zluaXlvLf1K+JcMdw6PJcZYwoozEkNdmlRzQLCGBMytlcd\nZe7yHfxmQyWNzW6uu6gnD4wr5PL+mcEuLSpZQBhjQk7VkRO88mEFL6/eycFjjVzevzslYwu57qKe\nxMTYlU9dxQLCGBOyjjU08evSSuauKGd37XEKs1OYPqaQ/zW8L4lxrmCXF/EsIIwxIa+p2c07m/cx\nZ1k5H1ceIjs1nntG53P3qP50T4kPdnkRywLCGBM2VJXV5bXMWbadxduqSIpz8e3iXKaPKSQvMznY\n5UWcswVEQO9cEZGJIrJNRMpE5DEfyx8UkU9EZJOIrBCRwV7Lvu9st01EbghkncaY0CEijC7K4oWp\nI/jr98Zy02W9WbB2F+N+spiHF2zg48qDwS4xagSsBSEiLuBz4DqgElgHTFbVLV7rpKvqYef9zcB3\nVHWiExSvAyOAPsD7wCBVbW7r+6wFYUzk2neonhdXVfDamp0cqW9iVGEmD4wtYtygHOvQ7qRgtSBG\nAGWqWq6qDcBC4BbvFVrCwZECtKTVLcBCVT2hqjuAMufzjDFRqFe3RB678UJWPXYNP7jpInbWHGPq\ni+u44WfLeLN0Nyea2vzb0XRCIAOiL7Dba7rSmXcaEXlYRLYDPwYe6eC2JSJSKiKlVVVVfivcGBOa\n0hLjmD6mkGX/MoH/uWMIrhjhX976mDFPL+ZXS7Zz6HhjsEuMKEEfPUtVZ6tqEfCvwA86uO0cVS1W\n1eKcnJzAFGiMCTlxrhi+NSyXt787hpfvH8Ggnmk8/c5nXPXUB/zoT1v48uDxYJcYEQI5atYeIM9r\nOteZ15aFwK/Oc1tjTBQSEcYOymHsoBw+3XOI55eX88KqCl5cVcE3hvRhxphCBvdJD3aZYSuQLYh1\nwEARKRCReGASsMh7BREZ6DV5E/CF834RMElEEkSkABgIrA1grcaYMHdJ3278fNIwlv3LBO69Mp+/\nbt7H136xnLvnrWHFF9VEyiX9XSlgLQhVbRKRmcC7gAuYr6qbReQJoFRVFwEzReRaoBE4ANzrbLtZ\nRN4EtgBNwMNnu4LJGGNa9M1I4t+/PphHrhnIa2t38sLKCqbMW8Pg3umUjC3kpst6E2fPpmgXu1HO\nGBPRTjQ184eNXzJneTll+4/SNyOJqVflM2lEP1IT7NkUdie1MSbqud3K4m37eW5ZOWt31JKeGMtd\no/oz9cp8eqQnBru8oLGAMMYYL5t2H2TOsu288+k+YmNi+OYwT4f2wJ5pwS6ty1lAGGOMDztr6pi3\nYgdvlu6mvtHNNRf2oGRsISMLMhGJjju0LSCMMeYsausaeOXDnbz0YQW1dQ0Mye1GydgiJl7SC1eE\nD+VhAWGMMe1Q39jMW+srmbu8nIqaY/TLTGb6mAJuvzyPpPjIfDaFBYQxxnRAs1t5b8s+nltWzsZd\nB+meHMfdo/O5d3R/slITgl2eX1lAGGPMeVBVSnce4Lml5by/9SsSYmO47XLPsykKslOCXZ5fnC0g\n7CJgY4xpg4hwRX4mV+Rnsr3qKHOXl/Pr9ZUsWLuL6wf3pGRsEZf37x7sMgPGWhDGGNMBVUdO8NKq\nCl5ZvZNDxxsp7t+dkrGFXHtRz7B8NoWdYjLGGD+rO9HEm6W7mbdiB5UHjlOYk8KMMYV8a1hfEuPC\np0PbAsIYYwKkqdnNXz7dx5xl2/l0z2GyU+O578p8pozqT0ZyfLDLOycLCGOMCTBV5cPtNcxZXs6S\nbVUkxbm444o8pl1dQF5mcrDLa5MFhDHGdKFt+44wZ1k5iz7aQ7Nb+dqlvXlgbBGX5nYLdmlnsIAw\nxpgg2HeonhdW7mDBml0cOdHE6MIsSsYVMn5QTsgM5WEBYYwxQXS4vpGFa3cxf0UF+w7Xc0HPNGaM\nLeTmIX2Ijw3usyksIIwxJgQ0NLn540df8vzycj7bd4Se6QlMvaqAO0f2Iz0xLig1WUAYY0wIUVWW\nfl7F88vLWVlWQ2pCLJNH5HH/1QX07pbUpbVYQBhjTIj6dM8h5iwr58+f7EWAm4f0YcbYQi7qnd4l\n328BYYwxIW537THmr9zBG+t2c6yhmbGDcnhgbCFXFmUFtEPbAsIYY8LEwWMNvLZmFy+srKD66Aku\n7pNOydhCbrq0N7Eu/3doW0AYY0yYqW9s5vcb9zBneTnlVXX0zUji/qsLmHRFHikJ/htn1QLCGGPC\nlNutfPDZfuYsK2dtRS3pibFMGdWf+67Kp0daYqc/3wLCGGMiwMZdB5izrJx3Nu8jLiaGbw3ry4yx\nBQzokXben2kBYYwxEaSiuo65K8r5dWklJ5rc3HRZb2ZNHnZendn2wCBjjIkg+dkp/Oibl/K9awfx\n8oc7aXK7A3KlkwWEMcaEqazUBL533aCAfX5wBwExxhgTsiwgjDHG+GQBYYwxxicLCGOMMT4FNCBE\nZKKIbBORMhF5zMfyR0Vki4h8LCJ/E5H+XsuaRWST81oUyDqNMcacKWBXMYmIC5gNXAdUAutEZJGq\nbvFabSNQrKrHROQh4MfAHc6y46o6NFD1GWOMObtAtiBGAGWqWq6qDcBC4BbvFVR1saoecyZXA7kB\nrMcYY0wHBDIg+gK7vaYrnXltmQa87TWdKCKlIrJaRL7pawMRKXHWKa2qqup8xcYYY04KiRvlRGQK\nUAyM85rdX1X3iEgh8IGIfKKq2723U9U5wBznM6pEZGcnysgGqjuxfaiIlP0A25dQFSn7Ein7AZ3b\nl/5tLQhkQOwB8rymc515pxGRa4F/A8ap6omW+aq6x/m3XESWAMOA7a2391o/pzPFikhpW+ORhJNI\n2Q+wfQlVkbIvkbIfELh9CeQppnXAQBEpEJF4YBJw2tVIIjIMeA64WVX3e83vLiIJzvts4CrAu3Pb\nGGNMgAWsBaGqTSIyE3gXcAHzVXWziDwBlKrqIuAnQCrwa2egqV2qejNwEfCciLjxhNhTra5+MsYY\nE2AB7YNQ1b8Af2k17z+83l/bxnargEsDWZsPc7r4+wIlUvYDbF9CVaTsS6TsBwRoXyLmeRDGGGP8\ny4baMMYY45MFhDHGGJ+iKiDaMTZUgoi84SxfIyL5XV9l+7RjX+5z7g1pGc9qejDqPBcRmS8i+0Xk\n0zaWi4j8wtnPj0VkeFfX2F7t2JfxInLI6zf5D1/rBZuI5InIYmectM0i8l0f64TF79LOfQmX3yVR\nRNaKyEfOvvynj3X8ewxT1ah44bmSajtQCMQDHwGDW63zHeBZ5/0k4I1g192JfbkPmBXsWtuxL2OB\n4cCnbSz/Gp477AUYBawJds2d2JfxwJ+CXWc79qM3MNx5nwZ87uO/r7D4Xdq5L+HyuwiQ6ryPA9YA\no1qt49djWDS1IM45NpQz/ZLz/i3g7yQQD3rtvPbsS1hQ1WVA7VlWuQV4WT1WAxki0rtrquuYduxL\nWFDVvaq6wXl/BNjKmcPkhMXv0s59CQvO/9ZHnck459X6KiO/HsOiKSDaMzbUyXVUtQk4BGR1SXUd\n095xrm51mv9viUiej+XhoKNjeoW60c4pgrdF5OJgF3MuzimKYXj+WvUWdr/LWfYFwuR3ERGXiGwC\n9gPvqWqbv4s/jmHRFBDR5o9AvqpeBrzHqb8qTPBswDPG2BDgGeD3Qa7nrEQkFfgN8A+qejjY9XTG\nOfYlbH4XVW1Wz2MQcoERInJJIL8vmgKiPWNDnVxHRGKBbkBNl1TXMefcF1Wt0VNjW80FLu+i2vyt\nXWN6hQNVPdxyikA9N5HGOUPJhBwRicNzQH1NVX/rY5Ww+V3OtS/h9Lu0UNWDwGJgYqtFfj2GRVNA\nnHNsKGf6Xuf9bcAH6vT2hJj2jHPlfT74ZjznXsPRIuAe56qZUcAhVd0b7KLOh4j0ajkfLCIj8Pz/\nL+T+AHFqnAdsVdWftrFaWPwu7dmXMPpdckQkw3mfhOdhbJ+1Ws2vx7CQGO67K2j7xoaaB7wiImV4\nOhsnBa/itrVzXx4RkZuBJjz7cl/QCj4LEXkdz1Uk2SJSCTyOp/MNVX0Wz1AtXwPKgGPA1OBUem7t\n2JfbgIdEpAk4DkwK0T9ArgLuBj5xzncD/G+gH4Td79KefQmX36U38JJ4ntYZA7ypqn8K5DHMhtow\nxhjjUzSdYjLGGNMBFhDGGGN8soAwxhjjkwWEMcYYnywgjDHG+GQBYUwHiEiz16ifm8THSLqd+Oz8\ntkaCNSYYouY+CGP85Lgz1IExEc9aEMb4gYhUiMiPReQTZ8z+Ac78fBH5wBk08W8i0s+Z31NEfucM\nEPeRiFzpfJRLRJ53xvv/q3PHrDFBYQFhTMcktTrFdIfXskOqeikwC/iZM+8Z4CVn0MTXgF84838B\nLHUGiBsObHbmDwRmq+rFwEHg1gDvjzFtsjupjekAETmqqqk+5lcA16hquTM43D5VzRKRaqC3qjY6\n8/eqaraIVAG5XgMqtgxH/Z6qDnSm/xWIU9UfBX7PjDmTtSCM8R9t431HnPB634z1E5ogsoAwxn/u\n8Pr3Q+f9Kk4NmHYXsNx5/zfgITj5EJhuXVWkMe1lf50Y0zFJXqOCAryjqi2XunYXkY/xtAImO/P+\nHnhBRP4ZqOLUqKffBeaIyDQ8LYWHgJAbLttEN+uDMMYPnD6IYlWtDnYtxviLnWIyxhjjk7UgjDHG\n+GQtCGOMMT5ZQBhjjPHJAsIYY4xPFhDGGGN8soAwxhjj0/8HJkbXRuOYpoQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "is75fcgvLMPE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# plot model\n",
        "model = build_esim()\n",
        "model.load_weights('/content/gdrive/My Drive/NLP/bilstm-max-ESIM-04-0.8724.hdf5')\n",
        "\n",
        "plot_model(model, to_file='model.png')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "keuor7QYfocq",
        "colab_type": "text"
      },
      "source": [
        "# 6. **Evaluation on SICK dataset - Inference Task**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-u4ckCOfnlz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "d3eba423-4055-47a0-9e5c-31d8cedb060f"
      },
      "source": [
        "# download dataset\n",
        "#! wget \"https://zenodo.org/record/2787612/files/SICK.zip?download=1\" -P /content/gdrive/My\\ Drive/NLP/\n",
        "!unzip /content/gdrive/My\\ Drive/NLP/SICK.zip -d /content/gdrive/My\\ Drive/NLP/"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/gdrive/My Drive/NLP/SICK.zip\n",
            "  inflating: /content/gdrive/My Drive/NLP/readme.txt  \n",
            "  inflating: /content/gdrive/My Drive/NLP/SICK.txt  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FOkziu_KgH45",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# read dataset\n",
        "sick_dataset = pd.read_csv('/content/gdrive/My Drive/NLP/SICK.txt',sep=\"\\t\")\n",
        "\n",
        "\n",
        "# preprocess drop nan entries and ignore entries with no label\n",
        "sick_dataset = sick_dataset.dropna(subset = ['sentence_B'])\n",
        "sick_dataset = sick_dataset[sick_dataset[\"entailment_label\"] != \"-\"]\n",
        "\n",
        "sick_train = sick_dataset[sick_dataset['SemEval_set'] == 'TRAIN']\n",
        "sick_trial = sick_dataset[sick_dataset['SemEval_set'] == 'TRIAL']\n",
        "sick_test = sick_dataset[sick_dataset['SemEval_set'] == 'TEST']\n",
        "\n",
        "# construct label arrays\n",
        "y_train = sick_train['entailment_label']\n",
        "y_val = sick_trial['entailment_label']\n",
        "y_test = sick_test['entailment_label']\n",
        "\n",
        "# constrauct X_train arrays\n",
        "x_train_raw = sick_train[['sentence_A','sentence_B']]\n",
        "x_val_raw = sick_trial[['sentence_A','sentence_B']]\n",
        "x_test_raw = sick_test[['sentence_A','sentence_B']]\n",
        "\n",
        "# one hot encode labels\n",
        "y_train_e = onehot(y_train)\n",
        "y_val_e = onehot(y_val)\n",
        "y_test_e = onehot(y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x7XmQ9oUiBsr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "outputId": "682c50bb-41b6-4cda-9a5f-2397f7c6d111"
      },
      "source": [
        "sick_train.head()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pair_ID</th>\n",
              "      <th>sentence_A</th>\n",
              "      <th>sentence_B</th>\n",
              "      <th>entailment_label</th>\n",
              "      <th>relatedness_score</th>\n",
              "      <th>entailment_AB</th>\n",
              "      <th>entailment_BA</th>\n",
              "      <th>sentence_A_original</th>\n",
              "      <th>sentence_B_original</th>\n",
              "      <th>sentence_A_dataset</th>\n",
              "      <th>sentence_B_dataset</th>\n",
              "      <th>SemEval_set</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>A group of kids is playing in a yard and an ol...</td>\n",
              "      <td>A group of boys in a yard is playing and a man...</td>\n",
              "      <td>NEUTRAL</td>\n",
              "      <td>4.5</td>\n",
              "      <td>A_neutral_B</td>\n",
              "      <td>B_neutral_A</td>\n",
              "      <td>A group of children playing in a yard, a man i...</td>\n",
              "      <td>A group of children playing in a yard, a man i...</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>TRAIN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>A group of children is playing in the house an...</td>\n",
              "      <td>A group of kids is playing in a yard and an ol...</td>\n",
              "      <td>NEUTRAL</td>\n",
              "      <td>3.2</td>\n",
              "      <td>A_contradicts_B</td>\n",
              "      <td>B_neutral_A</td>\n",
              "      <td>A group of children playing in a yard, a man i...</td>\n",
              "      <td>A group of children playing in a yard, a man i...</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>TRAIN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>The young boys are playing outdoors and the ma...</td>\n",
              "      <td>The kids are playing outdoors near a man with ...</td>\n",
              "      <td>ENTAILMENT</td>\n",
              "      <td>4.7</td>\n",
              "      <td>A_entails_B</td>\n",
              "      <td>B_entails_A</td>\n",
              "      <td>The children are playing outdoors, while a man...</td>\n",
              "      <td>The children are playing outdoors, while a man...</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>TRAIN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>The young boys are playing outdoors and the ma...</td>\n",
              "      <td>There is no boy playing outdoors and there is ...</td>\n",
              "      <td>CONTRADICTION</td>\n",
              "      <td>3.6</td>\n",
              "      <td>A_contradicts_B</td>\n",
              "      <td>B_contradicts_A</td>\n",
              "      <td>The children are playing outdoors, while a man...</td>\n",
              "      <td>The children are playing outdoors, while a man...</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>TRIAL</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>The kids are playing outdoors near a man with ...</td>\n",
              "      <td>A group of kids is playing in a yard and an ol...</td>\n",
              "      <td>NEUTRAL</td>\n",
              "      <td>3.4</td>\n",
              "      <td>A_neutral_B</td>\n",
              "      <td>B_neutral_A</td>\n",
              "      <td>A group of children playing in a yard, a man i...</td>\n",
              "      <td>The children are playing outdoors, while a man...</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>FLICKR</td>\n",
              "      <td>TRAIN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   pair_ID  ... SemEval_set\n",
              "0        1  ...       TRAIN\n",
              "1        2  ...       TRAIN\n",
              "2        3  ...       TRAIN\n",
              "3        4  ...       TRIAL\n",
              "4        5  ...       TRAIN\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZVsIJox2lYZO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 2.Tokenize\n",
        "main_train, hypo_train, main_val, hypo_val, main_test, hypo_test=[], [], [], [], [], []\n",
        "\n",
        "for entry in x_train_raw['sentence_A']:\n",
        "  main_train.append(entry)\n",
        "for entry in x_train_raw['sentence_B']:\n",
        "  hypo_train.append(entry)\n",
        "for entry in x_val_raw['sentence_A']:\n",
        "  main_val.append(entry)\n",
        "for entry in x_val_raw['sentence_B']:\n",
        "  hypo_val.append(entry)\n",
        "for entry in x_test_raw['sentence_A']:\n",
        "  main_test.append(entry)\n",
        "for entry in x_test_raw['sentence_B']:\n",
        "  hypo_test.append(entry)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Au3AE6OOmMCJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 4.Convert text to sequence\n",
        "main_sequences = tokenizer.texts_to_sequences(main_train)\n",
        "hypo_sequences = tokenizer.texts_to_sequences(hypo_train)\n",
        "main_sequences_val = tokenizer.texts_to_sequences(main_val)\n",
        "hypo_sequences_val = tokenizer.texts_to_sequences(hypo_val)\n",
        "main_sequences_test = tokenizer.texts_to_sequences(main_test)\n",
        "hypo_sequences_test = tokenizer.texts_to_sequences(hypo_test)\n",
        "# 5.Zero pad sequences\n",
        "main_seq_padded = pad_sequences(main_sequences, maxlen=82)\n",
        "hypo_seq_padded = pad_sequences(hypo_sequences, maxlen=82)\n",
        "main_seq_padded_val = pad_sequences(main_sequences_val, maxlen=82)\n",
        "hypo_seq_padded_val = pad_sequences(hypo_sequences_val, maxlen=82)\n",
        "main_seq_padded_test = pad_sequences(main_sequences_test, maxlen=82)\n",
        "hypo_seq_padded_test = pad_sequences(hypo_sequences_test, maxlen=82)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7vcbVda_mRX3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ffc7ec72-371d-4e43-82a3-a8956b136726"
      },
      "source": [
        "EMBEDDING_DIM=300\n",
        "word_index = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(word_index))\n",
        "\n",
        "# reserve zero for padding\n",
        "num_words = len(word_index) + 1\n",
        "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
        "\n",
        "for word, i in word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None :\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix[i] = embedding_vector"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 34606 unique tokens.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t-YQ56Ajmvxs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_sick_clf():\n",
        "  model = build_esim()\n",
        "  model.load_weights('/content/gdrive/My Drive/NLP/bilstm-max-ESIM-04-0.8724.hdf5')\n",
        "  # Freeze the layers except the last 2 layers\n",
        "  for layer in model.layers[:-2]:\n",
        "      layer.trainable = False\n",
        "  # Check the trainable status of the individual layers\n",
        "  for layer in model.layers:\n",
        "      print(layer, layer.trainable)\n",
        "\n",
        "  #model = Model(inputs, output)\n",
        "  model.compile(loss='categorical_crossentropy', metrics=['acc'], optimizer='adam')\n",
        "  return model\n",
        "\n",
        "model = build_sick_clf()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BYRtnaUbp0RU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 604
        },
        "outputId": "7cd0a171-6f70-4cf7-cc16-bfec34f8fd73"
      },
      "source": [
        "# train ESIM model with FastText\n",
        "batch_size = 64\n",
        "n_epoch = 8\n",
        "model_name='ESIM_SICK_FT'\n",
        "model_save_path = '/content/gdrive/My Drive/NLP/'+model_name+'-{epoch:02d}-{val_acc:.4f}.hdf5'\n",
        "checkpoint = ModelCheckpoint(model_save_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
        "callback_list=[checkpoint]\n",
        "hist = model.fit([main_seq_padded,hypo_seq_padded], y_train_e,\n",
        "          batch_size=batch_size,\n",
        "          epochs=n_epoch,\n",
        "          validation_data=([main_seq_padded_val,hypo_seq_padded_val], y_val_e),\n",
        "          callbacks=callback_list)"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 4439 samples, validate on 495 samples\n",
            "Epoch 1/8\n",
            "4439/4439 [==============================] - 47s 11ms/step - loss: 0.5014 - acc: 0.8117 - val_loss: 0.3453 - val_acc: 0.8566\n",
            "\n",
            "Epoch 00001: val_acc improved from -inf to 0.85657, saving model to /content/gdrive/My Drive/NLP/ESIM_SICK_FT-01-0.8566.hdf5\n",
            "Epoch 2/8\n",
            "4439/4439 [==============================] - 41s 9ms/step - loss: 0.3206 - acc: 0.8777 - val_loss: 0.3299 - val_acc: 0.8606\n",
            "\n",
            "Epoch 00002: val_acc improved from 0.85657 to 0.86061, saving model to /content/gdrive/My Drive/NLP/ESIM_SICK_FT-02-0.8606.hdf5\n",
            "Epoch 3/8\n",
            "4439/4439 [==============================] - 41s 9ms/step - loss: 0.2938 - acc: 0.8883 - val_loss: 0.3282 - val_acc: 0.8545\n",
            "\n",
            "Epoch 00003: val_acc did not improve from 0.86061\n",
            "Epoch 4/8\n",
            "4439/4439 [==============================] - 40s 9ms/step - loss: 0.2733 - acc: 0.8970 - val_loss: 0.3145 - val_acc: 0.8566\n",
            "\n",
            "Epoch 00004: val_acc did not improve from 0.86061\n",
            "Epoch 5/8\n",
            "4439/4439 [==============================] - 40s 9ms/step - loss: 0.2520 - acc: 0.9016 - val_loss: 0.3180 - val_acc: 0.8586\n",
            "\n",
            "Epoch 00005: val_acc did not improve from 0.86061\n",
            "Epoch 6/8\n",
            "4439/4439 [==============================] - 41s 9ms/step - loss: 0.2355 - acc: 0.9103 - val_loss: 0.3164 - val_acc: 0.8646\n",
            "\n",
            "Epoch 00006: val_acc improved from 0.86061 to 0.86465, saving model to /content/gdrive/My Drive/NLP/ESIM_SICK_FT-06-0.8646.hdf5\n",
            "Epoch 7/8\n",
            "4439/4439 [==============================] - 41s 9ms/step - loss: 0.2198 - acc: 0.9160 - val_loss: 0.3264 - val_acc: 0.8626\n",
            "\n",
            "Epoch 00007: val_acc did not improve from 0.86465\n",
            "Epoch 8/8\n",
            "4439/4439 [==============================] - 40s 9ms/step - loss: 0.2049 - acc: 0.9189 - val_loss: 0.3317 - val_acc: 0.8687\n",
            "\n",
            "Epoch 00008: val_acc improved from 0.86465 to 0.86869, saving model to /content/gdrive/My Drive/NLP/ESIM_SICK_FT-08-0.8687.hdf5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKdVqSAfys5E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "061dc842-fb5d-4f7b-a3ea-346a1264e77d"
      },
      "source": [
        "metrics = model.evaluate([main_seq_padded_test,hypo_seq_padded_test],y_test_e,batch_size=64)\n",
        "print(\"ESIM-FastText Accuracy on SICK test-set : \"+str(round(metrics[1]*100,2))+\" %\")\n",
        "print()"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4906/4906 [==============================] - 40s 8ms/step\n",
            "ESIM-FastText Accuracy on SICK test-set : 88.73 %\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swWa48dnJlWq",
        "colab_type": "text"
      },
      "source": [
        "# ||===|| Test Sentence Encoder ||===||"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nJn2RdExJvqX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_encoder():\n",
        "  model = build_esim()\n",
        "  model.load_weights('/content/gdrive/My Drive/NLP/bilstm-max-ESIM-04-0.8724.hdf5')\n",
        "  # Check the trainable status of the individual layers\n",
        "  model.layers.pop()\n",
        "  model.layers.pop()\n",
        "\n",
        "  model.trainable = False\n",
        "  return model\n",
        "\n",
        "model = build_encoder()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AARiJe2zbNed",
        "colab_type": "text"
      },
      "source": [
        "* **Generate Embedding for a given Sentence**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpmuPQI5KEYZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "27b89478-1215-42ba-a2b8-2df815225907"
      },
      "source": [
        "main_sent = tokenizer.texts_to_sequences(['The kids are playing outdoors near a man'])\n",
        "hypo_sent = tokenizer.texts_to_sequences(['The kids are playing outdoors near a man'])\n",
        "\n",
        "main_sent_pad = pad_sequences(main_sent, maxlen=82)\n",
        "hypo_sent_pad = pad_sequences(hypo_sent, maxlen=82)\n",
        "\n",
        "layer_name = 'concatenate_18'\n",
        "intermediate_layer_model = Model(inputs=model.input,\n",
        "                                 outputs=model.get_layer(layer_name).output)\n",
        "intermediate_output = intermediate_layer_model.predict([main_sent_pad,hypo_sent_pad])\n",
        "# embedding for main sentence --------------------------\n",
        "intermediate_output[:1024]"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.00986849, -0.00082916,  0.00212319, ...,  0.00308811,\n",
              "         0.07802185,  0.02487027]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    }
  ]
}