fb_test-with-colored-confusion-matrix-finaltesting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation,Conv2D,MaxPooling2D\n",
    "from keras.layers import Input, Embedding, Add\n",
    "from keras import layers\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.models import Model\n",
    "from keras.models import load_model\n",
    "from keras.utils import to_categorical\n",
    "from keras.utils import plot_model\n",
    "import nltk\n",
    "from gensim.models import FastText\n",
    "from gensim.models import Word2Vec\n",
    "import re\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score\n",
    "from sklearn import svm\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.layers import Bidirectional\n",
    "from keras.layers import K, Activation\n",
    "from keras.engine import Layer\n",
    "from keras.layers import Dense, Input, Embedding, Dropout, Bidirectional, GRU, Flatten, SpatialDropout1D\n",
    "from keras.models import load_model\n",
    "from sklearn.metrics import f1_score\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_excel('augmented.xlsx')\n",
    "test = pd.read_excel('fbtest_cleaned.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(39512, 150)\n",
      "(916, 150)\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(train['text'])\n",
    "sequences = tokenizer.texts_to_sequences(train['text'])\n",
    "data = pad_sequences(sequences, maxlen=150)\n",
    "print(data.shape)\n",
    "sequences_t = tokenizer.texts_to_sequences(test['text'])\n",
    "data_test = pad_sequences(sequences_t, maxlen=150)\n",
    "print(data_test.shape)\n",
    "vocabulary_size = len(tokenizer.word_index) + 1 # 20648"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_len = 128\n",
    "Routings = 5\n",
    "Num_capsule = 10\n",
    "Dim_capsule = 16\n",
    "dropout_p = 0.25\n",
    "rate_drop_dense = 0.28\n",
    "\n",
    "def squash(x, axis=-1):\n",
    "    # s_squared_norm is really small\n",
    "    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n",
    "    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n",
    "    # return scale * x\n",
    "    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n",
    "    scale = K.sqrt(s_squared_norm + K.epsilon())\n",
    "    return x / scale\n",
    "\n",
    "\n",
    "# A Capsule Implement with Pure Keras\n",
    "class Capsule(Layer):\n",
    "    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n",
    "                 activation='default', **kwargs):\n",
    "        super(Capsule, self).__init__(**kwargs)\n",
    "        self.num_capsule = num_capsule\n",
    "        self.dim_capsule = dim_capsule\n",
    "        self.routings = routings\n",
    "        self.kernel_size = kernel_size\n",
    "        self.share_weights = share_weights\n",
    "        if activation == 'default':\n",
    "            self.activation = squash\n",
    "        else:\n",
    "            self.activation = Activation(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        super(Capsule, self).build(input_shape)\n",
    "        input_dim_capsule = input_shape[-1]\n",
    "        if self.share_weights:\n",
    "            self.W = self.add_weight(name='capsule_kernel',\n",
    "                                     shape=(1, input_dim_capsule,\n",
    "                                            self.num_capsule * self.dim_capsule),\n",
    "                                     # shape=self.kernel_size,\n",
    "                                     initializer='glorot_uniform',\n",
    "                                     trainable=True)\n",
    "        else:\n",
    "            input_num_capsule = input_shape[-2]\n",
    "            self.W = self.add_weight(name='capsule_kernel',\n",
    "                                     shape=(input_num_capsule,\n",
    "                                            input_dim_capsule,\n",
    "                                            self.num_capsule * self.dim_capsule),\n",
    "                                     initializer='glorot_uniform',\n",
    "                                     trainable=True)\n",
    "\n",
    "    def call(self, u_vecs):\n",
    "        if self.share_weights:\n",
    "            u_hat_vecs = K.conv1d(u_vecs, self.W)\n",
    "        else:\n",
    "            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n",
    "\n",
    "        batch_size = K.shape(u_vecs)[0]\n",
    "        input_num_capsule = K.shape(u_vecs)[1]\n",
    "        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n",
    "                                            self.num_capsule, self.dim_capsule))\n",
    "        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n",
    "        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n",
    "\n",
    "        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n",
    "        for i in range(self.routings):\n",
    "            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n",
    "            c = K.softmax(b)\n",
    "            c = K.permute_dimensions(c, (0, 2, 1))\n",
    "            b = K.permute_dimensions(b, (0, 2, 1))\n",
    "            outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))\n",
    "            if i < self.routings - 1:\n",
    "                b = K.batch_dot(outputs, u_hat_vecs, [2, 3])\n",
    "\n",
    "        return outputs\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (None, self.num_capsule, self.dim_capsule)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = test['Agr_Class'].astype('category').cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FastText(vocab=24411, size=100, alpha=0.025)\n",
      "Loaded 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "codemixed_embeddings = FastText.load('augmented_ft.bin')\n",
    "print(codemixed_embeddings )\n",
    "embeddings_index = dict()\n",
    "f = open('/home/parth/research/english/new_version/glove.6B.100d.txt')\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:11: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs# create \n",
    "embedding_matrix = np.zeros((vocabulary_size, 100))\n",
    "for word, index in tokenizer.word_index.items():\n",
    "    if index > vocabulary_size - 1:\n",
    "        break\n",
    "    else:\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[index] = embedding_vector\n",
    "        else:\n",
    "            embedding_vector = codemixed_embeddings[word]\n",
    "            if embedding_vector is not None:\n",
    "                embedding_matrix[index] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_model(length, vocabulary_size,embedding_matrix):\n",
    "    input1 = Input(shape=(length,))\n",
    "    embedding1 = Embedding(vocabulary_size,100, weights=[embedding_matrix], trainable=False)(input1)\n",
    "    conv1 = Conv1D(filters=32, kernel_size=3, activation='relu')(embedding1)\n",
    "    conv1 = Dropout(0.5)(conv1)\n",
    "    capsule1 = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=3,share_weights=True)(conv1)\n",
    "    flat1 = Flatten()(capsule1)\n",
    "\n",
    "    input2 = Input(shape=(length,))\n",
    "    embedding2 = Embedding(vocabulary_size,100, weights=[embedding_matrix], trainable=False)(input2)\n",
    "    conv2 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding2)\n",
    "    conv2 = Dropout(0.5)(conv2)    \n",
    "    capsule2 = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=3,share_weights=True)(conv2)\n",
    "    flat2 = Flatten()(capsule2)    \n",
    "\n",
    "    input3 = Input(shape=(length,))\n",
    "    embedding3 = Embedding(vocabulary_size,100, weights=[embedding_matrix], trainable=False)(input3)\n",
    "    conv3 = Conv1D(filters=32, kernel_size=5, activation='relu')(embedding3)   \n",
    "    conv3 = Dropout(0.5)(conv3)    \n",
    "    capsule3 = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=3,share_weights=True)(conv3)\n",
    "    flat3 = Flatten()(capsule3)        \n",
    "\n",
    "    merged = concatenate([flat1, flat2, flat3])\n",
    "    \n",
    "    dense = Dense(32,activation='relu')(merged)\n",
    "    output = Dense(3, activation='softmax')(dense)\n",
    "    model = Model(inputs=[input1,input2,input3], outputs=output)\n",
    "    model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 150, 100)     2441200     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 150, 100)     2441200     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 150, 100)     2441200     input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 148, 32)      9632        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 147, 32)      12832       embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 146, 32)      16032       embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 148, 32)      0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 147, 32)      0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 146, 32)      0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "capsule_1 (Capsule)             (None, 10, 16)       5120        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "capsule_2 (Capsule)             (None, 10, 16)       5120        dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "capsule_3 (Capsule)             (None, 10, 16)       5120        dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 160)          0           capsule_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 160)          0           capsule_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 160)          0           capsule_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 480)          0           flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 32)           15392       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 3)            99          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 7,392,947\n",
      "Trainable params: 69,347\n",
      "Non-trainable params: 7,323,600\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = define_model(150, vocabulary_size,embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('aug_caps_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig = pd.read_csv('agr_en_fb_gold.csv')\n",
    "orig = orig.append({'facebook_corpus_msr_396174':'facebook_corpus_msr_396174',\"If government of india don't take strict action against China then soon india will be on the loosing side sure\":\"If government of india don't take strict action against China then soon india will be on the loosing side sure\",'NAG':'NAG'},ignore_index=True)\n",
    "orig = orig.rename(columns={'facebook_corpus_msr_396174': 'ID', \"If government of india don't take strict action against China then soon india will be on the loosing side sure\": 'text', 'NAG' : 'Agr_Class'})\n",
    "dct = {0:'CAG',1:'NAG',2:'OAG'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 37  89  16]\n",
      " [ 67 504  59]\n",
      " [ 20  62  62]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.30      0.26      0.28       142\n",
      "          1       0.77      0.80      0.78       630\n",
      "          2       0.45      0.43      0.44       144\n",
      "\n",
      "avg / total       0.65      0.66      0.65       916\n",
      "\n",
      "Accuracy :  0.6582969432314411\n",
      "F1 :  0.6520117859317077\n"
     ]
    }
   ],
   "source": [
    "pred = model.predict([data_test,data_test,data_test])\n",
    "prd = np.argmax(pred,axis = 1)\n",
    "orig['predicted'] = list(map(lambda x : dct[x],prd))\n",
    "print(confusion_matrix(y_test,prd))\n",
    "print(classification_report(y_test,prd))\n",
    "print(\"Accuracy : \",accuracy_score(prd,y_test))\n",
    "print(\"F1 : \",f1_score(y_test, prd, average='weighted', ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "orig[orig['predicted'] != orig['Agr_Class']].to_excel('misclassified.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[ 37  89  16]\n",
      " [ 67 504  59]\n",
      " [ 20  62  62]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAAEYCAYAAAAzhB+DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xu8VmP+//HXe5cilUYpKckkchhRUc7JMUIaoyJymjBjxgzGMMZvjDlgnAaNQ8OMHGYwaErOUhSVikKYxJchkUTpfPr8/lhrc2tq7/vO3q373vv9nMd6tO5rrften33b89nXda3rupYiAjMzy19Z1gGYmZUaJ04zswI5cZqZFciJ08ysQE6cZmYFcuI0MyuQE6eZWYGcOM2sxpH0nqTXJE2VNDkt21zS05LeTv/9TlouSTdKminpVUmdKvt8J04zq6kOjIjdIqJL+voiYFREtAdGpa8BegLt020QcEtlH1y3GoLNTNNmzaJNm7ZZh1HUVqxanXUIRa9+XdcnKvP+++8xd+5cVdXn1Wm8TcTKJXmdG0s+fTIiDl+PyxwDdE/3hwJjgF+m5XdFMo1ygqQmklpGxOx1fVCNSpxt2rTl2XETsw6jqH26YFnWIRS9Ns0aZB1C0duna5fKTypArFxK/Q798jp36Ss3NcvnI4GnJAVwW0QMAVrkJMOPgRbpfivgg5z3fpiW1Y7EaWYlSoDyrsA2K++3TA1JE2OufSNilqTmwNOS3so9GBGRJtX14sRpZsVBeXeRzM3pt1yriJiV/jtH0jBgT+CT8ia4pJbAnPT0WcDWOW9vnZatkztzzKw4SPltlX6MNpXUqHwfOBR4HRgBDExPGwgMT/dHACend9e7AfMr6t8E1zjNrCiokBpnZVoAw5Qk2brAPyLiCUmTgAcknQ68Dxyfnv8YcAQwE1gMnFrZBZw4zSx7AsrqVMlHRcS7QMe1lH8GHLSW8gB+XMg1nDjNrAjk1wwvFk6cZlYcqq6pXu2cOM2sOLjGaWZWiCq9OVTtnDjNLHuFDYDPnBOnmRUBQVnppKPSidTMarYy1zjNzPIn3MdpZlYw93GamRXCd9XNzArnGqeZWQGkKpurviE4cZpZcXBT3cysQG6qm5kVwjeHzMwK5xqnmVkB5CmXZmaFc43TzKxA7uM0MyuQa5xmZgWQ76qbmRWuhGqcpZPii9jSpUs5eP9u7Ne1E3t12ZUrfn8ZAEcccgD7d+vM/t06s1O7rRnQt0+mcWbtzttu4sgDutCrexfOO3sgy5YuZfy4MRx7yN706t6FX/70h6xcuTLrMDN15hmn0War5nTebZdvlN88+CY67tKBTh135lcXXZhRdNVHQFlZWV5bMSiOKEpc/fr1+fdjzzB24ss8P34Ko55+kkkvTeCxp5/j+QlTeH7CFLp07Uavo4/NOtTMfDL7I+664xYeemIsI8dMZtWq1Twy7H4uOncQ1906lJFjJrNV6zYMe+DerEPN1EkDT2H4yCe+UfbcmNGMfGQ4L02ZxsvTpvOz8y7IKLpqpAK2IuDEWQUk0bBhQwBWrFjByhUrUU6zY8GCBYx9bjRHHHVMViEWhVWrVrJ06RJWrlzJ0iWLadBgUzbaqB7btmsPwD779+CpR/+dcZTZ2ne//dl8882/UTbktlu44MKLqF+/PgDNmzfPIrRqJqT8tmLgxFlFVq1axf7dOrND25Z073EQXfbo+tWxxx4Zzv7de9C4ceMMI8xWi5ZbcdpZ53Jglw7s27EdDRs1pufR32fVypW8NvVlAJ4YOYyPP/ow40iLz8wZM3hh3Fj227srh/Q4gMmTJmUdUrVw4swhaUtJ90l6R9IUSY9J2j499jNJSyVttsZ79pQ0RtLbkl6W9Kik71V3rN9GnTp1eH7CFF6f8T4vT5nEG9Nf/+rYQ/+6j+//oF+G0WVv/hefM+rJkYyaOJ2xU2eyZPFiRjx0H9fdOpQrfvNLjuu5P5s2bERZndJZWmxDWblqJfPmzeP5FybwxyuvZsAJxxMRWYdV5Zw4U0p+ymHAmIhoFxGdgYuBFukp/YFJQJ+c97QAHgB+FRHtI6ITcAXQrjpjrSqbNWnCvvt3Z9TTTwLw2dy5vDxlEocefkTGkWXrxbGjad2mLZs324KNNtqIQ484mlcmT2T3Ll35x/CnefDx59mj2z60/W77rEMtOq1atab3sX2QxB577klZWRlz587NOqwq58T5tQOBFRFxa3lBREyLiLGS2gENgV+TJNBy5wBDI+LFnPeMi4ii7fya++mnzP/iCwCWLFnCmGefYfsddgBgxL8f4rDDj2TjjTfOMsTMbdVqa6ZNmcSSxYuJCMaPG0O79jvw2dw5ACxftoy//uU6+p18esaRFp+jju7Nc2NGA/D2jBksX76cZs2aZRxV1ZKEyvLbikF1j+PcBZiyjmP9gPuAscAOklpExCfAzsDQfC8gaRAwCKD11m2+XbTr6ZOPZ/OjQaexatUqVq9eTe/vH8dhPXsB8PCD93PueTVv+EihOnbag8N69ebYQ/ehbt067LhLR/oOOI3rr/otY55+gtWxmv4nn8Fe+3bPOtRMnTygP2OfG8PcuXNp17Y1l/6/3zLw1NM484zT6LzbLtTbqB63/21o0dS8qlIp/Uyqzr4SST8Fto2In6/l2OvAsRHxtqTrgHcjYrCkh0lqnMPT8yYCjYGnIuLciq63e6cu8ey4iVX/g9Qgny5YlnUIRa9NswZZh1D09unahSlTJldZpqvb9LvR+Ijf53Xu5/ecOCUiulTVtddHdTfVpwOd1yxMb/S0B56W9B5J7bN/zns6lZ8bEV2BS4HNMLMay32cX3sWqJ82pwGQtCtwI3BZRLRNt62ArSRtA/wFOEXS3jmf4yqAWU3mAfBfi6Qf4Fjg4HQ40nSSO+TdSe625xoG9IuIj4G+wBWSZkp6ETgOGFydsZpZtkqpxlnti3xExEfA8Xmcd17O/gTggOqMy8yKh1DRzEPPh1dHMrPiUByVybyUToo3s5pLVdtUl1RH0iuSRqavt5U0Me3+u19SvbS8fvp6Znq8bT6f78RpZkWhivs4zwXezHl9FXB9RGwHfA6Uz7Q4Hfg8Lb8+Pa9STpxmVhSqKnFKag0cCdyevhbQA3gwPWUo0DvdP4avJ9w8CBykPC7ixGlmmVPVLiv3Z+BCYHX6uinwRUSUr5L9IdAq3W8FfACQHp+fnl8hJ04zy54oZK56M0mTc7bcceK9gDkRsa6p3lXCd9XNrCgU0H85t4Ipl/sAR0s6AtiYZLr2DUATSXXTWmVrYFZ6/ixga+BDSXVJZih+VlkArnGaWVGoiqZ6RFwcEa0joi3JVO5nI+JEYDTJRBqAgcDwdH9E+pr0+LORxwIeTpxmVhyqd8rlL4HzJM0k6cO8Iy2/A2ialp8HXJTPh7mpbmZFoaqnU0bEGGBMuv8usOdazlkK/KDQz3biNLPMFdM89Hw4cZpZUfBcdTOzQpVOhdOJ08yKg5vqZmaFkBOnmVlBBJRQ3nTiNLNi4LvqZmYFKyuSZ6bnw4nTzLInN9XNzAoiXOM0MyuYa5xmZgXyzSEzs0K4j9PMrDB+rrqZ2XpwjdPMrEDu4zQzK4T7OM3MCpPMVS+dzOnEaWZFoYTyphOnmRUHzxzKyKrVwcKlK7MOo6h17Hlh1iEUvXdGX5d1CEVv5epKn6BbGK/HaWZWGK/HaWZWMK/HaWZWsBLKm06cZlYcXOM0MyuA5LvqZmYFc43TzKxAJZQ3nTjNrDi4xmlmVggv8mFmVhh5HKeZWeHq+K66mVlhSqjC6cRpZtlTTVnkQ1Ljit4YEQuqPhwzq61KqKVeYY1zOhAkC5eUK38dQJtqjMvMapkaUeOMiK03ZCBmVrtVVd6UtDHwPFCfJMc9GBG/kbQtcB/QFJgCnBQRyyXVB+4COgOfAX0j4r2KrpHXg4wl9ZP0q3S/taTO6/kzmZn9DwF1pLy2PCwDekRER2A34HBJ3YCrgOsjYjvgc+D09PzTgc/T8uvT8ypUaeKUNBg4EDgpLVoM3JpP9GZmeVEyjjOfrTKRWJi+3CjdAugBPJiWDwV6p/vHpK9Jjx+kSi6UT41z74g4E1iaBjUPqJfH+8zM8ibltwHNJE3O2Qb972epjqSpwBzgaeAd4IuIKH+2zodAq3S/FfABQHp8Pklzfp3yGY60QlIZScZGUlNgdR7vMzPLi4Cy/Ds550ZEl4pOiIhVwG6SmgDDgA7fLsJvyqfG+RfgIWALSb8FxpFHH4CZWSEKqHHmLSK+AEYDewFNJJVXFlsDs9L9WcDWSQyqC2xGcpNonSpNnBFxF/Br4BpgHvCDiLivsPDNzCpWVX2ckrZIa5pI2gQ4BHiTJIEel542EBie7o9IX5MefzYiKnyMZ74zh+oAK0ia63ndiTczy5dUpXPVWwJDJdUhyVcPRMRISW8A90n6PfAKcEd6/h3A3ZJmklQO+1V2gUoTp6RLgBNI+gkE/EPSvRFxxfr8RGZma1NVaTMiXgV2X0v5u8CeaylfCvygkGvkU+M8Gdg9IhYDSPoDSbZ24jSzKlMjZg7lmL3GeXXTMjOzKpHcVc86ivxVtMjH9SR9mvOA6ZKeTF8fCkzaMOGZWa2Q542fYlFRjfP19N/pwKM55ROqLxwzq61qxOOBI+KOdR0zM6tKNaapXk5SO+APwE7AxuXlEbF9NcZVcubP/4ILzz2bGW9ORxJX33Qbd9w6mHdnzgBgwfwvaLxZE5547qWMI92w3nr0t3y5aBmrVq9m5arV7Hvin/hO4wbcfdVpbLPV5rz/0TwGXHgHX3y55Kv3dN6pDWOGns/JF/+dYc9MzTD6bHTddXsaNmxIWZ061K1bl8dHj2f6a69y0fnnsHjhQlq32YbBQ4bSqHGFS+aWnJrSVC93J/B7kgHwPYFTSadf2tcuu/h8uh90CLfd+U+WL1/OkiWLufmOe746/rtLf1njftHzdfigG/jsi0Vfvb7g1EMY89J/uObvT3PBqYdwwamH8usbk7HIZWXi9+cewzMT3soq3KLwr0eeYvOmzb56/Ytzz+LS313JXvvsz3333MktN13HhZdcll2A1aB00mZ+g9kbRMSTABHxTkT8miSBWmrBgvm8NH4c/QacCkC9evXYbLMmXx2PCEb++0GO6dM3qxCLSq/uu3LPIxMBuOeRiRx14K5fHftRvwP496hpfDrvy6zCK0rvznybbnvvB8B+3Q/isUeGZRxR1ZKSuer5bMUgn8S5LF3k4x1JZ0k6CmhUzXGVlA/ef4/Nm27B+ef8kJ7du3LhuWexeNHXNayXxo+j2RYt2LbddhlGmY2I4JGbz+GFey/ktD77ANC8aSM+nps8eeXjuQto3jT5ddpqi804ukdHhvxrbGbxFgMJ+vc5ksO7d+OeO28HYPsOO/HkYyMAGDn8IT6a9WGWIVaL6pirXl3ySZw/BzYFfgrsA/wQOK2yN0kKSdfmvL5A0mVrnDNV0v/Me5d0nqS3JL0maZqk6yRtlEesmVi5ciWvv/oKJ506iMfHTGSTBpty8w1Xf3V8+EMPcMz3j88wwuwcdOr17H3CVfQ+52bO7Lsf+3Rq9z/nlM8KvvoX3+fXNwynkmnCNd6wx0fz5HMTuedfI7jz9luZ8MJYrht8G0PvuI3Du3dj0cKFbLRRzVvZsaxMeW3FoNI+zoiYmO5+ydeLGedjGdBH0hURMXfNg5J2JJkDv5+kTSNiUVp+FslY0W4R8YWkesB5wCYk8+WLTsutWtFyq1bs3iWZzXXE0cdyyw3XAElSfeLR4Tw66sUsQ8zMR5/OB+DTzxcy4tlX2WPntsz57Eu2bNaYj+cuYMtmjb9qlnfaqQ13XZl0dzRt0pDD9t2ZlStX88iYVzOLPwstt0qWiWy2RXN69jqGqS9P4qyfnMc/H34MgHdmzmDUU49nGWKVE8XTDM9HRQPgh1HBTaCI6FPJZ68EhpDUWC9Zy/H+wN3AjiQrMP8jLb8E2D9dDoqIWA5cWcm1MtW8xZa0bNWad96eQbv22/PC86Npv8OOAIx77lnatd+elq1aZxzlhtdg43qUlYmFi5fRYON6HLxXB/445HEefe41BhzVlWv+/jQDjurKyDQx7tjrsq/eO+S3A3h87Ou1LmkuXrSI1atX07BRIxYvWsRzzz7Dzy/8FXM/nUOzLZqzevVqbrjmSk469YdZh1q1iqgZno+KapyDq+Dz/wK8KulPaznWl2S5pw7AT0gWD2kMNIyI/8v3Aunqz4MAWrXO7vlyl195PT898xRWrFhOm2225ZrBQwAY8fADHF1Lbwo1b9qI+69L/g9et04d7n98Mk+/+CZTpv+Xe646jYG99+K/s+cx4MK/ZRxp8fj00084fUDSrbNq1Up6f78fBx58GLffehN33p48seaIXr3pe+LAij6mJJXScCRVV3+SpIUR0VDS5SRN7CUkSfEySV2AGyJin3Tpp/eBXUlqqe9HxHfSzziMZNHkJsAJEVFhe3fX3TrHo8/WziZxvrY/6PysQyh674y+LusQil7PA/di2itTqizTNd9ul+h79b/yOndwn52mVLYCfHXbEGtr/pnkKXKb5pT1BzpIeo/kWSCNge9HxAJgYfoYTyLiyYjYjWT6Z83rDTczIBnDWVULGW8I1Z4404e7PUD6KM50aNPxwPciom1EtCXp4+yfvuUK4JacFZxFzowlM6uZ6pbltxWDfFeAR1L9iFi2nte5Fjgn3d8PmBURH+Ucfx7YSVJL4BaS2ulEScuAhcALJGuAmlkNlIzRLI7aZD7ymau+J8nS8psBbSR1BM6IiJ9U9L6IaJiz/wnQIOdwtzXOXQVsmVN0dbqZWS1RJEM085JPxfdGoBfpU98iYhpwYHUGZWa1TynNHMqnqV4WEe+vUY1eVU3xmFktVOBz1TOXT+L8IG2uRzp06CfAjOoNy8xqmyK575OXfBLn2STN9TbAJ8AzaZmZWZWQVJWPB652+cxVn0Mezxk2M/s2Sqilntdd9b+yljnrETGoWiIys1qphCqceTXVn8nZ3xg4FvigesIxs9qoxt0cioj7c19LuhsYV20RmVmtVEJ5M/+ZQzm2BVpUdSBmVouphjXVJX3O132cZcA84KLqDMrMahcBdUqoyllh4kwX2OgIzEqLVkdtf66BmVWLUqpxVjjmNE2Sj0XEqnRz0jSzalHTlpWbKmn3ao/EzGqt5K56flsxqOiZQ3UjYiWwOzBJ0jvAIpKfMSKi0waK0cxquiJawCMfFfVxvgR0Ao7eQLGYWS1WU8ZxCiAi3tlAsZhZLSWgTgmt8lFR4txC0nnrOhgRfqKVmVURUUbNqHHWARpCCf00ZlaSkoe1ZR1F/ipKnLMj4vINFomZ1V5VeMdc0tbAXSQzHAMYEhE3SNocuB9oC7wHHB8Rn6fj1W8AjgAWA6dExMsVXaOiXoUSyv9mVurKpLy2PKwEzo+InUieb/ZjSTuRzHgcFRHtgVF8PQOyJ9A+3QaRPDCy4lgrOHZQPhGamX1b5U31qnjmUETMLq8xRsSXwJtAK5LHkA9NTxsK9E73jwHuisQEoEn6xN11WmdTPX0eupnZBlHACvDNJE3OeT0kIoas7URJbUnGok8EWkTE7PTQx3y9WFErvrlU5odp2WzWYX1WRzIzq1KioGcOzY2ILpV+ptQQeAj4WUQsyJ2uGREhab2nkJfQyCkzq7FUtXPVJW1EkjTvjYiH0+JPypvg6b9z0vJZwNY5b2/N1wsbrZUTp5kVBeW5Vfo5SXa9A3hzjfHmI4CB6f5AYHhO+clKdAPm5zTp18pNdTPLXBU/OmMf4CTgNUlT07JfAVcCD0g6HXgfOD499hjJUKSZJMORTq3sAk6cZlYUqiptRsS4Cj7uf0YLpctl/riQazhxmlkREGXFsmZcHpw4zSxzBd5Vz5wTp5kVhWJZ3T0fTpxmVhRKJ23WsMRZt45o2rBe1mEUtZnPXpt1CEXPj9bKgFzjNDMriPs4zczWQ015dIaZ2QZTQnnTidPMspc01UsnczpxmllRcI3TzKwgQq5xmpkVxjVOM7MCSFCnhDKnE6eZFYUSyptOnGZWHNzHaWZWgGQh46yjyJ8Tp5kVBdc4zcwK5D5OM7MCCN9VNzMrkAfAm5kVRm6qm5kVrITyphOnmWWvip+rXu2cOM2sKJRO2nTiNLMi4WcOmZkVqITyphOnmRWHEsqbTpxmViRKKHM6cZpZ5oTnqpuZFUZeHcnMrHBOnGZmhfBcdTOzgpXScKSyrAOoKT784AN6HtqDzh13pstuu/CXm24AYN68efTqeSi77rQ9vXoeyueff55xpNmZP/8LBg3szwFdd6V7145MeWkCv/t/F3NA1105eN8unH7S8cyf/0XWYWaqtn5HKmArBk6cVaRO3br88aprmDJtOqPHjmfIrTfz5ptvcO3VV9K9Rw9efWMG3Xv04Nqrr8w61Mz85uLz6X7QITw38VWeGjuJ7XbowP7dezDqhZd5ZtxkvtuuPYOvvzrrMDNVq7+jEsqcTpxVpGXLluy+eycAGjVqxA4dduSjWbN49JERnDhgIAAnDhjIyBHDswwzMwsWzGfii+Pof9KpANSrV4/NNmvCAT0OoW7dpMeoU5c9mf3Rh1mGmana/h2VSXltxcCJsxq8/957TJv2Cnvs2ZU5cz6hZcuWAGy55ZbMmfNJxtFl44P332PzZltw3jk/5LADunLBT89i8aJF3zjn/nuHcuDBh2UUYfZq+3dUVRVOSX+TNEfS6zllm0t6WtLb6b/fScsl6UZJMyW9KqlTPrFWa+KU1FrS8DTYdyTdIKlezvE/S5olqWyN9x0u6SVJb0maKul+SW2qM9aqsnDhQk7odxx/uuZ6Gjdu/I1jkkpqIYOqtHLlSl6f9gonnTqIJ5+bSIMGm/KXP3/d5Lzx2iupU7cufX7QP8Mos1Wrv6Oq7eS8Ezh8jbKLgFER0R4Ylb4G6Am0T7dBwC35XKDaEqeSDPEw8O802O2BhsAf0uNlwLHAB8ABOe/bBbgJGBgRHSJiN+BeoG11xVpVVqxYwQl9j6NvvxM4pncfAJo3b8Hs2bMBmD17Nlts0TzLEDPTcqtWtNyqFZ267AnAkcccy2uvTgXggX/cxTNPPs7g2+6stX9YwN+R8vxfZSLieWDeGsXHAEPT/aFA75zyuyIxAWgiqWVl16jOGmcPYGlE/B0gIlYBPwdOk9QA6A5MJ8nwuX9Cfwn8MSLeLC+IiBHpl1G0IoKzzzyDHTp04Kc/O++r8iN6HcW99yT/ve69ZyhHHnV0ViFmqnmLLdmqVWveeXsGAOOeG037HXZk9DNPccuN1/H3fzzIJg0aZBxltmrzdySS4Uj5bEAzSZNztkF5XKJFRMxO9z8GWqT7rUgqb+U+TMsqVJ3jOHcGpuQWRMQCSf8FtiNJlv8EhgN/lLRRRKxI33dNvhdJv7RBAFu3ya41P/7FF/jnvXez8y7fo9seuwNw2eV/4PxfXMRJJ/Tlrr//ja3bbMPd/7g/sxiz9rurrucnZ57C8uXL2abttlw7eAhHHrQPy5cto3+fI4Hk5seV1w3OONLs1ObvqIB69NyI6LK+14mIkBTr+37IdgD8EcB5EfGlpInAYcDI3BMkNSXpj2gADImI/0moETEEGALQqXOXb/VlfBt777Mvi5atXuuxx558ZgNHU5x2/l5HHnv2xW+UvTDljYyiKU61+Tuq5i6ITyS1jIjZaVN8Tlo+C9g657zWaVmFqrOp/gbQObdAUmOgDbAt0AR4TdJ7wL583VyfDnQCiIjP0j7OIST9o2ZWQxXQVF8fI4CB6f5AkpZuefnJ6d31bsD8nCb9OlVn4hwFNJB0MoCkOsC1JHe8+gJnRETbiGhLkkgPSfs+/wRcImnHnM+qmR07ZvaVKhyO9E9gPLCDpA8lnQ5cSZJj3gYOTl8DPAa8C8wE/gr8KJ9Yq62pnvYjHAvcLOlSkiT9GHA58H/AWTnnLpI0DjgqIu6XdC5wV1pDnQv8F/hNdcVqZkWgilrqEbGu8VoHreXcAH5c6DWqtY8zIj4AjlrLoc3Xcm6fnP1HgUerMTQzKyJeyNjMrFDfrv9yg3PiNLOi4MRpZlYQL2RsZlYw1zjNzApQREtt5sWJ08yKQwllTidOMysK7uM0MyuQn6tuZlYIj+M0M1sfpZM5nTjNLHPlCxmXCidOMysKJZQ3nTjNrDi4xmlmVqBSegidE6eZFYXSSZtOnGZWBL7lYzE2OCdOMysKnjlkZlao0smbTpxmVhxKKG86cZpZMRBlJdTJ6cRpZpkrtZlD1flcdTOzGsk1TjMrCqVU43TiNLOi4OFIZmaF8AB4M7PClNrNISdOMysKbqqbmRXINU4zswKVUN504jSzIlFCmdOJ08yKQin1cSoiso6hykj6FHg/6zhyNAPmZh1EkfN3VLli/I62iYgtqurDJD1B8nPmY25EHF5V114fNSpxFhtJkyOiS9ZxFDN/R5Xzd1R8PFfdzKxATpxmZgVy4qxeQ7IOoAT4O6qcv6Mi4z5OM7MCucZpZlYgJ04zswI5cVpRkUppxrLVVk6cG4CTQeUkbSmpTrjTfZ0ktZfUWlKDrGOp7Zw4q5mk/YDTJXXNOpZiJelw4G/AzyVtnHU8xSj9jv4O9AG2zjicWs9z1auRpJ7A74DbgGVrHJNrVyCpF/B74BySqXRL03J/PylJRwJ/As4AXo2IRWn5puX7tmF5OFI1kdQd+CswICIm5pTvERGT0v1anRwktQIeBn4WEeNzys8C3gDGR8SKrOIrBmmz/D7ghogYlVN+AdAAuDMi/ptVfLWVm+rVpwNwVURMlFQGIOka4FpJFwPU5qSZqgPMB6ZKqgMg6XLgCpIa6H7uH6YusAXwSXmBpN8APwIaA2dJappRbLWWE2f12Q7YGyAiVqd9nXsCVwLfldQ3y+CKxGpgOcnfkFWSvgNMjYjvAKOAE4FNsgwwC7l/LCJiATAJaJ1zyr8i4rvA7UBTkpqnbUBOnFWsvOYEPAQslNQBICLGRsT+EfEY8BlQP6sYsyTpq371iPgw3f13+vpzYERaNg+oR+38Ha23xuu5wNmS2gJExBtp+a4kCXXJBovMgNr5S1ktJHVco2gWSW2gn6Rdcs47HuhlI8V9AAAHNUlEQVQGjNuA4RUFSfsCN0natvwPTET0AupLeiptcjaUNAD4BfDHiFiYYcgbnKQ9SbouDpP0PYCIuByYA9wo6ThJe0g6HfgVcEFEFNtanTWebw5VAUlbAm8DzwJTgPsiYoak7wK/AZYCmwNTgZOBPhExPat4syLpVmAQ8AIwEng/Iu5Lj/0TaETSp7cp8KOIeC2rWLOS/tG4C7iJ5A/v/0XEpemxs4CdSfrPvwAur43fUTFw4qwCkjYD7gW+BEYDv0638SQr0m8HHE1SC50YETMyCjVTkrYALgH+S3JTaCAwA7g5Il5Om/HNgKUR8UV2kWZL0kPAf0gS6OUkfcETgcERsSLtA90oIpZnGGat5qb6tyCps6RuJAnzKmB7kr7No9JtJHAhyRjOP0XE3bUtaUpqI2nT9OVyIID5EXEHcB1J8jxf0mSgbUR8XNuSZjobqGnaQoFkMsBGEfEWSfI8BOgBvJnWSMucNLPlAfDrKZ3J8TtgMMmNjBeBZ4CWJDWErsDFQD+S4SS/Tc+rNSS1AM4HPpB0a0TMlzQMuEZSE+B0oF9EPCTpD5kGmxFJxwAXkQw3ailpOPAIsHf6nfQGTomIRyT1B56LiFXZRWzgpvp6kXQAyVCQE8oHs6flPycZf1gGnBsRI9LaVt2ImJ9NtNlJx6+eAHQC3iMZrL1A0qXAz4GTIuLRDEPMlKQDSWaV9QfeAVoAd5NMrfws/fenae3ciogT53qQdB6wKiJuSPvlVpUPZpd0P/BxRJwrqW5ErMw02AxIak/SnPxP2h/XCzgUmEkym6oj8OeI6JqeXxYRqzMLOCOSLiHpthgsaeOIWJoOORoGjAEWAA9HxLTa+h0VK/dxFiBnYPK2JM1vSJNmzvjNV4FW6Uo/tTFpNiW5sTFW0o+BM4FHSW5uNAZOS6dXvi/pFkgmCGQVbxZyfo9a8/UjcZelvzPvkYy86ADsSNIVVOu+o2LnPs4C5EyRHAb8SlLniJiSNknLjzUCDicZfvRpBmFmKiI+k3QwSX9vGUnt8n5gIcnNod0krSBZtGJWZoFmKOf36EHgopzfo5C0EclQo89JZk5tmVWctm5OnOtnIskA9r6SiIgpAJJOBL4H7BYRtS5plouIZyUdBtxIkjhbkNwV7kdy02w74IDadvd8LSaQjGntmy74MhlYLWkvktpo44iYnWmEtlbu41xP6co+pwMHAZNJpr0dB/zAg5IT6XJo1wPdImJeOhd9I6BB2iSt9dLfozNI/rCMJ6mVHwf0j4hpWcZm6+bE+S1I2gToDBwMzAZG17ZxmpVJ1yS9AdgrIj7LOp5ilP4edQEOI5mX/nhE/CfbqKwiTpxW7dKxipcBnX2Tw2oCJ07bICQ1rG0LdljN5cRpZlYgj+M0MyuQE6eZWYGcOM3MCuTEaWZWICfOWkbSKklTJb0u6V/p42fX97O6SxqZ7h8t6aIKzm0i6UfrcY3LlDwKN6/yNc65U9JxBVyrraTXC43Rah8nztpnSUTsFhG7kMxSOSv3oBIF/15ExIiIuLKCU5qQPNLWrOQ5cdZuY4Ht0prWfyTdBbwObC3pUEnjJb2c1kwbQrKAs6S3JL0M9Cn/IEmnSBqc7reQNEzStHTbm+SxyO3S2u7V6Xm/kDRJ0quSfpvzWZdImiFpHLBDZT+EpB+mnzNN0kNr1KIPljQ5/bxe6fl1JF2dc+0zv+0XabWLE2ctla4j2hMon1ffnuTZPzsDi0iemXRwRHQimYt/nqSNSdbTPIpkqum6Vu65kWSl8o4kixhPJ1nl/J20tvsLSYem19wT2A3oLGl/SZ1JFgPZDTgC2COPH+fhiNgjvd6bJGsIlGubXuNI4Nb0ZzidZB3MPdLP/6GkbfO4jhng1ZFqo00kTU33xwJ3AFuRPHFyQlreDdgJeCFdOrIeyQIUHUieuvg2gKR7SJ5auaYeJGtKkj7mYX66wEeuQ9PtlfR1Q5JE2ggYFhGL02uMoHK7SPo9SXdAQ+DJnGMPpNM835b0bvozHArsmtP/uVl6ba8zYHlx4qx9lkTEbrkFaXJclFsEPB0R/dc47xvv+5YEXBERt61xjZ+tx2fdCfROV0o/Beiec2zNqXGRXvsnEZGbYElXXzerlJvqtjYTgH0kbQcgaVNJ2wNvAW0ltUvP67+O948Czk7fW0fJ45O/JKlNlnsSOC2n77SVpObA80BvSZtIakTSLVCZRsDsdBHgE9c49gNJZWnM3yVZnf5J4Oz0fCRtr6+fxGlWKdc47X9ExKdpze2fkuqnxb+OiBmSBgGPSlpM0tRvtJaPOBcYIul0YBVwdkSMl/RCOtzn8bSfc0dgfFrjXQgMSJ+vfj8wDZgDTFrL56/pUpLFpT9N/82N6b/ASySP7Tgrfa7P7SR9ny8rufinJE+TNMuLF/kwMyuQm+pmZgVy4jQzK5ATp5lZgZw4zcwK5MRpZlYgJ04zswI5cZqZFej/A/QY7Tb/RQkdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnf_matrix = confusion_matrix(y_test, prd)\n",
    "np.set_printoptions(precision=2)\n",
    "class_names = ['CAG','NAG','OAG']\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cnf_matrix, classes=class_names,\n",
    "                      title='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_model(length, vocabulary_size,embedding_matrix):\n",
    "    input1 = Input(shape=(length,))\n",
    "    embedding1 = Embedding(vocabulary_size,100, weights=[embedding_matrix], trainable=False)(input1)\n",
    "    conv1 = Conv1D(filters=32, kernel_size=3, activation='relu')(embedding1)\n",
    "    conv1 = Dropout(0.5)(conv1)\n",
    "    capsule1 = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=3,share_weights=True)(conv1)\n",
    "    #flat1 = Flatten()(capsule1)\n",
    "    flat1 = Bidirectional(LSTM(300))(capsule1)\n",
    "\n",
    "    input2 = Input(shape=(length,))\n",
    "    embedding2 = Embedding(vocabulary_size,100, weights=[embedding_matrix], trainable=False)(input2)\n",
    "    conv2 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding2)\n",
    "    conv2 = Dropout(0.5)(conv2)    \n",
    "    capsule2 = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=3,share_weights=True)(conv2)\n",
    "    #flat2 = Flatten()(capsule2)    \n",
    "    flat2 = Bidirectional(LSTM(300))(capsule2)\n",
    "    \n",
    "    input3 = Input(shape=(length,))\n",
    "    embedding3 = Embedding(vocabulary_size,100, weights=[embedding_matrix], trainable=False)(input3)\n",
    "    conv3 = Conv1D(filters=32, kernel_size=5, activation='relu')(embedding3)   \n",
    "    conv3 = Dropout(0.5)(conv3)    \n",
    "    capsule3 = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=3,share_weights=True)(conv3)\n",
    "    #flat3 = Flatten()(capsule3)        \n",
    "    flat3 = Bidirectional(LSTM(300))(capsule3)\n",
    "    merged = concatenate([flat1, flat2, flat3])\n",
    "    \n",
    "    dense = Dense(32,activation='relu')(merged)\n",
    "    output = Dense(3, activation='softmax')(dense)\n",
    "    model = Model(inputs=[input1,input2,input3], outputs=output)\n",
    "    model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_5 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 150, 100)     2441200     input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 150, 100)     2441200     input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 150, 100)     2441200     input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 148, 32)      9632        embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 147, 32)      12832       embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 146, 32)      16032       embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 148, 32)      0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 147, 32)      0           conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 146, 32)      0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "capsule_4 (Capsule)             (None, 10, 16)       5120        dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "capsule_5 (Capsule)             (None, 10, 16)       5120        dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "capsule_6 (Capsule)             (None, 10, 16)       5120        dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 600)          760800      capsule_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 600)          760800      capsule_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 600)          760800      capsule_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 1800)         0           bidirectional_1[0][0]            \n",
      "                                                                 bidirectional_2[0][0]            \n",
      "                                                                 bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 32)           57632       concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 3)            99          dense_3[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 9,717,587\n",
      "Trainable params: 2,393,987\n",
      "Non-trainable params: 7,323,600\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = define_model(150, vocabulary_size,embedding_matrix)\n",
    "model.load_weights('reproduce_caps_bi.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 53  59  30]\n",
      " [128 423  79]\n",
      " [ 36  32  76]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.24      0.37      0.30       142\n",
      "          1       0.82      0.67      0.74       630\n",
      "          2       0.41      0.53      0.46       144\n",
      "\n",
      "avg / total       0.67      0.60      0.63       916\n",
      "\n",
      "Accuracy :  0.6026200873362445\n",
      "F1 :  0.6270175331585358\n"
     ]
    }
   ],
   "source": [
    "pred = model.predict([data_test,data_test,data_test])\n",
    "prd = np.argmax(pred,axis = 1)\n",
    "print(confusion_matrix(y_test,prd))\n",
    "print(classification_report(y_test,prd))\n",
    "print(\"Accuracy : \",accuracy_score(prd,y_test))\n",
    "print(\"F1 : \",f1_score(y_test, prd, average='weighted', ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}