ANALYSIS_AND_EVALUATION.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "import numpy as np\n",
    "from gensim.models import KeyedVectors\n",
    "from SKIP_THOUGHTS_ARCH import SkipThoughtsNN\n",
    "import VOCABULARY_EXPANSION\n",
    "from VOCABULARY_EXPANSION import VocabularyExpansionNN\n",
    "from VOCABULARY_EXPANSION import GetSTModelWordEmbedding\n",
    "from VOCABULARY_EXPANSION import GetW2VModelWordEmbedding\n",
    "import SEMANTIC_RELATEDNESS\n",
    "from SEMANTIC_RELATEDNESS import SemanticRelatednessNN\n",
    "from SEMANTIC_RELATEDNESS import GetEmbedding\n",
    "from SEMANTIC_RELATEDNESS import GetThought\n",
    "from SEMANTIC_RELATEDNESS import GetSTModelWordEmbeddingSize\n",
    "from SEMANTIC_RELATEDNESS import GetSTModelThoughtSize\n",
    "\n",
    "def LoadModelsAndUtils ( loc = 'MODELS' ) :\n",
    "    global ST_MODEL, VE_MODEL, SR_MODEL, WORD_2_VEC_MODEL, WORD_TO_ID, ID_TO_WORD, ST_VOCAB, W2V_VOCAB\n",
    "    ST_MODEL = torch.load(loc+'/SKIP_THOUGHTS_NN')\n",
    "    ST_MODEL.eval()\n",
    "    VE_MODEL = torch.load(loc+'/VOCABULARY_EXPANSION_NN')\n",
    "    VE_MODEL.eval()\n",
    "    SR_MODEL = torch.load(loc+'/SEMANTIC_RELATEDNESS_NN')\n",
    "    SR_MODEL.eval()\n",
    "\n",
    "    limit = int(1e7) # 10 Million words\n",
    "    WORD_2_VEC_MODEL = KeyedVectors.load_word2vec_format(loc+'/GoogleNews-vectors-negative300.bin.gz', \n",
    "                                                         binary = True, limit = limit)\n",
    "    \n",
    "    file = open(loc+'/UTILS/SKIP_THOUGHTS_NN_WORD2ID', 'rb')\n",
    "    WORD_TO_ID = pickle.load(file)\n",
    "    file.close()\n",
    "\n",
    "    file = open(loc+'/UTILS/SKIP_THOUGHTS_NN_ID2WORD', 'rb')\n",
    "    ID_TO_WORD = pickle.load(file)\n",
    "    file.close()\n",
    "\n",
    "    file = open(loc+'/UTILS/VOCAB_EXPANSION_NN_ST_VOCAB', 'rb')\n",
    "    ST_VOCAB = pickle.load(file)\n",
    "    file.close()\n",
    "\n",
    "    file = open(loc+'/UTILS/VOCAB_EXPANSION_NN_W2V_VOCAB', 'rb')\n",
    "    W2V_VOCAB = pickle.load(file)\n",
    "    file.close()\n",
    "\n",
    "    VOCABULARY_EXPANSION.ST_MODEL = ST_MODEL\n",
    "    VOCABULARY_EXPANSION.WORD_2_VEC_MODEL = WORD_2_VEC_MODEL\n",
    "    VOCABULARY_EXPANSION.WORD_TO_ID = WORD_TO_ID\n",
    "    VOCABULARY_EXPANSION.ID_TO_WORD = ID_TO_WORD\n",
    "    VOCABULARY_EXPANSION.ST_VOCAB = ST_VOCAB\n",
    "    VOCABULARY_EXPANSION.W2V_VOCAB = W2V_VOCAB\n",
    "\n",
    "    SEMANTIC_RELATEDNESS.ST_MODEL = ST_MODEL\n",
    "    SEMANTIC_RELATEDNESS.ST_VOCAB = ST_VOCAB\n",
    "    SEMANTIC_RELATEDNESS.W2V_VOCAB = W2V_VOCAB\n",
    "    SEMANTIC_RELATEDNESS.VE_MODEL = VE_MODEL\n",
    "\n",
    "def LoadTestDataset ( ) :\n",
    "    global TEST_SICK_DATA\n",
    "    file = open('DATA/SICK_TEST_DATA', 'rb')\n",
    "    TEST_SICK_DATA = pickle.load(file)\n",
    "    file.close()\n",
    "\n",
    "def GetSentencePairVector ( sent1 , sent2 ) :\n",
    "    thought1 = GetThought(sent1)\n",
    "    if thought1 is None : return None\n",
    "    thought2 = GetThought(sent2)\n",
    "    if thought2 is None : return None\n",
    "    prod = thought1 * thought2\n",
    "    diff = torch.abs(thought1 - thought2)\n",
    "    vec = torch.cat([prod, diff], 0)\n",
    "    return vec\n",
    "\n",
    "def GetSimilarityRelationScore ( sent1 , sent2 , precision = None ) :\n",
    "    vec = GetSentencePairVector(sent1, sent2)\n",
    "    if vec is None : return None\n",
    "    inp = torch.stack([vec])\n",
    "    score_dist = SR_MODEL(inp)[0]\n",
    "    possible_scores = torch.tensor([1., 2., 3., 4., 5.])\n",
    "    score = torch.dot(possible_scores, score_dist).item()\n",
    "    if not precision is None : return round(score, precision)\n",
    "    else : return score\n",
    "\n",
    "def PredictScores ( dataset ) :\n",
    "    true_scores = list()\n",
    "    predicted_scores = list()\n",
    "    for sample in TEST_SICK_DATA :\n",
    "        _, sent1, sent2, true = sample.values()\n",
    "        pred = GetSimilarityRelationScore(sent1, sent2)\n",
    "        if pred is None : continue\n",
    "        true_scores.append(true)\n",
    "        predicted_scores.append(pred)\n",
    "    true_scores = np.array(true_scores)\n",
    "    predicted_scores = np.array(predicted_scores)\n",
    "    return predicted_scores, true_scores\n",
    "\n",
    "def GetMSE ( x , y ) :\n",
    "    return ((x - y) ** 2).mean()\n",
    "\n",
    "def GetPearsonsCoeff ( x , y ) :\n",
    "    x_mean = x.mean()\n",
    "    y_mean = y.mean()\n",
    "    numer = (x - x_mean) * (y - y_mean)\n",
    "    numer = numer.sum()\n",
    "    denom = ((x - x_mean)**2).sum() * ((y - y_mean)**2).sum()\n",
    "    denom = denom ** 0.5\n",
    "    return numer / denom\n",
    "\n",
    "def GetSpearmansCoeff ( x , y ) :\n",
    "    rank_x = list(enumerate(x))\n",
    "    rank_x.sort(key = lambda x : -x[1])\n",
    "    rank_x = [first for first, second in rank_x]\n",
    "    rank_x = list(enumerate(rank_x))\n",
    "    rank_x.sort(key = lambda x : x[1])\n",
    "    rank_x = [first + 1 for first, second in rank_x]\n",
    "    x = np.array(rank_x)\n",
    "    \n",
    "    rank_y = list(enumerate(y))\n",
    "    rank_y.sort(key = lambda y : -y[1])\n",
    "    rank_y = [first for first, second in rank_y]\n",
    "    rank_y = list(enumerate(rank_y))\n",
    "    rank_y.sort(key = lambda y : y[1])\n",
    "    rank_y = [first + 1 for first, second in rank_y]\n",
    "    y = np.array(rank_y)\n",
    "    \n",
    "    n = x.shape[0]\n",
    "    coeff = (x - y) ** 2\n",
    "    coeff = coeff / (n * (n*n - 1))\n",
    "    coeff = 6 * coeff.sum()\n",
    "    return coeff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "LoadModelsAndUtils()\n",
    "LoadTestDataset()\n",
    "SEMANTIC_RELATEDNESS.WORD_EMBED_SIZE = GetSTModelWordEmbeddingSize()\n",
    "SEMANTIC_RELATEDNESS.THOUGHT_SIZE = GetSTModelThoughtSize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation On SICK Dataset\n",
    "The performance of the *Skip-Thoughts Model* with *expanded vocabulary* is checked on *SICK* dataset. Given two sentences, our goal is to produce a score of how semantically related these sentences are and hence compare the score generated by our model with the human generated score given in the *SICK* dataset. The evaluation metrics used are *Mean Sqaured Error*, *Pearson’s coefficient r* and *Spearman’s coefficient ρ*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_scores, true_scores = PredictScores(TEST_SICK_DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7858500278744335"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GetMSE(predicted_scores, true_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5541061984237643"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GetPearsonsCoeff(predicted_scores, true_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5016105357325269"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GetSpearmansCoeff(predicted_scores, true_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analogical Tasks\n",
    "The performance of the *Skip-Thoughts Model* with *expanded vocabulary* is checked on how well is it able to quantify the semantic relatedness between two sentences. Given two sentences, a score between 1 and 5 should be generated that is indicative of the extent of relationship between the sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.6"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A little girl is looking at a woman in costume\"\n",
    "sentence_2 = \"A young girl is looking at a woman in costume\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.6"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A little girl is looking at a woman in costume\"\n",
    "sentence_2 = \"The little girl is looking at a man in costume\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.3"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A little girl is looking at a woman in costume\"\n",
    "sentence_2 = \"A little girl in costume looks like a woman\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.3"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A sea turtle is hunting for fish\"\n",
    "sentence_2 = \"A sea turtle is hunting for food\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.2"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A sea turtle is not hunting for fish\"\n",
    "sentence_2 = \"A sea turtle is hunting for food\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.9"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A man is driving a car\"\n",
    "sentence_2 = \"The car is being driven by a man\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.5"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"There is no man driving the car\"\n",
    "sentence_2 = \"A man is driving a car\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.8"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A large duck is flying over a rocky stream\"\n",
    "sentence_2 = \"A duck, which is large, is flying over a rocky stream\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.9"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A large duck is flying over a rocky stream\"\n",
    "sentence_2 = \"A large stream is full of rocks, ducks and flies\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.9"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A person is performing acrobatics on a motorcycle\"\n",
    "sentence_2 = \"A person is performing tricks on a motorcycle\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.9"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"The performer is tricking a person on a motorcycle\"\n",
    "sentence_2 = \"A person is performing tricks on a motorcycle\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.5"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"Someone is pouring ingredients into a pot\"\n",
    "sentence_2 = \"Someone is adding ingredients to a pot\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.6"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"Nobody is pouring ingredients into a pot\"\n",
    "sentence_2 = \"Someone is pouring ingredients into a pot\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.8"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"Someone is pouring ingredients into a pot\"\n",
    "sentence_2 = \"A man is removing vegetables from a pot\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.6"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"I do not have enough money for this trip\"\n",
    "sentence_2 = \"His parents have more than enough money for this trip\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.3"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"I love dogs and she hates cats\"\n",
    "sentence_2 = \"She loves dogs and I hate cats\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"I love dogs and she hates cats\"\n",
    "sentence_2 = \"She loves badminton and I hate tennis\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.8"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A man is playing the guitar loudly\"\n",
    "sentence_2 = \"A flute isn't being played by a man\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.8"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"I love driving\"\n",
    "sentence_2 = \"I hated driving in the past and now I don't\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"An onion is being chopped\"\n",
    "sentence_2 = \"Someone is holding a hedgehog\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.7"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"The girl is carrying a sign and a group of people is following her\"\n",
    "sentence_2 = \"A young girl in a blue shirt is walking on the sidewalk and holding up a pink sign\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"There is no woman beating two eggs in a bowl and using a whisk made of wire\"\n",
    "sentence_2 = \"A man wearing a dyed black shirt is sitting at the table and laughing\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_1 = \"A woman is eating a piece of meat\"\n",
    "sentence_2 = \"A man is loading a rifle with bullets\"\n",
    "GetSimilarityRelationScore(sentence_1, sentence_2, 1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}