how_to_use_WordEmbedder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How To Use The Word Embedder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": "[nltk_data] Downloading package punkt to\n[nltk_data]     C:\\Users\\shreyanshchordia\\AppData\\Roaming\\nltk_data...\n[nltk_data]   Package punkt is already up-to-date!\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "True"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [\n",
    "    'Hey how are you?',\n",
    "    'I am great.',\n",
    "    \"What's the issue in it?\",\n",
    "    'We survived the  day!',\n",
    "    'I am tired.',\n",
    "    'I need to rest.',\n",
    "    'We need to resolve our differences',\n",
    "    'How far are you going to go with this, huh?'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizing the sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_sentences = [word_tokenize(sentence) for sentence in sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "['Hey', 'how', 'are', 'you', '?']\n['I', 'am', 'great', '.']\n['What', \"'s\", 'the', 'issue', 'in', 'it', '?']\n['We', 'survived', 'the', 'day', '!']\n['I', 'am', 'tired', '.']\n['I', 'need', 'to', 'rest', '.']\n['We', 'need', 'to', 'resolve', 'our', 'differences']\n['How', 'far', 'are', 'you', 'going', 'to', 'go', 'with', 'this', ',', 'huh', '?']\n"
    }
   ],
   "source": [
    "for sentence in tokenized_sentences:\n",
    "    print(sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Embedder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from WordEmbedder import Embedder, token_seq_to_num_seq, num_seq_to_token_seq, generate_vocabulary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### As simple as it could be!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb = Embedder(dimensions=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "\n[[ 0.54623    1.2042    -1.1288    -0.1325     0.95529    0.040524\n  -0.47863   -0.3397    -0.28056    0.71761   -0.53691   -0.0045698\n   0.73217    0.12101    0.28093   -0.088097   0.59733    0.55264\n   0.056646  -0.50247   -0.63204    1.1439    -0.31053    0.1263\n   1.3155    -0.52444   -1.5041     1.158      0.68795   -0.85051\n   2.3236    -0.41789    0.44519   -0.019216   0.28969    0.53258\n  -0.023008   0.58958   -0.72397   -0.85216   -0.17761    0.14432\n   0.40658   -0.52003    0.09081    0.082961  -0.021975  -1.6214\n   0.34579   -0.010919 ]\n [ 0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.         0.         0.         0.         0.\n   0.         0.       ]]\n<NDArray 2x50 @cpu(0)>\n"
    }
   ],
   "source": [
    "embedder = emb.get_embedder()\n",
    "# embedder returns embeddings of known words \n",
    "# for unknown (out of vocabulary) words it returns array of zeros\n",
    "print(embedder['beautiful', 'ffnceo'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating vocabulary for the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab, vocab_dict = generate_vocabulary(tokenized_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Vocabulary:\n['?', 'I', '.', 'to', 'are', 'you', 'am', 'the', 'We', 'need', 'Hey', 'how', 'great', 'What', \"'s\", 'issue', 'in', 'it', 'survived', 'day', '!', 'tired', 'rest', 'resolve', 'our', 'differences', 'How', 'far', 'going', 'go', 'with', 'this', ',', 'huh']\n\nVocabulary Dictionary that maps words to the count of their occurances:\n{'?': 3, 'I': 3, '.': 3, 'to': 3, 'are': 2, 'you': 2, 'am': 2, 'the': 2, 'We': 2, 'need': 2, 'Hey': 1, 'how': 1, 'great': 1, 'What': 1, \"'s\": 1, 'issue': 1, 'in': 1, 'it': 1, 'survived': 1, 'day': 1, '!': 1, 'tired': 1, 'rest': 1, 'resolve': 1, 'our': 1, 'differences': 1, 'How': 1, 'far': 1, 'going': 1, 'go': 1, 'with': 1, 'this': 1, ',': 1, 'huh': 1}\n"
    }
   ],
   "source": [
    "print(f'Vocabulary:\\n{vocab}\\n\\nVocabulary Dictionary that maps words to the count of their occurances:\\n{vocab_dict}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating embedding matrix for the vocabulary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Embedding Matrix is required when you use the Embedding Layer from tensorflow.keras.layers on Pre-Trained word embeddings.\n",
    "\n",
    "The Layer demands for a numpy matrix that contains embeddings of all the words in the chosen vocabulary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix, id2token, token2id = emb.get_embedding_matrix(vocab, num_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Embeddings of the first word of vocabulary:\n[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n 0. 0.]\n\nID-TO-TOKEN Mapper:\n{0: '<oov>', 1: '?', 2: 'i', 3: '.', 4: 'to', 5: 'are', 6: 'you', 7: 'am', 8: 'the', 9: 'we'}\n\nTOKEN-TO-ID Mapper:\n{'<oov>': 0, '?': 1, 'i': 2, '.': 3, 'to': 4, 'are': 5, 'you': 6, 'am': 7, 'the': 8, 'we': 9}\n"
    }
   ],
   "source": [
    "print(f\"Embeddings of the first word of vocabulary:\\n{embedding_matrix[0]}\\n\\nID-TO-TOKEN Mapper:\\n{id2token}\\n\\nTOKEN-TO-ID Mapper:\\n{token2id}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Converting Tokenized sentences to Number sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_sequences = token_seq_to_num_seq(tokenized_sentences, token2id, oov_token='<oov>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[0 0 5 6 1]\n[2 7 0 3]\n[0 0 8 0 0 0 1]\n[9 0 8 0 0]\n[2 7 0 3]\n[2 0 4 0 3]\n[9 0 4 0 0 0]\n[0 0 5 6 0 4 0 0 0 0 0 1]\n"
    }
   ],
   "source": [
    "# 0 for words that are not in the vocab of 10 words\n",
    "for sequence in num_sequences: print(sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieving Tokenized sentences back from the Number sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "retrieved_tokenized_sequences = num_seq_to_token_seq(num_sequences, id2token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "['<oov>', '<oov>', 'are', 'you', '?']\n['i', 'am', '<oov>', '.']\n['<oov>', '<oov>', 'the', '<oov>', '<oov>', '<oov>', '?']\n['we', '<oov>', 'the', '<oov>', '<oov>']\n['i', 'am', '<oov>', '.']\n['i', '<oov>', 'to', '<oov>', '.']\n['we', '<oov>', 'to', '<oov>', '<oov>', '<oov>']\n['<oov>', '<oov>', 'are', 'you', '<oov>', 'to', '<oov>', '<oov>', '<oov>', '<oov>', '<oov>', '?']\n"
    }
   ],
   "source": [
    "# we lose words ('<oov>') on retrieval because num_words < total number of words in the vocabulary\n",
    "for sequence in retrieved_tokenized_sequences: print(sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extra features of the Embedder Class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can use object of the Embedder class to find:\n",
    "\n",
    "   1. Similarity score between two words\n",
    "\n",
    "   2. Analogy Score that determines, how good an analogy is.\n",
    "\n",
    "   3. K most similar words\n",
    "\n",
    "   4. K most apt words to satisfy the analogy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb2 = Embedder(dimensions=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "bad ->  ['worse', 'unfortunately', 'too'] \n\nbaby ->  ['babies', 'boy', 'girl']\n"
    }
   ],
   "source": [
    "print('bad -> ',emb2.most_similar_to('bad',k=3),'\\n\\nbaby -> ',emb2.most_similar_to('baby',k=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Analogy of\n\nking -> queen    man ->  ['woman', 'girl']\n"
    }
   ],
   "source": [
    "print('Analogy of\\n\\nking -> queen    man -> ',emb2.get_top_k_by_analogy('king','queen','man',k=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Similarity score:\n\nbad -> worse     0.8878378 \n\nhello -> why     0.41916144\n"
    }
   ],
   "source": [
    "print('Similarity score:\\n\\nbad -> worse    ',emb2.cosine_similarity('bad','worse'), '\\n\\nhello -> why    ', emb2.cosine_similarity('hello','why'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Analogy Score:\n\ngood->best   bad->worst =  0.75127333 \n\ngood->woman  yes->hello =  0.48011282\n"
    }
   ],
   "source": [
    "print('Analogy Score:\\n\\ngood->best   bad->worst = ',emb2.cosine_sim_analogy('good','best','bad','worst'),'\\n\\ngood->woman  yes->hello = ',emb2.cosine_sim_analogy('good','woman','yes','hello'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}