bag_of_words_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bag of words model\n",
    "\n",
    "Considers all the words in corpus as entities disregarding their position in a sentence, and hence, context."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#each sentence can be considered as document\n",
    "\n",
    "corpus = ['king is a strong man', \n",
    "          'queen is a wise woman', \n",
    "          'boy is a young man',\n",
    "          'girl is a young woman',\n",
    "          'prince is a young king',\n",
    "          'princess is a young queen',\n",
    "          'man is strong', \n",
    "          'woman is pretty',\n",
    "          'prince is a boy will be king',\n",
    "          'princess is a girl will be queen']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Boolean Retrieval\n",
    "\n",
    "1. Checks whether a term is present in document or not. \n",
    "2. However, it does not care about the importance of words.\n",
    "\n",
    "### Methods:\n",
    "\n",
    "1. Term-document matrix\n",
    "2. Posting lists\n",
    "\n",
    "Local and discrete representation of words. \n",
    "- Local because it does not capture the context between words. \n",
    "- Discrete because the information about word is not distributed across the vector (like word embeddings)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Term - document matrix**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['boy',\n",
       " 'girl',\n",
       " 'strong',\n",
       " 'pretty',\n",
       " 'be',\n",
       " 'a',\n",
       " 'king',\n",
       " 'will',\n",
       " 'prince',\n",
       " 'princess',\n",
       " 'wise',\n",
       " 'queen',\n",
       " 'woman',\n",
       " 'is',\n",
       " 'man',\n",
       " 'young']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = list(map(lambda x: x.split(), corpus))\n",
    "vocab = list(set(itertools.chain(*words)))\n",
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 10)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def check_occurence(word, document):\n",
    "    \n",
    "    document=document.split()\n",
    "    \n",
    "    if (word in document):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "occurences = np.array(list(map(lambda y: list(map(lambda x: check_occurence(x, y), vocab)), corpus)))\n",
    "df = pd.DataFrame(occurences.transpose(), index=vocab, columns=range(len(corpus))) \n",
    "\n",
    "# (term, document) matrix\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>boy</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>girl</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strong</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pretty</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>be</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>king</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>will</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prince</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>princess</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wise</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>queen</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>woman</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>man</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>young</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0  1  2  3  4  5  6  7  8  9\n",
       "boy       0  0  1  0  0  0  0  0  1  0\n",
       "girl      0  0  0  1  0  0  0  0  0  1\n",
       "strong    1  0  0  0  0  0  1  0  0  0\n",
       "pretty    0  0  0  0  0  0  0  1  0  0\n",
       "be        0  0  0  0  0  0  0  0  1  1\n",
       "a         1  1  1  1  1  1  0  0  1  1\n",
       "king      1  0  0  0  1  0  0  0  1  0\n",
       "will      0  0  0  0  0  0  0  0  1  1\n",
       "prince    0  0  0  0  1  0  0  0  1  0\n",
       "princess  0  0  0  0  0  1  0  0  0  1\n",
       "wise      0  1  0  0  0  0  0  0  0  0\n",
       "queen     0  1  0  0  0  1  0  0  0  1\n",
       "woman     0  1  0  1  0  0  0  1  0  0\n",
       "is        1  1  1  1  1  1  1  1  1  1\n",
       "man       1  0  1  0  0  0  1  0  0  0\n",
       "young     0  0  1  1  1  1  0  0  0  0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28.125"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking sparsity of matrix\n",
    "sum(df==1) / (df.shape[0] * df.shape[1]) * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inverted Index\n",
    "\n",
    "Since the term-document matrix can be very sparse, we can store the document_ids in lists. \n",
    "\n",
    "#### Postings lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def posting_list(word, corpus):\n",
    "    doc_ids = np.array(list(range(len(corpus))))\n",
    "    occurence = np.array(list(map(lambda x: check_occurence(word, x), corpus)))\n",
    "\n",
    "    idx = (occurence==1)*doc_ids\n",
    "    return idx[idx!=0].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boy': [2, 8],\n",
       " 'girl': [3, 9],\n",
       " 'strong': [6],\n",
       " 'pretty': [7],\n",
       " 'be': [8, 9],\n",
       " 'a': [1, 2, 3, 4, 5, 8, 9],\n",
       " 'king': [4, 8],\n",
       " 'will': [8, 9],\n",
       " 'prince': [4, 8],\n",
       " 'princess': [5, 9],\n",
       " 'wise': [1],\n",
       " 'queen': [1, 5, 9],\n",
       " 'woman': [1, 3, 7],\n",
       " 'is': [1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
       " 'man': [2, 6],\n",
       " 'young': [2, 3, 4, 5]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "posting_dict={}\n",
    "\n",
    "for word in vocab:\n",
    "    posting_dict[word]=posting_list(word, corpus)\n",
    "    \n",
    "posting_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Positional Index\n",
    "\n",
    "As described before, **we are not considering the position of words** and so for **phrase queries** like **'Indian Statistical Institute'**, we need the words to appear in the exact order in a document to be relevant.\n",
    "\n",
    "And so, we can keep the information of position of words (positional index) in document in the posting list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_positional_index(word, doc_id, document):\n",
    "    \n",
    "    positions = (np.array(document)==word).nonzero()[0].tolist()\n",
    "    \n",
    "    return {doc_id: positions}\n",
    "    \n",
    "def get_posting_list(word, corpus):\n",
    "    \n",
    "    doc_ids = np.array(list(map(lambda x: check_occurence(word, x), corpus))).nonzero()[0].tolist()\n",
    "    documents = list(map(lambda x: x.split(), corpus))\n",
    "    \n",
    "    return list(map(lambda x: get_positional_index(word, x, documents[x]), doc_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 2, 6]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word='man'\n",
    "np.array(list(map(lambda x: check_occurence(word, x), corpus))).nonzero()[0].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boy': {8: [3], 2: [0]},\n",
       " 'girl': {9: [3], 3: [0]},\n",
       " 'strong': {6: [2], 0: [3]},\n",
       " 'pretty': {7: [2]},\n",
       " 'be': {9: [5], 8: [5]},\n",
       " 'a': {9: [2], 8: [2], 5: [2], 4: [2], 3: [2], 2: [2], 1: [2], 0: [2]},\n",
       " 'king': {8: [6], 4: [4], 0: [0]},\n",
       " 'will': {9: [4], 8: [4]},\n",
       " 'prince': {8: [0], 4: [0]},\n",
       " 'princess': {9: [0], 5: [0]},\n",
       " 'wise': {1: [3]},\n",
       " 'queen': {9: [6], 5: [4], 1: [0]},\n",
       " 'woman': {7: [0], 3: [4], 1: [4]},\n",
       " 'is': {9: [1],\n",
       "  8: [1],\n",
       "  7: [1],\n",
       "  6: [1],\n",
       "  5: [1],\n",
       "  4: [1],\n",
       "  3: [1],\n",
       "  2: [1],\n",
       "  1: [1],\n",
       "  0: [1]},\n",
       " 'man': {6: [0], 2: [4], 0: [4]},\n",
       " 'young': {5: [3], 4: [3], 3: [3], 2: [3]}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import ChainMap\n",
    "\n",
    "posting_dict={}\n",
    "\n",
    "for word in vocab:\n",
    "    docs = get_posting_list(word, corpus)\n",
    "    posting_dict[word] = dict(ChainMap(*docs))\n",
    "    \n",
    "posting_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-iDF weighting\n",
    "\n",
    "Captures how important words are relative to corpus.\n",
    "\n",
    "Local and continuous representation of words. Local because it does not capture the context of words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10, 16), (10, 16))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def calculate_term_freq(word, document):\n",
    "    words = np.array(document.split())\n",
    "    return sum(words==word) / len(words)\n",
    "    \n",
    "def calculate_inverse_doc_freq(word, corpus):\n",
    "    n_occurence = sum(map(lambda x: check_occurence(word, x), corpus))\n",
    "    \n",
    "    if (n_occurence==0):\n",
    "        n_occurence = 1\n",
    "        \n",
    "    return np.log(len(corpus) / n_occurence)\n",
    "\n",
    "tf = np.array(list(map(lambda y: list(map(lambda x: calculate_term_freq(x, y), vocab)), corpus)))\n",
    "idf = list(map(lambda x: calculate_inverse_doc_freq(x, corpus), vocab))\n",
    "idf = np.array([idf]*len(corpus))\n",
    "\n",
    "tf.shape, idf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>boy</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>girl</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>strong</th>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pretty</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>be</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>king</th>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>will</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prince</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>princess</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wise</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>queen</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>woman</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>man</th>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>young</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             0     1     2     3     4     5     6     7     8     9\n",
       "boy       0.00  0.00  0.32  0.00  0.00  0.00  0.00  0.00  0.23  0.00\n",
       "girl      0.00  0.00  0.00  0.32  0.00  0.00  0.00  0.00  0.00  0.23\n",
       "strong    0.32  0.00  0.00  0.00  0.00  0.00  0.54  0.00  0.00  0.00\n",
       "pretty    0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.77  0.00  0.00\n",
       "be        0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.23  0.23\n",
       "a         0.04  0.04  0.04  0.04  0.04  0.04  0.00  0.00  0.03  0.03\n",
       "king      0.24  0.00  0.00  0.00  0.24  0.00  0.00  0.00  0.17  0.00\n",
       "will      0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.23  0.23\n",
       "prince    0.00  0.00  0.00  0.00  0.32  0.00  0.00  0.00  0.23  0.00\n",
       "princess  0.00  0.00  0.00  0.00  0.00  0.32  0.00  0.00  0.00  0.23\n",
       "wise      0.00  0.46  0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00\n",
       "queen     0.00  0.24  0.00  0.00  0.00  0.24  0.00  0.00  0.00  0.17\n",
       "woman     0.00  0.24  0.00  0.24  0.00  0.00  0.00  0.40  0.00  0.00\n",
       "is        0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00  0.00\n",
       "man       0.24  0.00  0.24  0.00  0.00  0.00  0.40  0.00  0.00  0.00\n",
       "young     0.00  0.00  0.18  0.18  0.18  0.18  0.00  0.00  0.00  0.00"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tfidf = pd.DataFrame((tf*idf).transpose(), index=vocab, columns=range(len(corpus))).round(2)\n",
    "df_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}