Dataset Cleaning through Regex.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import re\n",
    "import string\n",
    "import warnings\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "import cache_magic\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from langdetect import detect\n",
    "\n",
    "pd.set_option('display.max_colwidth',100)\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('PM_MMS_Speech.xlsx',dtype={'title':'string',\n",
    "                                               'date':'datetime64',\n",
    "                                               'place':'string',\n",
    "                                               'url':'string',\n",
    "                                               'text':'string'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "      <th>place</th>\n",
       "      <th>url</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PM's address to the Nation</td>\n",
       "      <td>2012-09-21</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1226</td>\n",
       "      <td>My dear brothers and sisters,\r\n",
       "\tI am speaking to you tonight to explain the reasons for some ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PM's remarks at the Victory over Polio Celebrations</td>\n",
       "      <td>2014-02-11</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1434</td>\n",
       "      <td>“This is indeed a historic day. It is a day that we have worked for tirelessly and awaited eq...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PM’s speech at the Governors’ Conference</td>\n",
       "      <td>2014-02-15</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1435</td>\n",
       "      <td>Following is the text of the Prime Minister, Dr. Manmohan Singh’s address at the Governors’ C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PM's statement in Rajya Sabha on the Telangana Bill and a special package for the successor stat...</td>\n",
       "      <td>2014-02-20</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1436</td>\n",
       "      <td>I have listened very carefully to the views expressed by the Leader of Opposition and all ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PM’s interaction with media outside Parliament House</td>\n",
       "      <td>2014-02-05</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1429</td>\n",
       "      <td>\"As you have said, this is probably the last session of Parliament, and it is my sincere appe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                 title  \\\n",
       "0                                                                           PM's address to the Nation   \n",
       "1                                                  PM's remarks at the Victory over Polio Celebrations   \n",
       "2                                                             PM’s speech at the Governors’ Conference   \n",
       "3  PM's statement in Rajya Sabha on the Telangana Bill and a special package for the successor stat...   \n",
       "4                                                 PM’s interaction with media outside Parliament House   \n",
       "\n",
       "        date      place  \\\n",
       "0 2012-09-21  New Delhi   \n",
       "1 2014-02-11  New Delhi   \n",
       "2 2014-02-15  New Delhi   \n",
       "3 2014-02-20  New Delhi   \n",
       "4 2014-02-05  New Delhi   \n",
       "\n",
       "                                                                        url  \\\n",
       "0  https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1226   \n",
       "1  https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1434   \n",
       "2  https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1435   \n",
       "3  https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1436   \n",
       "4  https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1429   \n",
       "\n",
       "                                                                                                  text  \n",
       "0  \n",
       "\tMy dear brothers and sisters,\n",
       "\tI am speaking to you tonight to explain the reasons for some ...  \n",
       "1  \n",
       "\t“This is indeed a historic day. It is a day that we have worked for tirelessly and awaited eq...  \n",
       "2  \n",
       "\tFollowing is the text of the Prime Minister, Dr. Manmohan Singh’s address at the Governors’ C...  \n",
       "3  \n",
       "\t\n",
       "\tI have listened very carefully to the views expressed by the Leader of Opposition and all ...  \n",
       "4  \n",
       "\t\"As you have said, this is probably the last session of Parliament, and it is my sincere appe...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1380 entries, 0 to 1379\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype         \n",
      "---  ------  --------------  -----         \n",
      " 0   title   1380 non-null   string        \n",
      " 1   date    1380 non-null   datetime64[ns]\n",
      " 2   place   1380 non-null   string        \n",
      " 3   url     1380 non-null   string        \n",
      " 4   text    1335 non-null   string        \n",
      "dtypes: datetime64[ns](1), string(4)\n",
      "memory usage: 54.0 KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace= True)#Drops rows with null values inplace\n",
    "df.reset_index(inplace= True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    1313\n",
       "True       22\n",
       "Name: text, dtype: Int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "small_speech = (df.text.str.len()<500)\n",
    "small_speech.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dropping speeches which are small\n",
    "df.drop(labels = np.flatnonzero(small_speech),inplace = True)\n",
    "df.reset_index(inplace= True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "en    1294\n",
       "hi      19\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking the language of the speeches\n",
    "df_language = df.text.apply(detect)\n",
    "df_language.value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "hindi_indices = np.flatnonzero(df_language == 'hi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6       \n",
       "\tFollowing is the speech of the Prime Minister, Dr. Manmohan Singh, delivered in Hindi on the ...\n",
       "1010    \n",
       "\tFollowing is the address of the Prime Minister, Dr. Manmohan Singh, delivered in Hindi, at Ra...\n",
       "1192    \n",
       "\tFollowing is the address of the Prime Minister, Dr. Manmohan Singh, delivered in Hindi, at th...\n",
       "1202    \n",
       "\tFollowing is the address of the Prime Minister, Dr. Manmohan Singh, delivered in Hindi, at th...\n",
       "1208    \n",
       "\tFollowing is the address of the Prime Minister, Dr. Manmohan Singh, delivered in Hindi, at th...\n",
       "Name: text, dtype: string"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.text[hindi_indices][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dropping hindi speeches\n",
    "df.drop(labels = hindi_indices,inplace = True)\n",
    "df.reset_index(inplace= True,drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing unicode characters and extra spaces\n",
    "def remove_unicode(text):\n",
    "    return text.encode('ascii', 'ignore').decode('utf8')\n",
    "\n",
    "def remove_tabs(text):\n",
    "    return re.sub(r'[\\r\\n\\t]', ' ', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_text'] = df.text.apply(remove_unicode).apply(remove_tabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1294 entries, 0 to 1293\n",
      "Data columns (total 6 columns):\n",
      " #   Column      Non-Null Count  Dtype         \n",
      "---  ------      --------------  -----         \n",
      " 0   title       1294 non-null   string        \n",
      " 1   date        1294 non-null   datetime64[ns]\n",
      " 2   place       1294 non-null   string        \n",
      " 3   url         1294 non-null   string        \n",
      " 4   text        1294 non-null   string        \n",
      " 5   clean_text  1294 non-null   object        \n",
      "dtypes: datetime64[ns](1), object(1), string(4)\n",
      "memory usage: 60.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Concatenating all speeches to form a single string corpus_text\n",
    "corpus_text = df.clean_text.str.cat(sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Writing corpus_text to a file\n",
    "with open(\"corpus_raw2.txt\", \"w\") as file:  \n",
    "    file.write(corpus_text) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to view dictionary items\n",
    "def view_dict(dictionary,num):\n",
    "    for key in list(dictionary.keys())[:num]:\n",
    "        value = dictionary[key]\n",
    "        print (key,value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading Embeddings - Code taken from kaggle notebook\n",
    "def load_embed(file):\n",
    "    def get_coefs(word,*arr): \n",
    "        return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n",
    "    return embeddings_index\n",
    "\n",
    "glove = \"glove.840B.300d/glove.840B.300d.txt\"\n",
    "\n",
    "embed_glove = load_embed(glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[',', '.', 'the', 'and', 'to', 'of', 'a', 'in', '\"', ':']\n"
     ]
    }
   ],
   "source": [
    "print(list(embed_glove.keys())[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[True, True, True, True, True, True, True, True, True, True, False, False, True, True, False]\n"
     ]
    }
   ],
   "source": [
    "words = ['cant', \"can't\",'ca','nt',\"n't\" ,\"'s\",'her.','her','Her','##',' ', '\\n','.', ',' ,'#.#']\n",
    "print ([word in embed_glove for word in words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code taken from kaggle notebook\n",
    "#Code to check how much of vocabulary and corpus is covered by the words in glove\n",
    "#returns words which are not in glove vocab in the decraesing order of their occurence count\n",
    "def check_coverage(vocab, embeddings_index):\n",
    "    known_words = {}\n",
    "    unknown_words = {}\n",
    "    num_known_words = 0\n",
    "    num_unknown_words = 0\n",
    "    for word in vocab.keys():\n",
    "        try:\n",
    "            #if a vocabulary word is in glove, then adding that word to known_words and increasing num_known_words count by 1\n",
    "            known_words[word] = embeddings_index[word]\n",
    "            num_known_words += vocab[word]\n",
    "        except:\n",
    "            #if a vocabulary word is not in glove, then adding that word to unknown_words\n",
    "            # and increasing num_unknown_words count by 1\n",
    "            unknown_words[word] = vocab[word]\n",
    "            num_unknown_words += vocab[word]\n",
    "            \n",
    "\n",
    "    print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))\n",
    "    print('Found embeddings for  {:.2%} of all text'.format(num_known_words / (num_known_words + num_unknown_words)))\n",
    "    unknown_words = sorted(unknown_words.items(), key=lambda kv: kv[1], reverse=True)\n",
    "\n",
    "    return unknown_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['My', 'dear', 'brothers', 'and', 'sisters,', 'I', 'am', 'speaking', 'to', 'you', 'tonight', 'to', 'explain', 'the', 'reasons', 'for', 'some', 'important', 'economic', 'policy', 'decisions', 'the', 'government', 'has', 'recently', 'taken.', 'Some', 'political', 'parties', 'have', 'opposed', 'them.', 'You', 'have', 'a', 'right', 'to', 'know', 'the', 'truth', 'about', 'why', 'we', 'have', 'taken', 'these', 'decisions.', 'government', 'likes', 'to', 'impose', 'burdens', 'on', 'the', 'common', 'man.', 'Our', 'Government', 'has', 'been', 'voted', 'to', 'office', 'twice', 'to', 'protect', 'the', 'interests', 'of', 'the', 'aam', 'admi.', 'At', 'the', 'same', 'time,', 'it', 'is', 'the', 'responsibility', 'of', 'the', 'government', 'to', 'defend', 'the', 'national', 'interest,', 'and', 'protect', 'the', 'long', 'term', 'future', 'of', 'our', 'people.', 'This', 'means', 'that']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1511539"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Splitting corpus text into words. Using string split as tokenizer\n",
    "#Note from future self - Using string split is a blunder. Could've used any tokenizer\n",
    "#But this blunder helped you in learning regex\n",
    "corpus = corpus_text.split()\n",
    "print(corpus[:10])\n",
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('the', 83156), ('of', 66132), ('and', 54129), ('to', 50481), ('in', 36860), ('a', 26523), ('our', 21120), ('that', 20152), ('is', 19145), ('I', 16399)]\n"
     ]
    }
   ],
   "source": [
    "#creating a counter for words and from it vocab\n",
    "word_count = Counter(corpus)\n",
    "vocab = dict(word_count)\n",
    "print(word_count.most_common(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 55.18% of vocab\n",
      "Found embeddings for  93.56% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(\"India's\", 1016),\n",
       " ('However,', 673),\n",
       " ('therefore,', 621),\n",
       " ('India,', 527),\n",
       " ('years,', 463),\n",
       " ('sector.', 405),\n",
       " ('\"I', 392),\n",
       " ('time,', 388),\n",
       " ('country,', 323),\n",
       " ('Today,', 311)]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#I think this is a long tail distribution\n",
    "#All the operations performed below are intended to improve the coverage and reduce the number of unk tokens while feeding model\n",
    "oov = check_coverage(vocab,embed_glove)\n",
    "oov[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\"No',\n",
       " '\"Yes',\n",
       " '\"Let',\n",
       " '\"As',\n",
       " '\"I',\n",
       " '\"This',\n",
       " '\"Madam',\n",
       " '\"I',\n",
       " '\"Who',\n",
       " '\"I',\n",
       " '\"It',\n",
       " '\"the',\n",
       " '\"Social',\n",
       " '\"I']"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\"[A-Za-z]+',corpus_text[:100000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 56.73% of vocab\n",
      "Found embeddings for  93.71% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(\"India's\", 1018),\n",
       " ('However,', 673),\n",
       " ('therefore,', 621),\n",
       " ('India,', 527),\n",
       " ('years,', 463),\n",
       " ('\".', 416),\n",
       " ('sector.', 405),\n",
       " ('time,', 388),\n",
       " ('country,', 323),\n",
       " ('Today,', 311)]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Separating  quotations\n",
    "corpus_text_clean = re.sub(r'(\")([A-Za-z0-9])', r'\\1 \\2', corpus_text)#new numbers added\n",
    "corpus_text_clean = re.sub(r'([A-Za-z0-9])(\")', r'\\1 \\2', corpus_text_clean)#new entire line\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 83.32% of vocab\n",
      "Found embeddings for  99.16% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(\"India's\", 1032),\n",
       " ('.\"', 712),\n",
       " ('\".', 427),\n",
       " (\"world's\", 141),\n",
       " ('\",', 127),\n",
       " (\"country's\", 126),\n",
       " (\"Hon'ble\", 121),\n",
       " (\"today's\", 114),\n",
       " (\"nation's\", 104),\n",
       " (\"people's\", 101),\n",
       " (\"Government's\", 86),\n",
       " ('Indiraji', 64),\n",
       " (\"other's\", 62),\n",
       " (\"Gandhiji's\", 59),\n",
       " (\"year's\", 58),\n",
       " (\"one's\", 55),\n",
       " ('Rajivji', 52),\n",
       " (\"Minister's\", 52),\n",
       " ('8%', 51),\n",
       " ('9%', 50)]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Separating punctuation maarks from words\n",
    "corpus_text_clean = re.sub(r'([A-Za-z]+)([\\.,\\?])',r'\\1 \\2',corpus_text_clean)\n",
    "corpus_text_clean = re.sub(r'([.,])([A-Za-z]+)',r'\\1 \\2',corpus_text_clean)#added new\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 84.41% of vocab\n",
      "Found embeddings for  99.37% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('.\"', 712),\n",
       " ('\".', 427),\n",
       " ('\",', 127),\n",
       " (\"Hon'ble\", 121),\n",
       " ('Indiraji', 78),\n",
       " ('Rajivji', 65),\n",
       " ('8%', 51),\n",
       " ('9%', 50),\n",
       " ('2004,', 42),\n",
       " ('10%', 38),\n",
       " ('50%', 35),\n",
       " ('2008,', 33),\n",
       " ('7%', 33),\n",
       " ('BIMST-EC', 32),\n",
       " ('.,', 31),\n",
       " ('1991,', 30),\n",
       " ('2%', 30),\n",
       " ('2005,', 29),\n",
       " ('2009,', 29),\n",
       " ('2006,', 27)]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Separting 's from the word\n",
    "corpus_text_clean = re.sub(r'([a-z]+)(\\'s)',r'\\1 \\2',corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['200,0',\n",
       " '1991.',\n",
       " '1991,',\n",
       " '8.2',\n",
       " '1991.',\n",
       " '2.3',\n",
       " '5.',\n",
       " '14,',\n",
       " '2013,',\n",
       " '2013.',\n",
       " '2005.',\n",
       " '15.',\n",
       " '1814,',\n",
       " '1814,',\n",
       " '2008,',\n",
       " '2004,',\n",
       " '2014.',\n",
       " '14,',\n",
       " '1946,',\n",
       " '2006,',\n",
       " '2012,',\n",
       " '2007,',\n",
       " '2008,',\n",
       " '1.',\n",
       " '2.',\n",
       " '3.',\n",
       " '4.',\n",
       " '5.',\n",
       " '6.',\n",
       " '2004.',\n",
       " '7.',\n",
       " '8.',\n",
       " '9.',\n",
       " '10.',\n",
       " '11.',\n",
       " '12.',\n",
       " '13.',\n",
       " '2004,',\n",
       " '14.',\n",
       " '15.',\n",
       " '16.',\n",
       " '17.']"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'[0-9]+[.,][0-9]?',corpus_text[:100000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['80%',\n",
       " '40',\n",
       " '200,000',\n",
       " '1991.',\n",
       " '1991',\n",
       " '17',\n",
       " '160',\n",
       " '50%',\n",
       " '1991,',\n",
       " '8.2',\n",
       " '1991.',\n",
       " '2.3']"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'[0-9]+[\\S]+',corpus_text[:10000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 86.16% of vocab\n",
      "Found embeddings for  99.51% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('.\"', 712),\n",
       " ('\".', 427),\n",
       " ('\",', 127),\n",
       " (\"Hon'ble\", 121),\n",
       " ('G-', 90),\n",
       " ('Indiraji', 78),\n",
       " ('Rajivji', 65),\n",
       " ('BIMST-EC', 32),\n",
       " ('.,', 31),\n",
       " ('Hind!', 25),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Sharadji', 16),\n",
       " ('Nehruji', 16),\n",
       " ('Kantji', 15),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " (\"farmers'\", 14)]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#removing numbers\n",
    "#In hindsight - might be a bad choice. Might have replaced them with hashes\n",
    "corpus_text_clean = re.sub(r'[0-9]+[\\S]+',' ',corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 86.24% of vocab\n",
      "Found embeddings for  99.58% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('..', 137),\n",
       " (\"Hon'ble\", 121),\n",
       " ('G-', 90),\n",
       " ('Indiraji', 78),\n",
       " ('Rajivji', 65),\n",
       " ('BIMST-EC', 32),\n",
       " ('Hind!', 25),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Sharadji', 16),\n",
       " ('Nehruji', 16),\n",
       " ('Kantji', 15),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " ('quote:', 14),\n",
       " (\"farmers'\", 14),\n",
       " ('to:', 14),\n",
       " ('are:', 13)]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Separating punctuation marks from each other\n",
    "#[word for word,_ in oov if]\n",
    "corpus_text_clean = re.sub('([\\.\\,\\:\\;\\\"])([\\.\\,\\:\\;\\\"])',r'\\1 \\2',corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 86.53% of vocab\n",
      "Found embeddings for  99.61% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('..', 137),\n",
       " (\"Hon'ble\", 121),\n",
       " ('G-', 90),\n",
       " ('BIMST-EC', 32),\n",
       " ('Hind!', 25),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " ('quote:', 14),\n",
       " (\"farmers'\", 14),\n",
       " ('to:', 14),\n",
       " ('are:', 13),\n",
       " (\"'Sarva\", 13),\n",
       " (\"peoples'\", 12),\n",
       " ('people;', 12),\n",
       " (\"'Look\", 11),\n",
       " ('development;', 10)]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Separating ji from the name\n",
    "corpus_text_clean = re.sub(r'([A-Z][a-z]+)ji', r'\\1',corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 89.95% of vocab\n",
      "Found embeddings for  99.71% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('..', 137),\n",
       " (\"Hon'ble\", 121),\n",
       " ('G-', 90),\n",
       " ('BIMST-EC', 32),\n",
       " ('(Interruptions', 31),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " (\"farmers'\", 14),\n",
       " (\"'Sarva\", 13),\n",
       " (\"peoples'\", 12),\n",
       " (\"'Look\", 11),\n",
       " ('(a', 10),\n",
       " ('(i', 10),\n",
       " ('(ii', 10),\n",
       " ('(Prevention', 10),\n",
       " ('Hembhai', 10)]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_text_clean = re.sub(r'([A-Za-z0-9]+)([:;?!/)/(//])', r'\\1 \\2 ', corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_text_clean = re.sub(r'([A-Za-z0-9]+)([:;?!/)/(//])', r'\\1 \\2 ', corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 90.92% of vocab\n",
      "Found embeddings for  99.74% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('..', 138),\n",
       " (\"Hon'ble\", 121),\n",
       " ('G-', 91),\n",
       " ('BIMST-EC', 32),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " (\"farmers'\", 14),\n",
       " (\"'Sarva\", 13),\n",
       " (\"peoples'\", 12),\n",
       " ('IISERs', 12),\n",
       " (\"'Look\", 11),\n",
       " ('Hembhai', 10),\n",
       " ('Jagvan', 9),\n",
       " (\"workers'\", 9),\n",
       " ('NH-', 9),\n",
       " ('Mahalonobis', 9)]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Separating the brace\n",
    "corpus_text_clean = re.sub(r'(\\()', r' \\1  ', corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 90.92% of vocab\n",
      "Found embeddings for  99.75% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(\"Hon'ble\", 121),\n",
       " ('G-', 91),\n",
       " ('BIMST-EC', 32),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " (\"farmers'\", 14),\n",
       " (\"'Sarva\", 13),\n",
       " (\"peoples'\", 12),\n",
       " ('IISERs', 12),\n",
       " (\"'Look\", 11),\n",
       " ('Hembhai', 10),\n",
       " ('Jagvan', 9),\n",
       " (\"workers'\", 9),\n",
       " ('NH-', 9),\n",
       " ('Mahalonobis', 9),\n",
       " ('post-', 9)]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Removing serial dots\n",
    "corpus_text_clean = re.sub(r'\\.\\.', r' ', corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 90.93% of vocab\n",
      "Found embeddings for  99.76% of all text\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('G-', 91),\n",
       " (\".'\", 24),\n",
       " ('?\"', 20),\n",
       " ('Vidyutikaran', 19),\n",
       " ('mid-', 17),\n",
       " ('Deputy-Speaker', 15),\n",
       " ('India`s', 15),\n",
       " (\"farmers'\", 14),\n",
       " (\"'Sarva\", 13),\n",
       " (\"peoples'\", 12),\n",
       " ('IISERs', 12),\n",
       " (\"'Look\", 11),\n",
       " ('Hembhai', 10),\n",
       " ('Jagvan', 9),\n",
       " (\"workers'\", 9),\n",
       " ('NH-', 9),\n",
       " ('Mahalonobis', 9),\n",
       " ('post-', 9),\n",
       " ('Ela-ben', 9),\n",
       " (\"India'.\", 8)]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Correcting the abbreviations\n",
    "corpus_text_clean = re.sub(\"Hon'ble\",'Honourable' ,corpus_text_clean)\n",
    "corpus_text_clean = re.sub('BIMST-EC','BIMSTEC' ,corpus_text_clean)\n",
    "corpus_text_clean = re.sub(r'A\\s?S\\s?E\\s?A\\s?N','ASEAN',corpus_text_clean)\n",
    "corpus_clean = corpus_text_clean.split()\n",
    "word_count = Counter(corpus_clean)\n",
    "vocab_clean = dict(word_count)\n",
    "oov = check_coverage(vocab_clean,embed_glove)\n",
    "oov[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Writing cleaned corpus to a file \n",
    "with open(\"corpus_clean_2.txt\", \"w\") as file:  \n",
    "    file.write(corpus_text_clean) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_sorted = sorted(vocab_clean.items(), key=lambda kv: kv[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For vocab of size 1000, vocabulary coverage is 100.0% and corpus coverage is 79.0%\n",
      "For vocab of size 5000, vocabulary coverage is 100.0% and corpus coverage is 94.0%\n",
      "For vocab of size 10000, vocabulary coverage is 100.0% and corpus coverage is 98.0%\n",
      "For vocab of size 15000, vocabulary coverage is 98.0% and corpus coverage is 99.0%\n",
      "For vocab of size 20000, vocabulary coverage is 96.0% and corpus coverage is 99.0%\n"
     ]
    }
   ],
   "source": [
    "#Now we want to restrict the vocab size or else model becomes huge\n",
    "#In a way we could have done this earlier instead of improving the coverage, then we wouldn't have learnt regex\n",
    "# If we restrict the vocab words to 1k,5k etc...\n",
    "# We want to know how much fraction of restricted vocab words are present in glove and\n",
    "# how much fraction of corpus is covered by the words in restricted vocab\n",
    "corpus_len = len(corpus_clean)\n",
    "for i in [1000,5000,10000,15000,20000]:\n",
    "    vocab_ = dict(vocab_sorted[:i])\n",
    "    #view_dict(vocab_,5)\n",
    "    corpus_covered = 0\n",
    "    vocab_covered = 0\n",
    "    for (word,count) in vocab_.items():\n",
    "        if word in embed_glove:\n",
    "            corpus_covered += count\n",
    "            vocab_covered += 1\n",
    "    vocab_coverage = round((vocab_covered/len(vocab_)),2)*100\n",
    "    corpus_coverage = round((corpus_covered/corpus_len),2)*100\n",
    "    print (f'For vocab of size {i}, vocabulary coverage is {vocab_coverage}% and corpus coverage is {corpus_coverage}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Here I made a silly mistake, the 100% vocabulary coverage is misleading because it is a rounded off value. It is not exactly 100%. This created so many problems later becuase if you observe below, embed_5k has a size of 4998 but we are thinking its size is 5000. So, there is an offset of 2, which lead to feeding wrong word vectors to the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_embed(vocab, embeddings_index):\n",
    "    our_embeddings = {}\n",
    "    \n",
    "    for word in vocab.keys():\n",
    "        try:\n",
    "            our_embeddings[word] = embeddings_index[word]\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    return our_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_5k = dict(vocab_sorted[:5000])\n",
    "embed_5k = create_embed(vocab_5k, embed_glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 4998)"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab_5k),len(embed_5k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\".'\", 'G-']"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing missing keys from vocab_5k\n",
    "missing_keys = list(set(vocab_5k.keys()) - set(embed_5k.keys()))\n",
    "missing_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for word in missing_keys:\n",
    "    del vocab_5k[word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(set(), 4998, 4998)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(vocab_5k.keys()) - set(embed_5k.keys()), len(vocab_5k),len(embed_5k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pickling vocab and embeddings files\n",
    "with open('embed_5k_2.pickle', 'wb') as handle:\n",
    "    pickle.dump(embed_5k, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('vocab_5k_2.pickle', 'wb') as handle:\n",
    "    pickle.dump(vocab_5k, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}