Part3_Build_Text_Pre-processor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: textsearch in /anaconda3/lib/python3.6/site-packages (0.0.17)\n",
      "Requirement already satisfied: Unidecode in /anaconda3/lib/python3.6/site-packages (from textsearch) (1.1.0)\n",
      "Requirement already satisfied: pyahocorasick in /anaconda3/lib/python3.6/site-packages (from textsearch) (1.4.0)\n",
      "Requirement already satisfied: contractions in /anaconda3/lib/python3.6/site-packages (0.0.21)\n",
      "Requirement already satisfied: tqdm in /anaconda3/lib/python3.6/site-packages (4.32.2)\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/manishanker.talusani/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/manishanker.talusani/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!pip install textsearch\n",
    "!pip install contractions\n",
    "!pip install tqdm\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import unicodedata\n",
    "import contractions\n",
    "import spacy\n",
    "import nltk\n",
    "\n",
    "nlp = spacy.load('en', parse=False, tag=False, entity=False)\n",
    "ps = nltk.porter.PorterStemmer()\n",
    "\n",
    "\n",
    "def strip_html_tags(text):\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    [s.extract() for s in soup(['iframe', 'script'])]\n",
    "    stripped_text = soup.get_text()\n",
    "    stripped_text = re.sub(r'[\\r|\\n|\\r\\n]+', '\\n', stripped_text)\n",
    "    return stripped_text\n",
    "\n",
    "\n",
    "def remove_accented_chars(text):\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "    return text\n",
    "\n",
    "\n",
    "def expand_contractions(text):\n",
    "    return contractions.fix(text)\n",
    "\n",
    "\n",
    "def spacy_lemmatize_text(text):\n",
    "    text = nlp(text)\n",
    "    text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])\n",
    "    return text\n",
    "\n",
    "\n",
    "def simple_stemming(text, stemmer=ps):\n",
    "    text = ' '.join([stemmer.stem(word) for word in text.split()])\n",
    "    return text\n",
    "\n",
    "\n",
    "def remove_special_characters(text, remove_digits=False):\n",
    "    pattern = r'[^a-zA-Z0-9\\s]' if not remove_digits else r'[^a-zA-Z\\s]'\n",
    "    text = re.sub(pattern, '', text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def remove_stopwords(text, is_lower_case=False, stopwords=None):\n",
    "    if not stopwords:\n",
    "        stopwords = nltk.corpus.stopwords.words('english')\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    tokens = [token.strip() for token in tokens]\n",
    "    \n",
    "    if is_lower_case:\n",
    "        filtered_tokens = [token for token in tokens if token not in stopwords]\n",
    "    else:\n",
    "        filtered_tokens = [token for token in tokens if token.lower() not in stopwords]\n",
    "    \n",
    "    filtered_text = ' '.join(filtered_tokens)    \n",
    "    return filtered_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\r\n",
      "how are you doing\r\n",
      "I\tam\tdoing\tgreat\r\n",
      ":)\n"
     ]
    }
   ],
   "source": [
    "s = 'hello\\r\\nhow are you doing\\r\\nI\\tam\\tdoing\\tgreat\\r\\n:)'\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hello  how are you doing  I am doing great  :)'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.translate(s.maketrans(\"\\n\\t\\r\", \"   \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "\n",
    "def text_pre_processor(text, html_strip=True, accented_char_removal=True, contraction_expansion=True,\n",
    "                       text_lower_case=True, text_stemming=False, text_lemmatization=True, \n",
    "                       special_char_removal=True, remove_digits=True, stopword_removal=True, \n",
    "                       stopword_list=None):\n",
    "    \n",
    "    # strip HTML\n",
    "    if html_strip:\n",
    "        text = strip_html_tags(text)\n",
    "    \n",
    "    # remove extra newlines (often might be present in really noisy text)\n",
    "    text = text.translate(text.maketrans(\"\\n\\t\\r\", \"   \"))\n",
    "    \n",
    "    # remove accented characters\n",
    "    if accented_char_removal:\n",
    "        text = remove_accented_chars(text)\n",
    "    \n",
    "    # expand contractions    \n",
    "    if contraction_expansion:\n",
    "        text = expand_contractions(text)\n",
    "        \n",
    "    \n",
    "    # lemmatize text\n",
    "    if text_lemmatization:\n",
    "        text = spacy_lemmatize_text(text) \n",
    "        \n",
    "    # remove special characters and\\or digits    \n",
    "    if special_char_removal:\n",
    "        # insert spaces between special characters to isolate them    \n",
    "        special_char_pattern = re.compile(r'([{.(-)!}])')\n",
    "        text = special_char_pattern.sub(\" \\\\1 \", text)\n",
    "        text = remove_special_characters(text, remove_digits=remove_digits)  \n",
    "        \n",
    "    # stem text\n",
    "    if text_stemming and not text_lemmatization:\n",
    "        text = simple_stemming(text)\n",
    "        \n",
    "    # lowercase the text    \n",
    "    if text_lower_case:\n",
    "        text = text.lower()\n",
    "        \n",
    "        \n",
    "    # remove stopwords\n",
    "    if stopword_removal:\n",
    "        text = remove_stopwords(text, is_lower_case=text_lower_case, \n",
    "                                stopwords=stopword_list)\n",
    "        \n",
    "    # remove extra whitespace\n",
    "    text = re.sub(' +', ' ', text)\n",
    "    text = text.strip()\n",
    "    \n",
    "    return text\n",
    "\n",
    "  \n",
    "def corpus_pre_processor(corpus):\n",
    "  norm_corpus = []\n",
    "  for doc in tqdm.tqdm(corpus):\n",
    "    norm_corpus.append(text_pre_processor(doc))\n",
    "  return norm_corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test on a single document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>Héllo! Héllo! can you hear me! I just heard about <b>Python</b>!<br/>\r\n",
      " \n",
      "              It's an amazing language which can be used for [Scripting\tWeb development\tBackend development],\r\n",
      "\r\n",
      "\n",
      "              Information Retrieval, Natural Language Processing, Machine Learning & Artificial Intelligence!\n",
      "\n",
      "              What are you waiting for? Go and get started.<br/> He's learning, she's learning, they've already\n",
      "\n",
      "\n",
      "              got a headstart! GET PYTHON 3.6 NOW!</p>\n",
      "           \n"
     ]
    }
   ],
   "source": [
    "document = \"\"\"<p>Héllo! Héllo! can you hear me! I just heard about <b>Python</b>!<br/>\\r\\n \n",
    "              It's an amazing language which can be used for [Scripting\\tWeb development\\tBackend development],\\r\\n\\r\\n\n",
    "              Information Retrieval, Natural Language Processing, Machine Learning & Artificial Intelligence!\\n\n",
    "              What are you waiting for? Go and get started.<br/> He's learning, she's learning, they've already\\n\\n\n",
    "              got a headstart! GET PYTHON 3.6 NOW!</p>\n",
    "           \"\"\"\n",
    "print(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hello hello hear hear python amazing language use scripting web development backend development information retrieval natural language processing machine learning artificial intelligence wait go get start learn learn already get headstart get python'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_pre_processor(document)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test on a corpus of documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [\"\"\"<p>Héllo! Héllo! can you hear me! I just heard about <b>Python</b>!<br/>\\r\\n \n",
    "              It's an amazing language which can be used for [Scripting\\tWeb development\\tBackend development],\\r\\n\\r\\n\n",
    "              Information Retrieval, Natural Language Processing, Machine Learning & Artificial Intelligence!\\n\n",
    "              What are you waiting for? Go and get started.<br/> He's learning, she's learning, they've already\\n\\n\n",
    "              got a headstart! GET PYTHON 3.6 NOW!</p>\n",
    "           \"\"\",\n",
    "          \"\"\"US unveils world's most powerful supercomputer, beats China. \n",
    "             The US has unveiled the world's most powerful supercomputer \n",
    "             called 'Summit', beating the previous record-holder China's Sunway \n",
    "             TaihuLight. With a peak performance of 200,000 trillion calculations \n",
    "             per second, it is over twice as fast as Sunway TaihuLight, which is capable \n",
    "             of 93,000 trillion calculations per second. Summit has 4,608 servers, \n",
    "             which reportedly take up the size of two tennis courts.\"\"\",\n",
    "          \"\"\"The Lord of the Rings is an epic high fantasy novel written by English author and scholar J. R. R. Tolkien. \n",
    "            The story began as a sequel to Tolkien's 1937 fantasy novel The Hobbit, but eventually developed into \n",
    "            a much larger work. Written in stages between 1937 and 1949, The Lord of the Rings is one of the \n",
    "            best-selling novels ever written, with over 150 million copies sold.[1]\n",
    "          \"\"\",\n",
    "          \"\"\"The title of the novel refers to the story's main antagonist, the Dark Lord Sauron,[a] \n",
    "             who had in an earlier age created the One Ring to rule the other Rings of Power as the ultimate weapon \n",
    "             in his campaign to conquer and rule all of Middle-earth. From quiet beginnings in the Shire, a hobbit \n",
    "             land not unlike the English countryside, the story ranges across Middle-earth, following the course \n",
    "             of the War of the Ring through the eyes of its characters, not only the hobbits Frodo Baggins, \n",
    "             Samwise \"Sam\" Gamgee, Meriadoc \"Merry\" Brandybuck and Peregrin \"Pippin\" Took, but also the hobbits' \n",
    "             chief allies and travelling companions: the Men, Aragorn, a Ranger of the North, and Boromir, \n",
    "             a Captain of Gondor; Gimli, a Dwarf warrior; Legolas Greenleaf, an Elven prince; and Gandalf, a wizard.\n",
    "          \"\"\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:00<00:00, 34.64it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['hello hello hear hear python amazing language use scripting web development backend development information retrieval natural language processing machine learning artificial intelligence wait go get start learn learn already get headstart get python',\n",
       " 'us unveil world powerful supercomputer beat china us unveil world powerful supercomputer call summit beat previous record holder china sunway taihulight peak performance trillion calculation per second twice fast sunway taihulight capable trillion calculation per second summit server reportedly take size two tennis court']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_docs = corpus_pre_processor(corpus)\n",
    "norm_docs[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}