Sentiment Analysis With spaCy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function numba.decorators._jit.<locals>.wrapper(func)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<function numba.decorators._jit.<locals>.wrapper(func)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<function numba.decorators._jit.<locals>.wrapper(func)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<function numba.decorators._jit.<locals>.wrapper(func)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<function numba.decorators._jit.<locals>.wrapper(func)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14911f87af3548a68c6a3e8c19b40b58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2.482034921646118\n"
     ]
    }
   ],
   "source": [
    "# import packages #\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "from tqdm import tqdm_notebook\n",
    "from numba import jit\n",
    "\n",
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "\n",
    "# for removing accented characters\n",
    "import unicodedata\n",
    "\n",
    "# for expanding contractions\n",
    "from contractions import CONTRACTION_MAP\n",
    "from pycontractions import Contractions\n",
    "\n",
    "# multiple outputs in one chunk (jupyter notebook)\n",
    "from IPython.core.interactiveshell import InteractiveShell \n",
    "InteractiveShell.ast_node_interactivity = 'all'\n",
    "\n",
    "\n",
    "\n",
    "# load corpus #\n",
    "def load_corpus(articles):\n",
    "    '''\n",
    "    Load corpus, 100 articles.\n",
    "    '''\n",
    "    df = pd.read_csv(articles)\n",
    "    df = df.to_dict()\n",
    "    return df \n",
    "\n",
    "\n",
    "\n",
    "# load Loughran & McDonald finance dictionary #\n",
    "jit(nopython = True, parallel = True)\n",
    "def load_LM_dict(dictionary, pos = True, neg = True):\n",
    "    '''\n",
    "    Load positive/negative words list. Adopt dictionary to improve efficiency.\n",
    "    '''\n",
    "    if pos: \n",
    "        poslst = pd.read_excel(dictionary, sheet_name = \"Positive\", header = None)[0].tolist()\n",
    "        posdict = {pos:1 for pos in poslst}\n",
    "    if neg:\n",
    "        neglst = pd.read_excel(dictionary, sheet_name = \"Negative\", header = None)[0].tolist()\n",
    "        negdict = {neg:1 for neg in neglst}\n",
    "    return posdict, negdict\n",
    "\n",
    "\n",
    "\n",
    "# text pre-processing #\n",
    "\n",
    "## remove HTML tags ##\n",
    "def remove_html_tags(text):\n",
    "    '''\n",
    "    Remove the noise, html tags.\n",
    "    '''\n",
    "    bs = BeautifulSoup(text, 'html.parser')\n",
    "    text = bs.get_text()\n",
    "    return text\n",
    "\n",
    "\n",
    "## remove accented characters ##\n",
    "def remove_accented_characters(text):\n",
    "    '''\n",
    "    Convert the corpus into English pharsings.\n",
    "    '''\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "    return text\n",
    "\n",
    "\n",
    "## expand contractions ##\n",
    "def expand_contractions(text, contraction_mapping = CONTRACTION_MAP):\n",
    "    '''\n",
    "    Expand contractions. If necessary, add additional contraction-expansion pairs into contraction.py.\n",
    "    '''\n",
    "    contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())),\n",
    "                                      flags = re.IGNORECASE|re.DOTALL)\n",
    "    def expand_match(contraction):\n",
    "        match = contraction.group(0)\n",
    "        first_char = match[0]\n",
    "        expanded_contraction = contraction_mapping.get(match)\\\n",
    "            if contraction_mapping.get(match)\\\n",
    "            else contraction_mapping.get(match.lower())\n",
    "        expanded_contraction = first_char + expanded_contraction[1:]\n",
    "        return expanded_contraction\n",
    "    expanded_text = contractions_pattern.sub(expand_match, text)\n",
    "    expanded_text = re.sub(\"'\",\"\", expanded_text)\n",
    "    return expanded_text\n",
    "\n",
    "\n",
    "## remove special characters ##\n",
    "def remove_special_characters(text):\n",
    "    '''\n",
    "    Remove puntuations, numbers, redundant spaces.\n",
    "    '''\n",
    "    text = re.sub(r'[^a-zA-Z\\s]', ' ', text)\n",
    "    text = re.sub(' +' ,' ', text)\n",
    "    return text\n",
    "\n",
    "\n",
    "## intialize a nlp model ##\n",
    "jit(nopython = True, parallel = True)\n",
    "def set_custom_boundaries(doc):\n",
    "    '''\n",
    "    Fix the bug in spacy: specify $. and }. as additional forms to end a sentence.\n",
    "    '''\n",
    "    for token in doc[:-1]:\n",
    "        if \"$.\" in token.text or \"}.\" in token.text or token.text == \";\":\n",
    "            doc[token.i+1].is_sent_start = True\n",
    "    return doc\n",
    "\n",
    "def initialization(model):\n",
    "    '''\n",
    "    Multiple statistical models are available in the spaCy library,\n",
    "    including en_core_web_sm, en_core_web_md, en_core_web_lg, en.\n",
    "    '''\n",
    "    nlp = spacy.load(model, disable = ['parser', 'tagger', 'ner'])\n",
    "    try:\n",
    "        nlp.add_pipe(set_custom_boundaries, before = \"parser\")\n",
    "    except:\n",
    "        pass\n",
    "    return nlp\n",
    "\n",
    "\n",
    "# build a text normalizer #\n",
    "jit(nopython = True, parallel = True)\n",
    "def normalizer(text, nlp, stopwords_list,\n",
    "               no_accented_chars = True, no_contracted_chars = True, no_lammas = True, no_stopwords = False):\n",
    "    '''\n",
    "    Build a text normalizer. Assign the Boolean value True to the applied tokenizer. \n",
    "    By defualt, do not remove accented characters, do not expand contractions, \n",
    "    do not do words lemmatization and remove stopwords.\n",
    "    '''\n",
    "    text = remove_html_tags(text)\n",
    "    if not no_accented_chars:\n",
    "        text = remove_accented_characters(text)    \n",
    "    if not no_contracted_chars:\n",
    "        text = expand_contractions(text, contraction_mapping = CONTRACTION_MAP)   \n",
    "    text = remove_special_characters(text).lower()\n",
    "    \n",
    "    text = nlp(text)\n",
    "    if not no_lammas:\n",
    "        \n",
    "        tokens = [word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text]\n",
    "    else:\n",
    "        tokens = [token.text for token in text]\n",
    "    if not no_stopwords:\n",
    "        tokens = [token for token in tokens if not token in stopwords_list]\n",
    "#         tokens = list(filter(lambda x: x not in stopwords_list, tokens))\n",
    "    normalized_tokens = [token.upper() for token in tokens if len(token) > 1]\n",
    "    return normalized_tokens\n",
    "\n",
    "\n",
    "\n",
    "# sentiment analysis #\n",
    "jit(nopython = True, parallel = True)\n",
    "def sentiment(words, posdict, negdict):\n",
    "    '''\n",
    "    Count positive/negative words. \n",
    "    Calculate sentiment score.\n",
    "    Sentiment score is positive words fraction minus negative words fraction in the article content\n",
    "    '''\n",
    "    poswds, negwds = 0, 0\n",
    "    poslst, neglst = [], []\n",
    "    totwds = len(words)\n",
    "    for word in words:\n",
    "        try:\n",
    "            posdict[word] == 1\n",
    "            poswds += 1\n",
    "            poslst.append(word)\n",
    "        except:\n",
    "            try:\n",
    "                negdict[word] == 1\n",
    "                negwds += 1\n",
    "                neglst.append(word)\n",
    "            except:\n",
    "                continue        \n",
    "    sentisc = (poswds - negwds)/totwds \n",
    "    return poswds, negwds, totwds, sentisc, poslst, neglst\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    # record initial time\n",
    "    time_start = time.time()\n",
    "    \n",
    "    # load the corpus, dictionary\n",
    "    corpus = load_corpus(\"articles.csv\")\n",
    "    posdict, negdict = load_LM_dict(\"LoughranMcDonald_SentimentWordLists_2018.xlsx\")\n",
    "    \n",
    "    # update stopwords list, remove overlap: stopwords corpus, LM dictionary\n",
    "    stopwords_list = [word for word in STOP_WORDS if not word.upper() in posdict and not word.upper() in negdict]\n",
    "    \n",
    "    # prepare a dictionary for the output\n",
    "    opdict = {'article_id': list(corpus['aid'].values()), 'pos_word': [], 'neg_word': [], \n",
    "              'total_word': [], 'sentiment': [], 'pos_lst': [], 'neg_lst': []}\n",
    "    \n",
    "    # initialize nlp model\n",
    "    nlp = initialization('en')\n",
    "    \n",
    "    jit(nopython = True, parallel = True)\n",
    "    def output():\n",
    "        for i in tqdm_notebook(range(len(corpus['aid']))):\n",
    "\n",
    "            # text pre-processing\n",
    "            text = corpus['content'][i]\n",
    "            normalized_tokens = normalizer(text, nlp, stopwords_list)\n",
    "\n",
    "            # sentiment analysis\n",
    "            poswds, negwds, totwds, sentisc, poslst, neglst = sentiment(normalized_tokens, posdict, negdict)\n",
    "\n",
    "            # update output\n",
    "            opdict['pos_word'].append(poswds)\n",
    "            opdict['neg_word'].append(negwds)\n",
    "            opdict['total_word'].append(totwds)\n",
    "            opdict['sentiment'].append(sentisc)\n",
    "            opdict['pos_lst'].append(poslst)\n",
    "            opdict['neg_lst'].append(neglst)\n",
    "        \n",
    "        return pd.DataFrame(opdict)\n",
    "    \n",
    "    opdf = output()\n",
    "    opdf[['article_id', 'pos_word', 'neg_word', 'total_word', 'sentiment']].to_csv(\"results_spacy.csv\", index = False)\n",
    "    \n",
    "    \n",
    "    # record finish time\n",
    "    time_end = time.time()\n",
    "    print(time_end - time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}