Custom_Sentiment_API_Features_IdiomPhrases.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re,string\n",
    "import math, re, string, requests, json\n",
    "from itertools import product\n",
    "from inspect import getsourcefile\n",
    "from os.path import abspath, join, dirname\n",
    "import nbimporter\n",
    "from slistener import SListener\n",
    "import time, tweepy, sys\n",
    "import pandas as pd\n",
    "import json\n",
    "from pandas.io.json import json_normalize\n",
    "from sklearn.feature_extraction.text import CountVectorizer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "REMOVE_PUNCTUATION = re.compile('[%s]' % re.escape(string.punctuation))\n",
    "\n",
    "PUNCTUATION_LIST = [\".\", \"!\", \"?\", \",\", \";\", \":\", \"-\", \"'\", \"\\\"\",\n",
    "             \"!!\", \"!!!\", \"??\", \"???\", \"?!?\", \"!?!\", \"?!?!\", \"!?!?\"]\n",
    "\n",
    "NEGATION_WORDS = [\"aint\", \"arent\", \"cannot\", \"cant\", \"couldnt\", \"darent\", \"didnt\", \"doesnt\",\n",
    " \"ain't\", \"aren't\", \"can't\", \"couldn't\", \"daren't\", \"didn't\", \"doesn't\",\n",
    " \"dont\", \"hadnt\", \"hasnt\", \"havent\", \"isnt\", \"mightnt\", \"mustnt\", \"neither\",\n",
    " \"don't\", \"hadn't\", \"hasn't\", \"haven't\", \"isn't\", \"mightn't\", \"mustn't\",\n",
    " \"neednt\", \"needn't\", \"never\", \"none\", \"nope\", \"nor\", \"not\", \"nothing\", \"nowhere\",\n",
    " \"oughtnt\", \"shant\", \"shouldnt\", \"uhuh\", \"wasnt\", \"werent\",\n",
    " \"oughtn't\", \"shan't\", \"shouldn't\", \"uh-uh\", \"wasn't\", \"weren't\",\n",
    " \"without\", \"wont\", \"wouldnt\", \"won't\", \"wouldn't\", \"rarely\", \"seldom\", \"despite\"]\n",
    "\n",
    "CONTRASTING_CONJUNCTION = \\\n",
    "{\"but\", \"although\", \"though\", \"even though\", \"even if\", \"however\"}\n",
    "\n",
    "# Sentiment Incremental score for capitalization approach.\n",
    "CAPS_INCR = 1.00\n",
    "\n",
    "# Sentiment score for Punctuation Emphsis symbols\n",
    "QUESTION_INCREASE = 0.1\n",
    "EXCLAMATION_INCREASE = 0.25\n",
    "NEGATION_MODIFIER = -1.00\n",
    "\n",
    "# Score for Degree Modifiers\n",
    "\n",
    "SCORE_INCREASE = 0.5\n",
    "SCORE_DECREASE = -0.5\n",
    "\n",
    "\n",
    "DEGREE_DICTIONARY = \\\n",
    "    {\"100 percent\": SCORE_INCREASE, \"a good deal\": SCORE_INCREASE, \"a great deal\": SCORE_INCREASE,\n",
    "     \"a lot\": SCORE_INCREASE,\n",
    "     \"aboundingly\": SCORE_INCREASE, \"absolutely\": SCORE_INCREASE, \"absurdly\": SCORE_INCREASE,\n",
    "     \"abundantly\": SCORE_INCREASE,\n",
    "     \"admirably\": SCORE_INCREASE, \"alarmingly\": SCORE_INCREASE, \"amazingly\": SCORE_INCREASE,\n",
    "     \"astronomically\": SCORE_INCREASE, \"awfully\": SCORE_INCREASE, \"breathtakingly\": SCORE_INCREASE,\n",
    "     \"clearly\": SCORE_INCREASE, \"completely\": SCORE_INCREASE, \"considerably\": SCORE_INCREASE, \"crazy\": SCORE_INCREASE,\n",
    "     \"damn\": SCORE_INCREASE, \"damned\": SCORE_INCREASE, \"darn\": SCORE_INCREASE, \"darned\": SCORE_INCREASE,\n",
    "     \"decidedly\": SCORE_INCREASE, \"deeply\": SCORE_INCREASE, \"deservedly\": SCORE_INCREASE, \"downright\": SCORE_INCREASE,\n",
    "     \"dreadfully\": SCORE_INCREASE,  # up to here so far\n",
    "     \"effing\": SCORE_INCREASE, \"enormously\": SCORE_INCREASE,\n",
    "     \"entirely\": SCORE_INCREASE, \"especially\": SCORE_INCREASE, \"exceptionally\": SCORE_INCREASE,\n",
    "     \"extremely\": SCORE_INCREASE,\n",
    "     \"fabulously\": SCORE_INCREASE, \"flipping\": SCORE_INCREASE, \"flippin\": SCORE_INCREASE,\n",
    "     \"fricking\": SCORE_INCREASE, \"frickin\": SCORE_INCREASE, \"frigging\": SCORE_INCREASE, \"friggin\": SCORE_INCREASE,\n",
    "     \"fully\": SCORE_INCREASE, \"fucking\": SCORE_INCREASE,\n",
    "     \"greatly\": SCORE_INCREASE, \"hella\": SCORE_INCREASE, \"highly\": SCORE_INCREASE, \"hugely\": SCORE_INCREASE,\n",
    "     \"incredibly\": SCORE_INCREASE,\n",
    "     \"intensely\": SCORE_INCREASE, \"majorly\": SCORE_INCREASE, \"more\": SCORE_INCREASE, \"most\": SCORE_INCREASE,\n",
    "     \"particularly\": SCORE_INCREASE,\n",
    "     \"purely\": SCORE_INCREASE, \"quite\": SCORE_INCREASE, \"really\": SCORE_INCREASE, \"remarkably\": SCORE_INCREASE,\n",
    "     \"so\": SCORE_INCREASE, \"substantially\": SCORE_INCREASE,\n",
    "     \"thoroughly\": SCORE_INCREASE, \"totally\": SCORE_INCREASE, \"tremendously\": SCORE_INCREASE,\n",
    "     \"uber\": SCORE_INCREASE, \"unbelievably\": SCORE_INCREASE, \"unusually\": SCORE_INCREASE, \"utterly\": SCORE_INCREASE,\n",
    "     \"very\": SCORE_INCREASE,\n",
    "     \"almost\": SCORE_DECREASE, \"barely\": SCORE_DECREASE, \"hardly\": SCORE_DECREASE, \"just enough\": SCORE_DECREASE,\n",
    "     \n",
    "     \"less\": SCORE_DECREASE, \"little\": SCORE_DECREASE, \"marginally\": SCORE_DECREASE, \"occasionally\": SCORE_DECREASE,\n",
    "     \"partly\": SCORE_DECREASE,\n",
    "     \"scarcely\": SCORE_DECREASE, \"slightly\": SCORE_DECREASE, \"somewhat\": SCORE_DECREASE,\n",
    "     \"sort of\": SCORE_DECREASE, \"sorta\": SCORE_DECREASE, \"sortof\": SCORE_DECREASE, \"sort-of\": SCORE_DECREASE}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Check if there is negation contect in the sentence or not.\n",
    "    Still continuing\n",
    "\"\"\"\n",
    "def negated(input_words, include_nt=True):\n",
    "    \n",
    "    neg_words = []\n",
    "    neg_words.extend(NEGATION_WORDS)\n",
    "      \n",
    "    for word in neg_words:\n",
    "        if word in input_words:\n",
    "            return True\n",
    "    if include_nt:\n",
    "        for word in input_words:\n",
    "            if \"n't\" in word:\n",
    "                return True\n",
    "       \n",
    "    return False\n",
    "\n",
    "def capitalized(words):\n",
    "        \n",
    "        emphasis = False\n",
    "        capitalized_words = 0\n",
    "\n",
    "        for word in words:\n",
    "            if word.isupper():\n",
    "                capitalized_words += 1\n",
    "        cap_differential = len(words) - capitalized_words\n",
    "        if 0 < cap_differential < len(words):\n",
    "            emphasis = True\n",
    "\n",
    "        return emphasis\n",
    "\n",
    "\n",
    "def alter_valence(word, valence, is_cap_diff):\n",
    "   \n",
    "    modifier = 0.0\n",
    "    word_lower = word.lower()\n",
    "    if word_lower in DEGREE_DICTIONARY:\n",
    "        modifier = DEGREE_DICTIONARY[word_lower]\n",
    "        \n",
    "        if valence < 0:\n",
    "            modifier = modifier * -1\n",
    "                    \n",
    "        if word.isupper() and is_cap_diff:\n",
    "            if valence > 0:\n",
    "                modifier += CAPS_INCR\n",
    "            else: modifier -= CAPS_INCR\n",
    "    return modifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SentimentAnalyzer(object):\n",
    "    \n",
    "    def __init__(self, lexicon_file=\"New_Lexicon.txt\"):\n",
    "        _this_module_file_path_ = abspath(getsourcefile(lambda:0))\n",
    "        lexicon_full_filepath = join(dirname(_this_module_file_path_), lexicon_file)\n",
    "        with open(lexicon_full_filepath) as f:\n",
    "            self.lexicon_full_filepath = f.read()\n",
    "        self.lexicon = self.make_lex_dict()\n",
    "    \n",
    "    def make_lex_dict(self):\n",
    "        \n",
    "        lex_dict = {}\n",
    "        for line in self.lexicon_full_filepath.split('\\n'):\n",
    "            (word, measure) = line.strip().split('\\t')[0:2]\n",
    "            lex_dict[word] = float(measure)\n",
    "            \n",
    "        return lex_dict\n",
    "\n",
    "    def polarity_scores(self, text):\n",
    "        \n",
    "        sentimenttext = SentimentText(text)\n",
    "        sentiments = []\n",
    "        words_and_emoticons = sentimenttext.words_and_emoticons\n",
    "        print(words_and_emoticons)\n",
    "        \n",
    "        for item in words_and_emoticons:\n",
    "            valence = 0\n",
    "            i = words_and_emoticons.index(item)\n",
    "            \n",
    "            if (i < len(words_and_emoticons) - 1 and item.lower() == \"kind\" and\n",
    "                        words_and_emoticons[i + 1].lower() == \"of\") and item.lower() in DEGREE_DICTIONARY:\n",
    "                sentiments.append(valence)\n",
    "            \n",
    "                continue\n",
    "\n",
    "            sentiments = self.sentiment_score(valence, sentimenttext, item, i, sentiments)\n",
    "            \n",
    "        sentiments = self._contains_but(words_and_emoticons, sentiments)\n",
    "        \n",
    "        return(sentiments)\n",
    "    \n",
    "    def _contains_negation(self, valence, words_and_emoticons, start_i, i):\n",
    "        \n",
    "       \n",
    "        if start_i == 0:\n",
    "            if negated([words_and_emoticons[i-1]]):\n",
    "                    valence = valence*NEGATION_MODIFIER                   \n",
    "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
    "        if start_i == 1:\n",
    "            if words_and_emoticons[i-2] == \"never\" and\\\n",
    "               (words_and_emoticons[i-1] == \"so\" or\n",
    "                words_and_emoticons[i-1] == \"this\"):\n",
    "                valence = valence*1.5\n",
    "            elif negated([words_and_emoticons[i-(start_i+1)]]):\n",
    "                valence = valence*NEGATION_MODIFIER\n",
    "        if start_i == 2:\n",
    "            if words_and_emoticons[i-3] == \"never\" and \\\n",
    "               (words_and_emoticons[i-2] == \"so\" or words_and_emoticons[i-2] == \"this\") or \\\n",
    "               (words_and_emoticons[i-1] == \"so\" or words_and_emoticons[i-1] == \"this\"):\n",
    "                valence = valence*1.25\n",
    "            elif negated([words_and_emoticons[i-(start_i+1)]]):\n",
    "                valence = valence*NEGATION_MODIFIER\n",
    "\n",
    "        \n",
    "        return valence\n",
    "        \n",
    "        \n",
    "    def sentiment_score(self, valence, sentitext, item, i, sentiments):\n",
    "        \n",
    "        is_cap_diff = sentitext.is_cap_diff\n",
    "        words_and_emoticons = sentitext.words_and_emoticons\n",
    "        item_lowercase = item.lower()\n",
    "        if item_lowercase in self.lexicon:\n",
    "            \n",
    "            valence = self.lexicon[item_lowercase]\n",
    "            \n",
    "            if item.isupper() and is_cap_diff:\n",
    "                if valence > 0:\n",
    "                    valence += CAPS_INCR\n",
    "                else:\n",
    "                    valence -= CAPS_INCR\n",
    "\n",
    "            for start_i in range(0,4):\n",
    "                if i > start_i and words_and_emoticons[i-(start_i+1)].lower() not in self.lexicon:\n",
    "                    s = alter_valence(words_and_emoticons[i-(start_i+1)], valence, is_cap_diff)\n",
    "                    valence = valence+s\n",
    "                    valence = self._contains_negation(valence, words_and_emoticons, start_i, i)\n",
    "                    \n",
    "        \n",
    "        sentiments.append(valence)\n",
    "        return sentiments\n",
    "            \n",
    "    \n",
    "    \"\"\"\n",
    "    Algorithm is defined as such:\n",
    "    The contextual conjective \"but\" clause works in way to calculate the valence score , algorithm states that the there is 50% \n",
    "    reduction in valence score before \"but\" clause words and 150% increase in the valence score after the \"but\" clause\n",
    "    sentence.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    def _contains_but(self,words_and_emoticons, sentiments):\n",
    "       \n",
    "        CONTRASTING_CONJUNCTION = \\\n",
    "        {\"but\", \"although\", \"though\", \"even though\", \"even if\", \"however\"}\n",
    "        \n",
    "        for conj_word in CONTRASTING_CONJUNCTION:\n",
    "            if conj_word in words_and_emoticons:\n",
    "                but_index = words_and_emoticons.index(conj_word)\n",
    "                \n",
    "                for sentiment in sentiments:\n",
    "                    sentiment_index = sentiments.index(sentiment)\n",
    "                    \n",
    "                    if sentiment_index < but_index:\n",
    "                        sentiments.pop(sentiment_index)\n",
    "                        sentiments.insert(sentiment_index, sentiment * 0.5)\n",
    "                    \n",
    "                    elif sentiment_index > but_index:\n",
    "                        sentiments.pop(sentiment_index)\n",
    "                        sentiments.insert(sentiment_index, sentiment * 1.5)   \n",
    "                        \n",
    "        return sentiments          \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SentimentText(object):\n",
    "    \n",
    "    def __init__(self, text):\n",
    "        if not isinstance(text, str):\n",
    "            text = str(text.encode('utf-8'))\n",
    "        self.text = text\n",
    "        self.words_and_emoticons = self._words_and_emoticons()\n",
    "        self.is_cap_diff = capitalized(self.words_and_emoticons)\n",
    "\n",
    "    def _words_and_punc(self):\n",
    "        \n",
    "        no_punc_text = REMOVE_PUNCTUATION.sub('', self.text)\n",
    "        words_only = no_punc_text.split()\n",
    "        \n",
    "        words_only = set( w for w in words_only if len(w) > 1 )\n",
    "        punc_before = {''.join(p): p[1] for p in product(PUNCTUATION_LIST, words_only)}\n",
    "        punc_after = {''.join(p): p[0] for p in product(words_only, PUNCTUATION_LIST)}\n",
    "        words_punc_dict = punc_before\n",
    "        words_punc_dict.update(punc_after)\n",
    "        return words_punc_dict\n",
    "\n",
    "    def _words_and_emoticons(self):\n",
    "        \n",
    "        wes = self.text.split()\n",
    "        words_punc_dict = self._words_and_punc()\n",
    "               \n",
    "        wes = [we for we in wes if len(we) > 1]\n",
    "        for i, we in enumerate(wes):\n",
    "            if we in words_punc_dict:\n",
    "                wes[i] = words_punc_dict[we]\n",
    "        \n",
    "        return wes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Cleaning of textual data remove URLS's and mentions and hashtags\n",
    "\n",
    "def strip_links(text):\n",
    "    link_regex    = re.compile('((https?):((//)|(\\\\\\\\))+([\\w\\d:#@%/;$()~_?\\+-=\\\\\\.&](#!)?)*)', re.DOTALL)\n",
    "    links         = re.findall(link_regex, text)\n",
    "    for link in links:\n",
    "        text = text.replace(link[0], ', ')    \n",
    "    return text\n",
    "\n",
    "def strip_all_entities(text):\n",
    "    entity_prefixes = ['@','#']\n",
    "    for separator in  string.punctuation:\n",
    "        if separator not in entity_prefixes :\n",
    "            text = text.replace(separator,' ')\n",
    "    words = []\n",
    "    for word in text.split():\n",
    "        word = word.strip()\n",
    "        if word:\n",
    "            if word[0] not in entity_prefixes:\n",
    "                words.append(word)\n",
    "    return ' '.join(words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Retrieve text from the json file that is retrived from Tweepy Streaming API\n",
    "\n",
    "raw_objs_string = open('myprefix.20170601-174814.json').read() \n",
    "objs_string = '[%s]'%(raw_objs_string) \n",
    "p = re.compile( '}\\s*{' )\n",
    "jsonstr = p.sub( '}\\n{', objs_string )\n",
    "raw_objs_string = jsonstr.replace(\"}\\n{\", \"},\\n{\")\n",
    "objs = json.loads(raw_objs_string) \n",
    "json_output = json.dumps(objs)\n",
    "df = pd.io.json.json_normalize(objs)\n",
    "textual_tweets = pd.DataFrame(df.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "   \n",
    "    Idioms_sent_score = pd.read_csv(\"Idioms_Sentiment_Score.csv\")\n",
    "    Idioms_valence = Idioms_sent_score[['idiom','Sentiment_Score']]\n",
    "    Convert_to_dict = Idioms_valence.set_index('idiom')['Sentiment_Score'].to_dict()  \n",
    "    analyzer = SentimentAnalyzer()\n",
    "    \n",
    "    for t in textual_tweets['text']:\n",
    "        result = strip_all_entities(strip_links(t))\n",
    "        result = re.sub('[0-9]+', '', result)\n",
    "        result = re.sub('RT ','',result)\n",
    "        result = result.lower()\n",
    "\n",
    "        vs = analyzer.polarity_scores(result)\n",
    "        print(\"{:-<65} {}\".format(result, str(vs))) \n",
    "        \n",
    "        #Building Count vectorizer where text are tokenized into n grams.\n",
    "        \n",
    "        vectorizer = CountVectorizer(ngram_range=(1,6))\n",
    "        analyer = vectorizer.build_analyzer()\n",
    "        result = analyer(result)\n",
    "        ns = set(result)\n",
    "        sum_s = float(sum(vs))\n",
    "        \n",
    "        #Check if idiom is present in the textual tweets.\n",
    "        valence_score = 0.0 \n",
    "        for key, value in Convert_to_dict.items():\n",
    "            if key in ns:\n",
    "                valence_score = value\n",
    "                valence_score = valence_score + sum_s \n",
    "                \n",
    "        ep_count = result.count(\"!\")\n",
    "        ep_amplifier = 0\n",
    "        if ep_count > 4:\n",
    "            ep_count = 4\n",
    "        ep_amplifier = ep_count * EXCLAMATION_INCREASE    \n",
    "    \n",
    "        qm_count = result.count(\"?\")\n",
    "        qm_amplifier = 0\n",
    "        if qm_count > 1:\n",
    "            if qm_count <= 3:\n",
    "                qm_amplifier = qm_count * QUESTION_INCREASE\n",
    "            else:\n",
    "                qm_amplifier = 1.00\n",
    "            \n",
    "        punct_emph_amplifier = ep_amplifier+qm_amplifier\n",
    "        \n",
    "        # Deriving the sentiment score (positive , negative and neutral sentiment)\n",
    "        if valence_score > 0:\n",
    "            valence_score =  valence_score + punct_emph_amplifier\n",
    "        elif  valence_score < 0:\n",
    "            valence_score = valence_score -  punct_emph_amplifier\n",
    "        \n",
    "        pos_sum = 0.0\n",
    "        neg_sum = 0.0\n",
    "        neu_count = 0\n",
    "    \n",
    "        for sentiment_score in vs:\n",
    "            if sentiment_score > 0:\n",
    "                pos_sum += (float(sentiment_score) +1) \n",
    "            if sentiment_score < 0:\n",
    "                neg_sum += (float(sentiment_score) -1) \n",
    "            if sentiment_score == 0:\n",
    "                neu_count += 1      \n",
    "        \n",
    "        if pos_sum > math.fabs(neg_sum):\n",
    "            pos_sum += (punct_emph_amplifier)\n",
    "        elif pos_sum < math.fabs(neg_sum):\n",
    "            neg_sum -= (punct_emph_amplifier)\n",
    "\n",
    "        total = pos_sum + math.fabs(neg_sum) + neu_count\n",
    "        pos = math.fabs(pos_sum / total)\n",
    "        neg = math.fabs(neg_sum / total)\n",
    "        neu = math.fabs(neu_count / total) \n",
    "        sentiment_dict = {\"neg\" : neg,\"neu\" : neu,\"pos\" : pos}\n",
    "        print(sentiment_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}