auto_label_eval_news.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auto Labeller\n",
    "Applied to news dataset to evaluate model performance\n",
    "* entire dataset\n",
    "* full dictionary size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard Libary Imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "\n",
    "from src.toolkit.autolabel import Preprocessor, AutoLabeller, check_labels\n",
    "from src.toolkit.autolabel import recommend_words, Evaluator\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file path to text data\n",
    "text_path = \"data/news/news.csv\"\n",
    "labelled_path = \"data/news/news_labelled.csv\"  # INPUT YOUR PREFERED OUTPUT PATH\n",
    "score_path = \"data/news/news_score.csv\"\n",
    "labels_path = \"data/news/news_labels.csv\"  # INPUT PATH TO LABELS DICTIONARY\n",
    "\n",
    "stopwords_path = \"data/stopwords.csv\"  ## ADJUST IF YOU HAVE CUSTOM STOPWORDS\n",
    "text_column_name = \"content\"\n",
    "\n",
    "news = pd.read_csv(text_path)\n",
    "data = news[[text_column_name]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unions representing workers at Turner   Newall...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SPACE.com - TORONTO, Canada -- A second\\team o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AP - A company founded by a chemistry research...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AP - It's barely dawn when Mike Fitzpatrick st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AP - Southern California's smog-fighting agenc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             content\n",
       "0  Unions representing workers at Turner   Newall...\n",
       "1  SPACE.com - TORONTO, Canada -- A second\\team o...\n",
       "2  AP - A company founded by a chemistry research...\n",
       "3  AP - It's barely dawn when Mike Fitzpatrick st...\n",
       "4  AP - Southern California's smog-fighting agenc..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jinjun/Desktop/aisg/makerspace/bricks/bricks-auto-labeller/env/lib/python3.7/site-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "corpus = data[text_column_name]\n",
    "\n",
    "preprocessor = Preprocessor()\n",
    "\n",
    "# Text Preprocessing\n",
    "preprocessed_corpus = preprocessor.corpus_preprocess(corpus=corpus, stopwords_path=stopwords_path)\n",
    "\n",
    "# Replace bigrams\n",
    "data[text_column_name] = preprocessor.corpus_replace_bigrams(corpus=preprocessed_corpus, min_df=50, max_df=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recommended themes and words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>kill</td>\n",
       "      <td>people</td>\n",
       "      <td>baghdad</td>\n",
       "      <td>iraq</td>\n",
       "      <td>least</td>\n",
       "      <td>iraqi</td>\n",
       "      <td>bomb</td>\n",
       "      <td>police</td>\n",
       "      <td>wound</td>\n",
       "      <td>car</td>\n",
       "      <td>attack</td>\n",
       "      <td>city</td>\n",
       "      <td>official_say</td>\n",
       "      <td>soldier</td>\n",
       "      <td>injure</td>\n",
       "      <td>insurgent</td>\n",
       "      <td>explode</td>\n",
       "      <td>force</td>\n",
       "      <td>troop</td>\n",
       "      <td>three</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gt</td>\n",
       "      <td>lt</td>\n",
       "      <td>font</td>\n",
       "      <td>strong</td>\n",
       "      <td>color</td>\n",
       "      <td>size</td>\n",
       "      <td>sans</td>\n",
       "      <td>arial</td>\n",
       "      <td>serif</td>\n",
       "      <td>verdana</td>\n",
       "      <td>helvetica</td>\n",
       "      <td>face</td>\n",
       "      <td>br</td>\n",
       "      <td>washington</td>\n",
       "      <td>http</td>\n",
       "      <td>nobr</td>\n",
       "      <td>em</td>\n",
       "      <td>washingtonpostcom</td>\n",
       "      <td>fff</td>\n",
       "      <td>analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>quot</td>\n",
       "      <td>get</td>\n",
       "      <td>call</td>\n",
       "      <td>go</td>\n",
       "      <td>tell</td>\n",
       "      <td>player</td>\n",
       "      <td>look</td>\n",
       "      <td>word</td>\n",
       "      <td>news</td>\n",
       "      <td>spokesman</td>\n",
       "      <td>merriamwebster</td>\n",
       "      <td>many</td>\n",
       "      <td>like</td>\n",
       "      <td>brother</td>\n",
       "      <td>microsoft</td>\n",
       "      <td>dictionary</td>\n",
       "      <td>see</td>\n",
       "      <td>death</td>\n",
       "      <td>come</td>\n",
       "      <td>hit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>high</td>\n",
       "      <td>stock</td>\n",
       "      <td>price</td>\n",
       "      <td>oil_price</td>\n",
       "      <td>oil</td>\n",
       "      <td>low</td>\n",
       "      <td>record</td>\n",
       "      <td>crude</td>\n",
       "      <td>barrel</td>\n",
       "      <td>investor</td>\n",
       "      <td>dollar</td>\n",
       "      <td>rise</td>\n",
       "      <td>market</td>\n",
       "      <td>energy</td>\n",
       "      <td>new_reuters</td>\n",
       "      <td>supply</td>\n",
       "      <td>rate</td>\n",
       "      <td>economic</td>\n",
       "      <td>london</td>\n",
       "      <td>concern</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>season</td>\n",
       "      <td>team</td>\n",
       "      <td>victory</td>\n",
       "      <td>lead</td>\n",
       "      <td>sunday</td>\n",
       "      <td>night</td>\n",
       "      <td>run</td>\n",
       "      <td>second</td>\n",
       "      <td>saturday</td>\n",
       "      <td>play</td>\n",
       "      <td>cup</td>\n",
       "      <td>final</td>\n",
       "      <td>point</td>\n",
       "      <td>first</td>\n",
       "      <td>last</td>\n",
       "      <td>champion</td>\n",
       "      <td>championship</td>\n",
       "      <td>beat</td>\n",
       "      <td>score</td>\n",
       "      <td>coach</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>service</td>\n",
       "      <td>software</td>\n",
       "      <td>internet</td>\n",
       "      <td>announce</td>\n",
       "      <td>computer</td>\n",
       "      <td>search</td>\n",
       "      <td>technology</td>\n",
       "      <td>microsoft</td>\n",
       "      <td>use</td>\n",
       "      <td>system</td>\n",
       "      <td>phone</td>\n",
       "      <td>launch</td>\n",
       "      <td>mobile</td>\n",
       "      <td>user</td>\n",
       "      <td>release</td>\n",
       "      <td>version</td>\n",
       "      <td>network</td>\n",
       "      <td>product</td>\n",
       "      <td>business</td>\n",
       "      <td>music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>lt_gt</td>\n",
       "      <td>lt_href</td>\n",
       "      <td>http_wwwinvestorreuterscomfullquoteaspx</td>\n",
       "      <td>targetstocksquickinfofullquote_gt</td>\n",
       "      <td>new_reuters</td>\n",
       "      <td>inc</td>\n",
       "      <td>co</td>\n",
       "      <td>chicago</td>\n",
       "      <td>quarterly</td>\n",
       "      <td>business</td>\n",
       "      <td>earnings</td>\n",
       "      <td>profit</td>\n",
       "      <td>billion</td>\n",
       "      <td>maker</td>\n",
       "      <td>company_say</td>\n",
       "      <td>would</td>\n",
       "      <td>buy</td>\n",
       "      <td>drug</td>\n",
       "      <td>stock</td>\n",
       "      <td>percent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>government</td>\n",
       "      <td>united</td>\n",
       "      <td>would</td>\n",
       "      <td>talk</td>\n",
       "      <td>country</td>\n",
       "      <td>european</td>\n",
       "      <td>leader</td>\n",
       "      <td>nuclear</td>\n",
       "      <td>group</td>\n",
       "      <td>state</td>\n",
       "      <td>afp</td>\n",
       "      <td>official</td>\n",
       "      <td>washington</td>\n",
       "      <td>prime_minister</td>\n",
       "      <td>federal</td>\n",
       "      <td>former</td>\n",
       "      <td>could</td>\n",
       "      <td>election</td>\n",
       "      <td>international</td>\n",
       "      <td>nation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>athens</td>\n",
       "      <td>olympic</td>\n",
       "      <td>gold</td>\n",
       "      <td>medal</td>\n",
       "      <td>american</td>\n",
       "      <td>men</td>\n",
       "      <td>second</td>\n",
       "      <td>greece</td>\n",
       "      <td>woman</td>\n",
       "      <td>meter</td>\n",
       "      <td>champion</td>\n",
       "      <td>united</td>\n",
       "      <td>olympics</td>\n",
       "      <td>basketball</td>\n",
       "      <td>time</td>\n",
       "      <td>become</td>\n",
       "      <td>final</td>\n",
       "      <td>silver</td>\n",
       "      <td>gymnastics</td>\n",
       "      <td>competition</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>israeli</td>\n",
       "      <td>palestinian</td>\n",
       "      <td>gaza</td>\n",
       "      <td>israel</td>\n",
       "      <td>army</td>\n",
       "      <td>kill</td>\n",
       "      <td>militant</td>\n",
       "      <td>leader</td>\n",
       "      <td>military</td>\n",
       "      <td>camp</td>\n",
       "      <td>refugee</td>\n",
       "      <td>fire</td>\n",
       "      <td>prime_minister</td>\n",
       "      <td>jerusalem</td>\n",
       "      <td>troop</td>\n",
       "      <td>settlement</td>\n",
       "      <td>security</td>\n",
       "      <td>rocket</td>\n",
       "      <td>egyptian</td>\n",
       "      <td>border</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>percent</td>\n",
       "      <td>sale</td>\n",
       "      <td>research</td>\n",
       "      <td>profit</td>\n",
       "      <td>share</td>\n",
       "      <td>rise</td>\n",
       "      <td>quote_profile</td>\n",
       "      <td>quarter</td>\n",
       "      <td>quarterly</td>\n",
       "      <td>thirdquarter</td>\n",
       "      <td>maker</td>\n",
       "      <td>inc</td>\n",
       "      <td>earnings</td>\n",
       "      <td>post</td>\n",
       "      <td>strong</td>\n",
       "      <td>large</td>\n",
       "      <td>revenue</td>\n",
       "      <td>cut</td>\n",
       "      <td>big</td>\n",
       "      <td>loss</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0            1                                        2   \\\n",
       "0         kill       people                                  baghdad   \n",
       "1           gt           lt                                     font   \n",
       "2         quot          get                                     call   \n",
       "3         high        stock                                    price   \n",
       "4       season         team                                  victory   \n",
       "5      service     software                                 internet   \n",
       "6        lt_gt      lt_href  http_wwwinvestorreuterscomfullquoteaspx   \n",
       "7   government       united                                    would   \n",
       "8       athens      olympic                                     gold   \n",
       "9      israeli  palestinian                                     gaza   \n",
       "10     percent         sale                                 research   \n",
       "\n",
       "                                   3            4         5              6   \\\n",
       "0                                iraq        least     iraqi           bomb   \n",
       "1                              strong        color      size           sans   \n",
       "2                                  go         tell    player           look   \n",
       "3                           oil_price          oil       low         record   \n",
       "4                                lead       sunday     night            run   \n",
       "5                            announce     computer    search     technology   \n",
       "6   targetstocksquickinfofullquote_gt  new_reuters       inc             co   \n",
       "7                                talk      country  european         leader   \n",
       "8                               medal     american       men         second   \n",
       "9                              israel         army      kill       militant   \n",
       "10                             profit        share      rise  quote_profile   \n",
       "\n",
       "           7          8             9               10        11  \\\n",
       "0      police      wound           car          attack      city   \n",
       "1       arial      serif       verdana       helvetica      face   \n",
       "2        word       news     spokesman  merriamwebster      many   \n",
       "3       crude     barrel      investor          dollar      rise   \n",
       "4      second   saturday          play             cup     final   \n",
       "5   microsoft        use        system           phone    launch   \n",
       "6     chicago  quarterly      business        earnings    profit   \n",
       "7     nuclear      group         state             afp  official   \n",
       "8      greece      woman         meter        champion    united   \n",
       "9      leader   military          camp         refugee      fire   \n",
       "10    quarter  quarterly  thirdquarter           maker       inc   \n",
       "\n",
       "                12              13           14          15            16  \\\n",
       "0     official_say         soldier       injure   insurgent       explode   \n",
       "1               br      washington         http        nobr            em   \n",
       "2             like         brother    microsoft  dictionary           see   \n",
       "3           market          energy  new_reuters      supply          rate   \n",
       "4            point           first         last    champion  championship   \n",
       "5           mobile            user      release     version       network   \n",
       "6          billion           maker  company_say       would           buy   \n",
       "7       washington  prime_minister      federal      former         could   \n",
       "8         olympics      basketball         time      become         final   \n",
       "9   prime_minister       jerusalem        troop  settlement      security   \n",
       "10        earnings            post       strong       large       revenue   \n",
       "\n",
       "                   17             18           19  \n",
       "0               force          troop        three  \n",
       "1   washingtonpostcom            fff     analysis  \n",
       "2               death           come          hit  \n",
       "3            economic         london      concern  \n",
       "4                beat          score        coach  \n",
       "5             product       business        music  \n",
       "6                drug          stock      percent  \n",
       "7            election  international       nation  \n",
       "8              silver     gymnastics  competition  \n",
       "9              rocket       egyptian       border  \n",
       "10                cut            big         loss  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_words = 20  # CHANGE THE NUMBER OF WORDS RECOMMENDED (IF YOU WANT TO)\n",
    "\n",
    "# Returns a matrix of recommended words\n",
    "topic_model, dtm, best_n = recommend_words(corpus) \n",
    "topic_model.show_topics(dtm=dtm, best_n=best_n, n_words=n_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Business</th>\n",
       "      <th>SciTech</th>\n",
       "      <th>Sports</th>\n",
       "      <th>World</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>stock</td>\n",
       "      <td>software</td>\n",
       "      <td>olympic</td>\n",
       "      <td>iraq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>price</td>\n",
       "      <td>internet</td>\n",
       "      <td>champion</td>\n",
       "      <td>soldier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>oil_price</td>\n",
       "      <td>network</td>\n",
       "      <td>basketball</td>\n",
       "      <td>washington</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sale</td>\n",
       "      <td>technology</td>\n",
       "      <td>champion</td>\n",
       "      <td>international</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>percent</td>\n",
       "      <td>research</td>\n",
       "      <td>silver</td>\n",
       "      <td>official_say</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Business     SciTech      Sports          World\n",
       "0      stock    software     olympic           iraq\n",
       "1      price    internet    champion        soldier\n",
       "2  oil_price     network  basketball     washington\n",
       "3       sale  technology    champion  international\n",
       "4    percent    research      silver   official_say"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = pd.read_csv(labels_path)\n",
    "labels = check_labels(data[[text_column_name]], labels)\n",
    "labels.head(5)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Business</th>\n",
       "      <th>SciTech</th>\n",
       "      <th>Sports</th>\n",
       "      <th>World</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>stock</td>\n",
       "      <td>software</td>\n",
       "      <td>olympic</td>\n",
       "      <td>iraq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>price</td>\n",
       "      <td>internet</td>\n",
       "      <td>champion</td>\n",
       "      <td>soldier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>oil_price</td>\n",
       "      <td>network</td>\n",
       "      <td>basketball</td>\n",
       "      <td>washington</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sale</td>\n",
       "      <td>technology</td>\n",
       "      <td>champion</td>\n",
       "      <td>international</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>percent</td>\n",
       "      <td>research</td>\n",
       "      <td>silver</td>\n",
       "      <td>official_say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>business</td>\n",
       "      <td>computer</td>\n",
       "      <td>medal</td>\n",
       "      <td>competition</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>revenue</td>\n",
       "      <td>microsoft</td>\n",
       "      <td>season</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Business     SciTech      Sports          World\n",
       "0      stock    software     olympic           iraq\n",
       "1      price    internet    champion        soldier\n",
       "2  oil_price     network  basketball     washington\n",
       "3       sale  technology    champion  international\n",
       "4    percent    research      silver   official_say\n",
       "5   business    computer       medal    competition\n",
       "6    revenue   microsoft      season            NaN"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.head(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Business</th>\n",
       "      <th>SciTech</th>\n",
       "      <th>Sports</th>\n",
       "      <th>World</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>profit</td>\n",
       "      <td>gt</td>\n",
       "      <td>silver</td>\n",
       "      <td>competition</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>crude</td>\n",
       "      <td>sans</td>\n",
       "      <td>competition</td>\n",
       "      <td>soldier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>worry</td>\n",
       "      <td>network</td>\n",
       "      <td>olympics</td>\n",
       "      <td>najaf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fell</td>\n",
       "      <td>business</td>\n",
       "      <td>team</td>\n",
       "      <td>clash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>investor</td>\n",
       "      <td>helvetica</td>\n",
       "      <td>athens</td>\n",
       "      <td>international</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>fuel</td>\n",
       "      <td>microsoft</td>\n",
       "      <td>individual</td>\n",
       "      <td>samarra</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>business</td>\n",
       "      <td>internet</td>\n",
       "      <td>basketball</td>\n",
       "      <td>injure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>energy</td>\n",
       "      <td>software</td>\n",
       "      <td>gold</td>\n",
       "      <td>alsadr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>rise</td>\n",
       "      <td>antispam</td>\n",
       "      <td>event</td>\n",
       "      <td>moqtada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>earnings</td>\n",
       "      <td>spyware</td>\n",
       "      <td>gymnastics</td>\n",
       "      <td>bomb</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>percent</td>\n",
       "      <td>lt</td>\n",
       "      <td>cyclist</td>\n",
       "      <td>insurgent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>oil_price</td>\n",
       "      <td>serif</td>\n",
       "      <td>bronze</td>\n",
       "      <td>wound</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>low</td>\n",
       "      <td>arial</td>\n",
       "      <td>olympic</td>\n",
       "      <td>iraq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>price</td>\n",
       "      <td>verdana</td>\n",
       "      <td>champion</td>\n",
       "      <td>baghdad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>sharp</td>\n",
       "      <td>size</td>\n",
       "      <td>finish</td>\n",
       "      <td>iraqi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>sale</td>\n",
       "      <td>color</td>\n",
       "      <td>swim</td>\n",
       "      <td>car</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>new_reuters</td>\n",
       "      <td>research</td>\n",
       "      <td>allaround</td>\n",
       "      <td>holy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>stock</td>\n",
       "      <td>br</td>\n",
       "      <td>meter</td>\n",
       "      <td>troop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>share</td>\n",
       "      <td>font</td>\n",
       "      <td>medal</td>\n",
       "      <td>official_say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>economic</td>\n",
       "      <td>washington</td>\n",
       "      <td>men</td>\n",
       "      <td>kill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>barrel</td>\n",
       "      <td>computer</td>\n",
       "      <td>mens</td>\n",
       "      <td>usiraqi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>oil</td>\n",
       "      <td>technology</td>\n",
       "      <td>greece</td>\n",
       "      <td>shiite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>concern</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>washington</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>high</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>cleric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>military</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Business     SciTech       Sports          World\n",
       "0        profit          gt       silver    competition\n",
       "1         crude        sans  competition        soldier\n",
       "2         worry     network     olympics          najaf\n",
       "3          fell    business         team          clash\n",
       "4      investor   helvetica       athens  international\n",
       "5          fuel   microsoft   individual        samarra\n",
       "6      business    internet   basketball         injure\n",
       "7        energy    software         gold         alsadr\n",
       "8          rise    antispam        event        moqtada\n",
       "9      earnings     spyware   gymnastics           bomb\n",
       "10      percent          lt      cyclist      insurgent\n",
       "11    oil_price       serif       bronze          wound\n",
       "12          low       arial      olympic           iraq\n",
       "13        price     verdana     champion        baghdad\n",
       "14        sharp        size       finish          iraqi\n",
       "15         sale       color         swim            car\n",
       "16  new_reuters    research    allaround           holy\n",
       "17        stock          br        meter          troop\n",
       "18        share        font        medal   official_say\n",
       "19     economic  washington          men           kill\n",
       "20       barrel    computer         mens        usiraqi\n",
       "21          oil  technology       greece         shiite\n",
       "22      concern        None         None     washington\n",
       "23         high        None         None         cleric\n",
       "24         None        None         None       military"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autoLabeller = AutoLabeller(labels.head(6), corpus, data)\n",
    "enriched_labels = autoLabeller.train()\n",
    "\n",
    "enriched_labels  ## Enriched suggested labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnb = MultinomialNB()\n",
    "ypred = autoLabeller.apply(mnb, 'content')\n",
    "ypred.to_csv(labelled_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Business</th>\n",
       "      <th>SciTech</th>\n",
       "      <th>Sports</th>\n",
       "      <th>World</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.4759</td>\n",
       "      <td>0.5988</td>\n",
       "      <td>0.5897</td>\n",
       "      <td>0.7022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.6880</td>\n",
       "      <td>0.4937</td>\n",
       "      <td>0.5913</td>\n",
       "      <td>0.6890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score</th>\n",
       "      <td>0.5626</td>\n",
       "      <td>0.5412</td>\n",
       "      <td>0.5905</td>\n",
       "      <td>0.6955</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Business  SciTech  Sports   World\n",
       "Precision    0.4759   0.5988  0.5897  0.7022\n",
       "Recall       0.6880   0.4937  0.5913  0.6890\n",
       "F1-score     0.5626   0.5412  0.5905  0.6955"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator = Evaluator()\n",
    "score = evaluator.evaluate_predictions(news, ypred, labels, split=0.2, random_state=42)\n",
    "score.to_csv(score_path, index=False)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Automatic Labeling</th>\n",
       "      <th>MLP Neural Network</th>\n",
       "      <th>Gradient Boosted Trees</th>\n",
       "      <th>Random Forest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.592</td>\n",
       "      <td>0.804</td>\n",
       "      <td>0.814</td>\n",
       "      <td>0.872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.616</td>\n",
       "      <td>0.681</td>\n",
       "      <td>0.454</td>\n",
       "      <td>0.511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score</th>\n",
       "      <td>0.597</td>\n",
       "      <td>0.736</td>\n",
       "      <td>0.583</td>\n",
       "      <td>0.644</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Automatic Labeling  MLP Neural Network  Gradient Boosted Trees  \\\n",
       "Precision               0.592               0.804                   0.814   \n",
       "Recall                  0.616               0.681                   0.454   \n",
       "F1-score                0.597               0.736                   0.583   \n",
       "\n",
       "           Random Forest  \n",
       "Precision          0.872  \n",
       "Recall             0.511  \n",
       "F1-score           0.644  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.compare_to_other_models(score, news, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>Business</th>\n",
       "      <th>SciTech</th>\n",
       "      <th>Sports</th>\n",
       "      <th>World</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unions representing workers at Turner   Newall...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SPACE.com - TORONTO, Canada -- A second\\team o...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AP - A company founded by a chemistry research...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AP - It's barely dawn when Mike Fitzpatrick st...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AP - Southern California's smog-fighting agenc...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             content  Business  SciTech  \\\n",
       "0  Unions representing workers at Turner   Newall...       0.0      0.0   \n",
       "1  SPACE.com - TORONTO, Canada -- A second\\team o...       0.0      0.0   \n",
       "2  AP - A company founded by a chemistry research...       1.0      1.0   \n",
       "3  AP - It's barely dawn when Mike Fitzpatrick st...       0.0      0.0   \n",
       "4  AP - Southern California's smog-fighting agenc...       0.0      0.0   \n",
       "\n",
       "   Sports  World  \n",
       "0     0.0    0.0  \n",
       "1     0.0    0.0  \n",
       "2     0.0    0.0  \n",
       "3     0.0    0.0  \n",
       "4     0.0    1.0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demo = news[['content']].join(ypred)\n",
    "demo.head(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}