bricks_auto_label.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auto Labeller - A step by step guide\n",
    "Here we provide a step by step guide for you to experiment with the semi auto-labelling tool."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Standard Library Imports\n",
    "First lets perform the standard library imports."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard Libary Imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from src.toolkit.autolabel import Preprocessor, AutoLabeller, check_labels\n",
    "from src.toolkit.autolabel import recommend_words\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Adjust the Input\n",
    "Input the file path to your text data.\n",
    "\n",
    "The tabular data should have the following format\n",
    "\n",
    "<img style=\"float: left;\" src=\"data/images/input_sample.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overview</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A family wedding reignites the ancient feud be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Cheated on, mistreated and stepped on, the wom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Obsessive master thief, Neil McCauley leads a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>An ugly duckling having undergone a remarkable...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A mischievous young boy, Tom Sawyer, witnesses...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            overview\n",
       "0  A family wedding reignites the ancient feud be...\n",
       "1  Cheated on, mistreated and stepped on, the wom...\n",
       "2  Obsessive master thief, Neil McCauley leads a ...\n",
       "3  An ugly duckling having undergone a remarkable...\n",
       "4  A mischievous young boy, Tom Sawyer, witnesses..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# file path to text data\n",
    "text_path = \"data/movies/movies500.csv\"  ##  PUT YOUR FILEPATH\n",
    "\n",
    "# name of column that contains text data\n",
    "text_column_name = \"overview\"  ##  PUT YOUR TEXT COLUMN\n",
    "\n",
    "data = pd.read_csv(text_path)[[text_column_name]]\n",
    "data.head(5)  ## CHECK THE FORMAT OF YOUR DATA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 - Data Preprocessing\n",
    "Run the following code to prepare your text data for model input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = data[text_column_name]\n",
    "\n",
    "preprocessor = Preprocessor()\n",
    "\n",
    "# Text Preprocessing\n",
    "stopwords_path = \"data/stopwords.csv\"  # STOPWORDS PATH\n",
    "preprocessed_corpus = preprocessor.corpus_preprocess(corpus=corpus, stopwords_path=stopwords_path)\n",
    "\n",
    "# Replace bigrams\n",
    "data[text_column_name] = preprocessor.corpus_replace_bigrams(corpus=preprocessed_corpus, min_df=50, max_df=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4 - Generate recommmended labels\n",
    "Run the following code to generate recommended labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>man</td>\n",
       "      <td>demon</td>\n",
       "      <td>convince</td>\n",
       "      <td>story</td>\n",
       "      <td>discovers</td>\n",
       "      <td>political</td>\n",
       "      <td>guest</td>\n",
       "      <td>friend</td>\n",
       "      <td>true</td>\n",
       "      <td>like</td>\n",
       "      <td>look</td>\n",
       "      <td>know</td>\n",
       "      <td>fear</td>\n",
       "      <td>pretend</td>\n",
       "      <td>nature</td>\n",
       "      <td>confront</td>\n",
       "      <td>descend</td>\n",
       "      <td>bright</td>\n",
       "      <td>personality</td>\n",
       "      <td>hunt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>love</td>\n",
       "      <td>fall</td>\n",
       "      <td>french</td>\n",
       "      <td>student</td>\n",
       "      <td>king</td>\n",
       "      <td>two</td>\n",
       "      <td>play</td>\n",
       "      <td>romantic</td>\n",
       "      <td>story</td>\n",
       "      <td>dr</td>\n",
       "      <td>counterpart</td>\n",
       "      <td>exchange</td>\n",
       "      <td>marry</td>\n",
       "      <td>brother</td>\n",
       "      <td>tale</td>\n",
       "      <td>famous</td>\n",
       "      <td>mysterious</td>\n",
       "      <td>think</td>\n",
       "      <td>hard</td>\n",
       "      <td>mother</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>young</td>\n",
       "      <td>husband</td>\n",
       "      <td>try</td>\n",
       "      <td>officer</td>\n",
       "      <td>beautiful</td>\n",
       "      <td>story</td>\n",
       "      <td>dangerous</td>\n",
       "      <td>help</td>\n",
       "      <td>boy</td>\n",
       "      <td>home</td>\n",
       "      <td>escape</td>\n",
       "      <td>relationship</td>\n",
       "      <td>war</td>\n",
       "      <td>girl</td>\n",
       "      <td>men</td>\n",
       "      <td>name</td>\n",
       "      <td>doctor</td>\n",
       "      <td>fantasy</td>\n",
       "      <td>begin</td>\n",
       "      <td>decide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>killer</td>\n",
       "      <td>serial</td>\n",
       "      <td>detective</td>\n",
       "      <td>law</td>\n",
       "      <td>female</td>\n",
       "      <td>victim</td>\n",
       "      <td>violence</td>\n",
       "      <td>person</td>\n",
       "      <td>gay</td>\n",
       "      <td>one</td>\n",
       "      <td>lead</td>\n",
       "      <td>two</td>\n",
       "      <td>duo</td>\n",
       "      <td>clue</td>\n",
       "      <td>personality</td>\n",
       "      <td>journalist</td>\n",
       "      <td>past</td>\n",
       "      <td>intelligent</td>\n",
       "      <td>effort</td>\n",
       "      <td>capture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>daughter</td>\n",
       "      <td>wife</td>\n",
       "      <td>face</td>\n",
       "      <td>leave</td>\n",
       "      <td>suffer</td>\n",
       "      <td>die</td>\n",
       "      <td>car</td>\n",
       "      <td>accident</td>\n",
       "      <td>second</td>\n",
       "      <td>poor</td>\n",
       "      <td>castle</td>\n",
       "      <td>rule</td>\n",
       "      <td>need</td>\n",
       "      <td>become</td>\n",
       "      <td>begin</td>\n",
       "      <td>two</td>\n",
       "      <td>look</td>\n",
       "      <td>experiment</td>\n",
       "      <td>professor</td>\n",
       "      <td>mute</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>team</td>\n",
       "      <td>stop</td>\n",
       "      <td>evil</td>\n",
       "      <td>try</td>\n",
       "      <td>former</td>\n",
       "      <td>cop</td>\n",
       "      <td>computer</td>\n",
       "      <td>agent</td>\n",
       "      <td>kgb</td>\n",
       "      <td>protect</td>\n",
       "      <td>steal</td>\n",
       "      <td>world</td>\n",
       "      <td>rescue</td>\n",
       "      <td>russian</td>\n",
       "      <td>plan</td>\n",
       "      <td>use</td>\n",
       "      <td>chip</td>\n",
       "      <td>new</td>\n",
       "      <td>nuclear</td>\n",
       "      <td>game</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>woman</td>\n",
       "      <td>town</td>\n",
       "      <td>come</td>\n",
       "      <td>pursue</td>\n",
       "      <td>small</td>\n",
       "      <td>notorious</td>\n",
       "      <td>beautiful</td>\n",
       "      <td>mysterious</td>\n",
       "      <td>work</td>\n",
       "      <td>comedy</td>\n",
       "      <td>along</td>\n",
       "      <td>successful</td>\n",
       "      <td>show</td>\n",
       "      <td>move</td>\n",
       "      <td>many</td>\n",
       "      <td>compete</td>\n",
       "      <td>dream</td>\n",
       "      <td>fortune</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>within</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>film</td>\n",
       "      <td>story</td>\n",
       "      <td>work</td>\n",
       "      <td>tell</td>\n",
       "      <td>base</td>\n",
       "      <td>red</td>\n",
       "      <td>comedy</td>\n",
       "      <td>set</td>\n",
       "      <td>boy</td>\n",
       "      <td>strange</td>\n",
       "      <td>world</td>\n",
       "      <td>bring</td>\n",
       "      <td>novel</td>\n",
       "      <td>movie</td>\n",
       "      <td>studio</td>\n",
       "      <td>village</td>\n",
       "      <td>adventure</td>\n",
       "      <td>romantic</td>\n",
       "      <td>mafia</td>\n",
       "      <td>company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>get</td>\n",
       "      <td>thing</td>\n",
       "      <td>friend</td>\n",
       "      <td>go</td>\n",
       "      <td>best</td>\n",
       "      <td>marry</td>\n",
       "      <td>tell</td>\n",
       "      <td>old</td>\n",
       "      <td>help</td>\n",
       "      <td>marriage</td>\n",
       "      <td>boyfriend</td>\n",
       "      <td>hire</td>\n",
       "      <td>cop</td>\n",
       "      <td>mother</td>\n",
       "      <td>business</td>\n",
       "      <td>try</td>\n",
       "      <td>brother</td>\n",
       "      <td>wife</td>\n",
       "      <td>kid</td>\n",
       "      <td>give</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>find</td>\n",
       "      <td>new</td>\n",
       "      <td>way</td>\n",
       "      <td>time</td>\n",
       "      <td>hit</td>\n",
       "      <td>job</td>\n",
       "      <td>secret</td>\n",
       "      <td>die</td>\n",
       "      <td>friend</td>\n",
       "      <td>make</td>\n",
       "      <td>mob</td>\n",
       "      <td>bring</td>\n",
       "      <td>bos</td>\n",
       "      <td>work</td>\n",
       "      <td>advertising</td>\n",
       "      <td>ancient</td>\n",
       "      <td>human</td>\n",
       "      <td>virus</td>\n",
       "      <td>miss</td>\n",
       "      <td>discover</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>take</td>\n",
       "      <td>one</td>\n",
       "      <td>meet</td>\n",
       "      <td>become</td>\n",
       "      <td>plan</td>\n",
       "      <td>job</td>\n",
       "      <td>decide</td>\n",
       "      <td>involve</td>\n",
       "      <td>night</td>\n",
       "      <td>son</td>\n",
       "      <td>large</td>\n",
       "      <td>case</td>\n",
       "      <td>new</td>\n",
       "      <td>someone</td>\n",
       "      <td>blackmail</td>\n",
       "      <td>power</td>\n",
       "      <td>kill</td>\n",
       "      <td>begin</td>\n",
       "      <td>money</td>\n",
       "      <td>everything</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>american</td>\n",
       "      <td>war</td>\n",
       "      <td>expatriate</td>\n",
       "      <td>paris</td>\n",
       "      <td>affair</td>\n",
       "      <td>world</td>\n",
       "      <td>ii</td>\n",
       "      <td>arrive</td>\n",
       "      <td>france</td>\n",
       "      <td>meet</td>\n",
       "      <td>control</td>\n",
       "      <td>fellow</td>\n",
       "      <td>fight</td>\n",
       "      <td>civil</td>\n",
       "      <td>struggle</td>\n",
       "      <td>writer</td>\n",
       "      <td>south</td>\n",
       "      <td>major</td>\n",
       "      <td>former</td>\n",
       "      <td>turbulent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>family</td>\n",
       "      <td>father</td>\n",
       "      <td>search</td>\n",
       "      <td>year</td>\n",
       "      <td>irish</td>\n",
       "      <td>son</td>\n",
       "      <td>neighbor</td>\n",
       "      <td>widow</td>\n",
       "      <td>marriage</td>\n",
       "      <td>farm</td>\n",
       "      <td>relationship</td>\n",
       "      <td>wound</td>\n",
       "      <td>face</td>\n",
       "      <td>local</td>\n",
       "      <td>adaptation</td>\n",
       "      <td>fear</td>\n",
       "      <td>captain</td>\n",
       "      <td>spend</td>\n",
       "      <td>fall</td>\n",
       "      <td>flee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>drug</td>\n",
       "      <td>want</td>\n",
       "      <td>name</td>\n",
       "      <td>operation</td>\n",
       "      <td>lord</td>\n",
       "      <td>local</td>\n",
       "      <td>sell</td>\n",
       "      <td>cia</td>\n",
       "      <td>undercover</td>\n",
       "      <td>agent</td>\n",
       "      <td>prison</td>\n",
       "      <td>dealer</td>\n",
       "      <td>work</td>\n",
       "      <td>help</td>\n",
       "      <td>police</td>\n",
       "      <td>eye</td>\n",
       "      <td>lead</td>\n",
       "      <td>embroil</td>\n",
       "      <td>stay</td>\n",
       "      <td>steal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>life</td>\n",
       "      <td>move</td>\n",
       "      <td>learn</td>\n",
       "      <td>housewife</td>\n",
       "      <td>friend</td>\n",
       "      <td>four</td>\n",
       "      <td>day</td>\n",
       "      <td>end</td>\n",
       "      <td>lesbian</td>\n",
       "      <td>year</td>\n",
       "      <td>best</td>\n",
       "      <td>apartment</td>\n",
       "      <td>make</td>\n",
       "      <td>invite</td>\n",
       "      <td>threaten</td>\n",
       "      <td>good</td>\n",
       "      <td>student</td>\n",
       "      <td>photographer</td>\n",
       "      <td>past</td>\n",
       "      <td>circle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>school</td>\n",
       "      <td>girl</td>\n",
       "      <td>high</td>\n",
       "      <td>parent</td>\n",
       "      <td>teacher</td>\n",
       "      <td>go</td>\n",
       "      <td>trip</td>\n",
       "      <td>society</td>\n",
       "      <td>come</td>\n",
       "      <td>city</td>\n",
       "      <td>teenage</td>\n",
       "      <td>age</td>\n",
       "      <td>gang</td>\n",
       "      <td>new</td>\n",
       "      <td>mrs</td>\n",
       "      <td>look</td>\n",
       "      <td>child</td>\n",
       "      <td>tough</td>\n",
       "      <td>summer</td>\n",
       "      <td>catholic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>criminal</td>\n",
       "      <td>police</td>\n",
       "      <td>leader</td>\n",
       "      <td>kill</td>\n",
       "      <td>run</td>\n",
       "      <td>know</td>\n",
       "      <td>group</td>\n",
       "      <td>old</td>\n",
       "      <td>money</td>\n",
       "      <td>kidnap</td>\n",
       "      <td>gang</td>\n",
       "      <td>pose</td>\n",
       "      <td>mountain</td>\n",
       "      <td>rich</td>\n",
       "      <td>cult</td>\n",
       "      <td>revenge</td>\n",
       "      <td>lead</td>\n",
       "      <td>use</td>\n",
       "      <td>force</td>\n",
       "      <td>violent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>murder</td>\n",
       "      <td>two</td>\n",
       "      <td>run</td>\n",
       "      <td>accuse</td>\n",
       "      <td>project</td>\n",
       "      <td>put</td>\n",
       "      <td>girlfriend</td>\n",
       "      <td>crime</td>\n",
       "      <td>boy</td>\n",
       "      <td>investigate</td>\n",
       "      <td>deal</td>\n",
       "      <td>rap</td>\n",
       "      <td>case</td>\n",
       "      <td>effect</td>\n",
       "      <td>witness</td>\n",
       "      <td>los</td>\n",
       "      <td>year</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>parent</td>\n",
       "      <td>contain</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0        1           2          3          4          5   \\\n",
       "0        man    demon    convince      story  discovers  political   \n",
       "1       love     fall      french    student       king        two   \n",
       "2      young  husband         try    officer  beautiful      story   \n",
       "3     killer   serial   detective        law     female     victim   \n",
       "4   daughter     wife        face      leave     suffer        die   \n",
       "5       team     stop        evil        try     former        cop   \n",
       "6      woman     town        come     pursue      small  notorious   \n",
       "7       film    story        work       tell       base        red   \n",
       "8        get    thing      friend         go       best      marry   \n",
       "9       find      new         way       time        hit        job   \n",
       "10      take      one        meet     become       plan        job   \n",
       "11  american      war  expatriate      paris     affair      world   \n",
       "12    family   father      search       year      irish        son   \n",
       "13      drug     want        name  operation       lord      local   \n",
       "14      life     move       learn  housewife     friend       four   \n",
       "15    school     girl        high     parent    teacher         go   \n",
       "16  criminal   police      leader       kill        run       know   \n",
       "17    murder      two         run     accuse    project        put   \n",
       "\n",
       "            6           7           8            9             10  \\\n",
       "0        guest      friend        true         like          look   \n",
       "1         play    romantic       story           dr   counterpart   \n",
       "2    dangerous        help         boy         home        escape   \n",
       "3     violence      person         gay          one          lead   \n",
       "4          car    accident      second         poor        castle   \n",
       "5     computer       agent         kgb      protect         steal   \n",
       "6    beautiful  mysterious        work       comedy         along   \n",
       "7       comedy         set         boy      strange         world   \n",
       "8         tell         old        help     marriage     boyfriend   \n",
       "9       secret         die      friend         make           mob   \n",
       "10      decide     involve       night          son         large   \n",
       "11          ii      arrive      france         meet       control   \n",
       "12    neighbor       widow    marriage         farm  relationship   \n",
       "13        sell         cia  undercover        agent        prison   \n",
       "14         day         end     lesbian         year          best   \n",
       "15        trip     society        come         city       teenage   \n",
       "16       group         old       money       kidnap          gang   \n",
       "17  girlfriend       crime         boy  investigate          deal   \n",
       "\n",
       "              11        12       13           14          15          16  \\\n",
       "0           know      fear  pretend       nature    confront     descend   \n",
       "1       exchange     marry  brother         tale      famous  mysterious   \n",
       "2   relationship       war     girl          men        name      doctor   \n",
       "3            two       duo     clue  personality  journalist        past   \n",
       "4           rule      need   become        begin         two        look   \n",
       "5          world    rescue  russian         plan         use        chip   \n",
       "6     successful      show     move         many     compete       dream   \n",
       "7          bring     novel    movie       studio     village   adventure   \n",
       "8           hire       cop   mother     business         try     brother   \n",
       "9          bring       bos     work  advertising     ancient       human   \n",
       "10          case       new  someone    blackmail       power        kill   \n",
       "11        fellow     fight    civil     struggle      writer       south   \n",
       "12         wound      face    local   adaptation        fear     captain   \n",
       "13        dealer      work     help       police         eye        lead   \n",
       "14     apartment      make   invite     threaten        good     student   \n",
       "15           age      gang      new          mrs        look       child   \n",
       "16          pose  mountain     rich         cult     revenge        lead   \n",
       "17           rap      case   effect      witness         los        year   \n",
       "\n",
       "              17           18          19  \n",
       "0         bright  personality        hunt  \n",
       "1          think         hard      mother  \n",
       "2        fantasy        begin      decide  \n",
       "3    intelligent       effort     capture  \n",
       "4     experiment    professor        mute  \n",
       "5            new      nuclear        game  \n",
       "6        fortune   unemployed      within  \n",
       "7       romantic        mafia     company  \n",
       "8           wife          kid        give  \n",
       "9          virus         miss    discover  \n",
       "10         begin        money  everything  \n",
       "11         major       former   turbulent  \n",
       "12         spend         fall        flee  \n",
       "13       embroil         stay       steal  \n",
       "14  photographer         past      circle  \n",
       "15         tough       summer    catholic  \n",
       "16           use        force     violent  \n",
       "17        lawyer       parent     contain  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_words = 20  # CHANGE THE NUMBER OF RECOMMENDED WORDS (IF YOU WANT TO)\n",
    "\n",
    "# Returns a matrix of recommended words\n",
    "topic_model, dtm, best_n = recommend_words(corpus) \n",
    "topic_model.show_topics(dtm=dtm, best_n=best_n, n_words=n_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explanation of the above matrix:\n",
    "* Each row relates to a \"theme\" identified by the model\n",
    "* Read row by row to identify themes which you want for the following step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5 - Create and feed your label dictionary\n",
    "Open your favourite tabular editor (excel/Numbers/libre) to create a labels csv file based on the suggested vocabulary shown below.\n",
    "\n",
    "The label dictionary should be in a tabular format with keys that relates to each label\n",
    "\n",
    "**Note: these labels needs to exist within the dataset**\n",
    "\n",
    "<img style=\"float: left;\" src=\"data/images/labels_sample.png\"> </img>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_path = \"data/movies/movies500_labels.csv\"  # INPUT PATH TO LABELS DICTIONARY\n",
    "labels = pd.read_csv(labels_path)\n",
    "labels = check_labels(data, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Action</th>\n",
       "      <th>Romance</th>\n",
       "      <th>Science Fiction</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>War</th>\n",
       "      <th>Western</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>criminal</td>\n",
       "      <td>love</td>\n",
       "      <td>science</td>\n",
       "      <td>killer</td>\n",
       "      <td>war</td>\n",
       "      <td>criminal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kill</td>\n",
       "      <td>french</td>\n",
       "      <td>NaN</td>\n",
       "      <td>detective</td>\n",
       "      <td>world</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kidnap</td>\n",
       "      <td>romantic</td>\n",
       "      <td>NaN</td>\n",
       "      <td>law</td>\n",
       "      <td>ii</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gang</td>\n",
       "      <td>marry</td>\n",
       "      <td>NaN</td>\n",
       "      <td>victim</td>\n",
       "      <td>struggle</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>revenge</td>\n",
       "      <td>marriage</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cop</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Action   Romance Science Fiction   Thriller       War   Western\n",
       "0  criminal      love         science     killer       war  criminal\n",
       "1      kill    french             NaN  detective     world       NaN\n",
       "2    kidnap  romantic             NaN        law        ii       NaN\n",
       "3      gang     marry             NaN     victim  struggle       NaN\n",
       "4   revenge  marriage             NaN        cop       NaN       NaN"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show label data\n",
    "labels.head(5)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6 - Enriching Labels\n",
    "In this step, the model enriches your current labels with other words close to the current topic. This enriched dictionary is used for topic labelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Action</th>\n",
       "      <th>Romance</th>\n",
       "      <th>Science Fiction</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>War</th>\n",
       "      <th>Western</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>married</td>\n",
       "      <td>notorious</td>\n",
       "      <td>case</td>\n",
       "      <td>world</td>\n",
       "      <td>world</td>\n",
       "      <td>case</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>marry</td>\n",
       "      <td>world</td>\n",
       "      <td>attend</td>\n",
       "      <td>law</td>\n",
       "      <td>singer</td>\n",
       "      <td>kill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>street</td>\n",
       "      <td>attend</td>\n",
       "      <td>havoc</td>\n",
       "      <td>nuclear</td>\n",
       "      <td>expatriate</td>\n",
       "      <td>law</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gang</td>\n",
       "      <td>marry</td>\n",
       "      <td>marry</td>\n",
       "      <td>kill</td>\n",
       "      <td>french</td>\n",
       "      <td>assign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>yearold</td>\n",
       "      <td>french</td>\n",
       "      <td>science</td>\n",
       "      <td>clean</td>\n",
       "      <td>slave</td>\n",
       "      <td>retired</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>french</td>\n",
       "      <td>science</td>\n",
       "      <td>comedy</td>\n",
       "      <td>civil</td>\n",
       "      <td>ii</td>\n",
       "      <td>killer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>violent</td>\n",
       "      <td>comedy</td>\n",
       "      <td>nuclear</td>\n",
       "      <td>renegade</td>\n",
       "      <td>disappearance</td>\n",
       "      <td>victim</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>kill</td>\n",
       "      <td>view</td>\n",
       "      <td>view</td>\n",
       "      <td>killer</td>\n",
       "      <td>holiday</td>\n",
       "      <td>next</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>clean</td>\n",
       "      <td>center</td>\n",
       "      <td>assign</td>\n",
       "      <td>russian</td>\n",
       "      <td>paris</td>\n",
       "      <td>detective</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>throw</td>\n",
       "      <td>bunch</td>\n",
       "      <td>side</td>\n",
       "      <td>terrorist</td>\n",
       "      <td>nazi</td>\n",
       "      <td>serial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>intense</td>\n",
       "      <td>modern</td>\n",
       "      <td>different</td>\n",
       "      <td>victim</td>\n",
       "      <td>bodyguard</td>\n",
       "      <td>cop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marriage</td>\n",
       "      <td>marriage</td>\n",
       "      <td>terrorist</td>\n",
       "      <td>detective</td>\n",
       "      <td>war</td>\n",
       "      <td>criminal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>love</td>\n",
       "      <td>nazi</td>\n",
       "      <td>personal</td>\n",
       "      <td>cop</td>\n",
       "      <td>jazz</td>\n",
       "      <td>innocence</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>resolve</td>\n",
       "      <td>serial</td>\n",
       "      <td>resolve</td>\n",
       "      <td>chip</td>\n",
       "      <td>refuse</td>\n",
       "      <td>murderer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>criminal</td>\n",
       "      <td>love</td>\n",
       "      <td>reason</td>\n",
       "      <td>operation</td>\n",
       "      <td>club</td>\n",
       "      <td>late</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>grows</td>\n",
       "      <td>war</td>\n",
       "      <td>intrigue</td>\n",
       "      <td>attorney</td>\n",
       "      <td>persuade</td>\n",
       "      <td>attorney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>operation</td>\n",
       "      <td>idea</td>\n",
       "      <td>america</td>\n",
       "      <td>agent</td>\n",
       "      <td>late</td>\n",
       "      <td>danger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>wife</td>\n",
       "      <td>intrigue</td>\n",
       "      <td>army</td>\n",
       "      <td>hostage</td>\n",
       "      <td>chase</td>\n",
       "      <td>team</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>kidnap</td>\n",
       "      <td>america</td>\n",
       "      <td>danger</td>\n",
       "      <td>gold</td>\n",
       "      <td>struggle</td>\n",
       "      <td>wife</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>fall</td>\n",
       "      <td>wife</td>\n",
       "      <td>wife</td>\n",
       "      <td>missile</td>\n",
       "      <td>fall</td>\n",
       "      <td>clue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>romantic</td>\n",
       "      <td>compete</td>\n",
       "      <td>challenge</td>\n",
       "      <td>team</td>\n",
       "      <td>bar</td>\n",
       "      <td>revenge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>revenge</td>\n",
       "      <td>challenge</td>\n",
       "      <td>None</td>\n",
       "      <td>sight</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>boyfriend</td>\n",
       "      <td>introduce</td>\n",
       "      <td>None</td>\n",
       "      <td>kgb</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>None</td>\n",
       "      <td>fall</td>\n",
       "      <td>None</td>\n",
       "      <td>design</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>None</td>\n",
       "      <td>romantic</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>None</td>\n",
       "      <td>boyfriend</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Action    Romance Science Fiction   Thriller            War    Western\n",
       "0     married  notorious            case      world          world       case\n",
       "1       marry      world          attend        law         singer       kill\n",
       "2      street     attend           havoc    nuclear     expatriate        law\n",
       "3        gang      marry           marry       kill         french     assign\n",
       "4     yearold     french         science      clean          slave    retired\n",
       "5      french    science          comedy      civil             ii     killer\n",
       "6     violent     comedy         nuclear   renegade  disappearance     victim\n",
       "7        kill       view            view     killer        holiday       next\n",
       "8       clean     center          assign    russian          paris  detective\n",
       "9       throw      bunch            side  terrorist           nazi     serial\n",
       "10    intense     modern       different     victim      bodyguard        cop\n",
       "11   marriage   marriage       terrorist  detective            war   criminal\n",
       "12       love       nazi        personal        cop           jazz  innocence\n",
       "13    resolve     serial         resolve       chip         refuse   murderer\n",
       "14   criminal       love          reason  operation           club       late\n",
       "15      grows        war        intrigue   attorney       persuade   attorney\n",
       "16  operation       idea         america      agent           late     danger\n",
       "17       wife   intrigue            army    hostage          chase       team\n",
       "18     kidnap    america          danger       gold       struggle       wife\n",
       "19       fall       wife            wife    missile           fall       clue\n",
       "20   romantic    compete       challenge       team            bar    revenge\n",
       "21    revenge  challenge            None      sight           None       None\n",
       "22  boyfriend  introduce            None        kgb           None       None\n",
       "23       None       fall            None     design           None       None\n",
       "24       None   romantic            None       None           None       None\n",
       "25       None  boyfriend            None       None           None       None"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autoLabeller = AutoLabeller(labels, corpus, data)\n",
    "enriched_labels = autoLabeller.train()\n",
    "\n",
    "enriched_labels  ## Enriched labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7 - Apply the auto labeller\n",
    "Input file path to output your labelled dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "labelled_path = \"data/movies/movies500_labelled.csv\"  # INPUT YOUR PREFERED OUTPUT PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnb = MultinomialNB()\n",
    "ypred = autoLabeller.apply(mnb, text_column_name)\n",
    "ypred.to_csv(labelled_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}