Stock_Price_Fluctuation_Sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    },
    "colab": {
      "name": "Stock_Price_Fluctuation_Sentiment_Analysis.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/StockPricePrediction/blob/master/Stock_Price_Fluctuation_Sentiment_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6so3i4-8lJz"
      },
      "source": [
        "**In this notebook, we would see how the stock price fluctuate with news headlines.**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "929mrQydIBMP"
      },
      "source": [
        "## Stock Sentiment Analysis using News Headlines"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_6x9RciAIBMW"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IzMbYJiPIBMg"
      },
      "source": [
        "df=pd.read_csv('/content/Data.csv', encoding = \"ISO-8859-1\")"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SIHNI4YIBMo",
        "outputId": "cdb503df-88ba-48bf-ec24-9c0523804e12",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>Label</th>\n",
              "      <th>Top1</th>\n",
              "      <th>Top2</th>\n",
              "      <th>Top3</th>\n",
              "      <th>Top4</th>\n",
              "      <th>Top5</th>\n",
              "      <th>Top6</th>\n",
              "      <th>Top7</th>\n",
              "      <th>Top8</th>\n",
              "      <th>Top9</th>\n",
              "      <th>Top10</th>\n",
              "      <th>Top11</th>\n",
              "      <th>Top12</th>\n",
              "      <th>Top13</th>\n",
              "      <th>Top14</th>\n",
              "      <th>Top15</th>\n",
              "      <th>Top16</th>\n",
              "      <th>Top17</th>\n",
              "      <th>Top18</th>\n",
              "      <th>Top19</th>\n",
              "      <th>Top20</th>\n",
              "      <th>Top21</th>\n",
              "      <th>Top22</th>\n",
              "      <th>Top23</th>\n",
              "      <th>Top24</th>\n",
              "      <th>Top25</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2000-01-03</td>\n",
              "      <td>0</td>\n",
              "      <td>A 'hindrance to operations': extracts from the...</td>\n",
              "      <td>Scorecard</td>\n",
              "      <td>Hughes' instant hit buoys Blues</td>\n",
              "      <td>Jack gets his skates on at ice-cold Alex</td>\n",
              "      <td>Chaos as Maracana builds up for United</td>\n",
              "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
              "      <td>Hungry Spurs sense rich pickings</td>\n",
              "      <td>Gunners so wide of an easy target</td>\n",
              "      <td>Derby raise a glass to Strupar's debut double</td>\n",
              "      <td>Southgate strikes, Leeds pay the penalty</td>\n",
              "      <td>Hammers hand Robson a youthful lesson</td>\n",
              "      <td>Saints party like it's 1999</td>\n",
              "      <td>Wear wolves have turned into lambs</td>\n",
              "      <td>Stump mike catches testy Gough's taunt</td>\n",
              "      <td>Langer escapes to hit 167</td>\n",
              "      <td>Flintoff injury piles on woe for England</td>\n",
              "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
              "      <td>Kohl's successor drawn into scandal</td>\n",
              "      <td>The difference between men and women</td>\n",
              "      <td>Sara Denver, nurse turned solicitor</td>\n",
              "      <td>Diana's landmine crusade put Tories in a panic</td>\n",
              "      <td>Yeltsin's resignation caught opposition flat-f...</td>\n",
              "      <td>Russian roulette</td>\n",
              "      <td>Sold out</td>\n",
              "      <td>Recovering a title</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2000-01-04</td>\n",
              "      <td>0</td>\n",
              "      <td>Scorecard</td>\n",
              "      <td>The best lake scene</td>\n",
              "      <td>Leader: German sleaze inquiry</td>\n",
              "      <td>Cheerio, boyo</td>\n",
              "      <td>The main recommendations</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>Hopkins 'furious' at Foster's lack of Hannibal...</td>\n",
              "      <td>Has Cubie killed fees?</td>\n",
              "      <td>A tale of two tails</td>\n",
              "      <td>I say what I like and I like what I say</td>\n",
              "      <td>Elbows, Eyes and Nipples</td>\n",
              "      <td>Task force to assess risk of asteroid collision</td>\n",
              "      <td>How I found myself at last</td>\n",
              "      <td>On the critical list</td>\n",
              "      <td>The timing of their lives</td>\n",
              "      <td>Dear doctor</td>\n",
              "      <td>Irish court halts IRA man's extradition to Nor...</td>\n",
              "      <td>Burundi peace initiative fades after rebels re...</td>\n",
              "      <td>PE points the way forward to the ECB</td>\n",
              "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
              "      <td>Jane Ratcliffe</td>\n",
              "      <td>Yet more things you wouldn't know without the ...</td>\n",
              "      <td>Millennium bug fails to bite</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2000-01-05</td>\n",
              "      <td>0</td>\n",
              "      <td>Coventry caught on counter by Flo</td>\n",
              "      <td>United's rivals on the road to Rio</td>\n",
              "      <td>Thatcher issues defence before trial by video</td>\n",
              "      <td>Police help Smith lay down the law at Everton</td>\n",
              "      <td>Tale of Trautmann bears two more retellings</td>\n",
              "      <td>England on the rack</td>\n",
              "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
              "      <td>Cullinan continues his Cape monopoly</td>\n",
              "      <td>McGrath puts India out of their misery</td>\n",
              "      <td>Blair Witch bandwagon rolls on</td>\n",
              "      <td>Pele turns up heat on Ferguson</td>\n",
              "      <td>Party divided over Kohl slush fund scandal</td>\n",
              "      <td>Manchester United (England)</td>\n",
              "      <td>Women in record South Pole walk</td>\n",
              "      <td>Vasco da Gama (Brazil)</td>\n",
              "      <td>South Melbourne (Australia)</td>\n",
              "      <td>Necaxa (Mexico)</td>\n",
              "      <td>Real Madrid (Spain)</td>\n",
              "      <td>Raja Casablanca (Morocco)</td>\n",
              "      <td>Corinthians (Brazil)</td>\n",
              "      <td>Tony's pet project</td>\n",
              "      <td>Al Nassr (Saudi Arabia)</td>\n",
              "      <td>Ideal Holmes show</td>\n",
              "      <td>Pinochet leaves hospital after tests</td>\n",
              "      <td>Useful links</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2000-01-06</td>\n",
              "      <td>1</td>\n",
              "      <td>Pilgrim knows how to progress</td>\n",
              "      <td>Thatcher facing ban</td>\n",
              "      <td>McIlroy calls for Irish fighting spirit</td>\n",
              "      <td>Leicester bin stadium blueprint</td>\n",
              "      <td>United braced for Mexican wave</td>\n",
              "      <td>Auntie back in fashion, even if the dress look...</td>\n",
              "      <td>Shoaib appeal goes to the top</td>\n",
              "      <td>Hussain hurt by 'shambles' but lays blame on e...</td>\n",
              "      <td>England's decade of disasters</td>\n",
              "      <td>Revenge is sweet for jubilant Cronje</td>\n",
              "      <td>Our choice, not theirs</td>\n",
              "      <td>Profile of former US Nazi Party officer Willia...</td>\n",
              "      <td>New evidence shows record of war crimes suspec...</td>\n",
              "      <td>The rise of the supernerds</td>\n",
              "      <td>Written on the body</td>\n",
              "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
              "      <td>BBC worst hit as digital TV begins to bite</td>\n",
              "      <td>How much can you pay for...</td>\n",
              "      <td>Christmas glitches</td>\n",
              "      <td>Upending a table, Chopping a line and Scoring ...</td>\n",
              "      <td>Scientific evidence 'unreliable', defence claims</td>\n",
              "      <td>Fusco wins judicial review in extradition case</td>\n",
              "      <td>Rebels thwart Russian advance</td>\n",
              "      <td>Blair orders shake-up of failing NHS</td>\n",
              "      <td>Lessons of law's hard heart</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2000-01-07</td>\n",
              "      <td>1</td>\n",
              "      <td>Hitches and Horlocks</td>\n",
              "      <td>Beckham off but United survive</td>\n",
              "      <td>Breast cancer screening</td>\n",
              "      <td>Alan Parker</td>\n",
              "      <td>Guardian readers: are you all whingers?</td>\n",
              "      <td>Hollywood Beyond</td>\n",
              "      <td>Ashes and diamonds</td>\n",
              "      <td>Whingers - a formidable minority</td>\n",
              "      <td>Alan Parker - part two</td>\n",
              "      <td>Thuggery, Toxins and Ties</td>\n",
              "      <td>Met faces fresh attack on race crime</td>\n",
              "      <td>Everton fans top racist 'league of shame'</td>\n",
              "      <td>Our breasts, ourselves</td>\n",
              "      <td>Russia's new boss has an extremely strange his...</td>\n",
              "      <td>Always and forever</td>\n",
              "      <td>Most everywhere:  UDIs</td>\n",
              "      <td>Most wanted:  Chloe lunettes</td>\n",
              "      <td>Return of the cane 'completely off the agenda'</td>\n",
              "      <td>From Sleepy Hollow to Greeneland</td>\n",
              "      <td>Blunkett outlines vision for over 11s</td>\n",
              "      <td>Embattled Dobson attacks 'play now, pay later'...</td>\n",
              "      <td>Doom and the Dome</td>\n",
              "      <td>What is the north-south divide?</td>\n",
              "      <td>Aitken released from jail</td>\n",
              "      <td>Gone aloft</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         Date  ...                         Top25\n",
              "0  2000-01-03  ...            Recovering a title\n",
              "1  2000-01-04  ...  Millennium bug fails to bite\n",
              "2  2000-01-05  ...                  Useful links\n",
              "3  2000-01-06  ...   Lessons of law's hard heart\n",
              "4  2000-01-07  ...                    Gone aloft\n",
              "\n",
              "[5 rows x 27 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MKPEzB614EBS"
      },
      "source": [
        "#Split the data set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o40WwXjhIBMx"
      },
      "source": [
        "train = df[df['Date'] < '20150101']\n",
        "test = df[df['Date'] > '20141231']"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99mHjyv44HxI"
      },
      "source": [
        "#Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XOZyMtABIBM4",
        "outputId": "7c789c28-6c0e-4681-e723-ee4de22e2a20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "# Removing punctuations\n",
        "data=train.iloc[:,2:27]\n",
        "data.replace(\"[^a-zA-Z]\",\" \",regex=True, inplace=True)\n",
        "\n",
        "# Renaming column names for ease of access\n",
        "list1= [i for i in range(25)]\n",
        "new_Index=[str(i) for i in list1]\n",
        "data.columns= new_Index\n",
        "data.head(5)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A  hindrance to operations   extracts from the...</td>\n",
              "      <td>Scorecard</td>\n",
              "      <td>Hughes  instant hit buoys Blues</td>\n",
              "      <td>Jack gets his skates on at ice cold Alex</td>\n",
              "      <td>Chaos as Maracana builds up for United</td>\n",
              "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
              "      <td>Hungry Spurs sense rich pickings</td>\n",
              "      <td>Gunners so wide of an easy target</td>\n",
              "      <td>Derby raise a glass to Strupar s debut double</td>\n",
              "      <td>Southgate strikes  Leeds pay the penalty</td>\n",
              "      <td>Hammers hand Robson a youthful lesson</td>\n",
              "      <td>Saints party like it s</td>\n",
              "      <td>Wear wolves have turned into lambs</td>\n",
              "      <td>Stump mike catches testy Gough s taunt</td>\n",
              "      <td>Langer escapes to hit</td>\n",
              "      <td>Flintoff injury piles on woe for England</td>\n",
              "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
              "      <td>Kohl s successor drawn into scandal</td>\n",
              "      <td>The difference between men and women</td>\n",
              "      <td>Sara Denver  nurse turned solicitor</td>\n",
              "      <td>Diana s landmine crusade put Tories in a panic</td>\n",
              "      <td>Yeltsin s resignation caught opposition flat f...</td>\n",
              "      <td>Russian roulette</td>\n",
              "      <td>Sold out</td>\n",
              "      <td>Recovering a title</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Scorecard</td>\n",
              "      <td>The best lake scene</td>\n",
              "      <td>Leader  German sleaze inquiry</td>\n",
              "      <td>Cheerio  boyo</td>\n",
              "      <td>The main recommendations</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>Hopkins  furious  at Foster s lack of Hannibal...</td>\n",
              "      <td>Has Cubie killed fees</td>\n",
              "      <td>A tale of two tails</td>\n",
              "      <td>I say what I like and I like what I say</td>\n",
              "      <td>Elbows  Eyes and Nipples</td>\n",
              "      <td>Task force to assess risk of asteroid collision</td>\n",
              "      <td>How I found myself at last</td>\n",
              "      <td>On the critical list</td>\n",
              "      <td>The timing of their lives</td>\n",
              "      <td>Dear doctor</td>\n",
              "      <td>Irish court halts IRA man s extradition to Nor...</td>\n",
              "      <td>Burundi peace initiative fades after rebels re...</td>\n",
              "      <td>PE points the way forward to the ECB</td>\n",
              "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
              "      <td>Jane Ratcliffe</td>\n",
              "      <td>Yet more things you wouldn t know without the ...</td>\n",
              "      <td>Millennium bug fails to bite</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Coventry caught on counter by Flo</td>\n",
              "      <td>United s rivals on the road to Rio</td>\n",
              "      <td>Thatcher issues defence before trial by video</td>\n",
              "      <td>Police help Smith lay down the law at Everton</td>\n",
              "      <td>Tale of Trautmann bears two more retellings</td>\n",
              "      <td>England on the rack</td>\n",
              "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
              "      <td>Cullinan continues his Cape monopoly</td>\n",
              "      <td>McGrath puts India out of their misery</td>\n",
              "      <td>Blair Witch bandwagon rolls on</td>\n",
              "      <td>Pele turns up heat on Ferguson</td>\n",
              "      <td>Party divided over Kohl slush fund scandal</td>\n",
              "      <td>Manchester United  England</td>\n",
              "      <td>Women in record South Pole walk</td>\n",
              "      <td>Vasco da Gama  Brazil</td>\n",
              "      <td>South Melbourne  Australia</td>\n",
              "      <td>Necaxa  Mexico</td>\n",
              "      <td>Real Madrid  Spain</td>\n",
              "      <td>Raja Casablanca  Morocco</td>\n",
              "      <td>Corinthians  Brazil</td>\n",
              "      <td>Tony s pet project</td>\n",
              "      <td>Al Nassr  Saudi Arabia</td>\n",
              "      <td>Ideal Holmes show</td>\n",
              "      <td>Pinochet leaves hospital after tests</td>\n",
              "      <td>Useful links</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Pilgrim knows how to progress</td>\n",
              "      <td>Thatcher facing ban</td>\n",
              "      <td>McIlroy calls for Irish fighting spirit</td>\n",
              "      <td>Leicester bin stadium blueprint</td>\n",
              "      <td>United braced for Mexican wave</td>\n",
              "      <td>Auntie back in fashion  even if the dress look...</td>\n",
              "      <td>Shoaib appeal goes to the top</td>\n",
              "      <td>Hussain hurt by  shambles  but lays blame on e...</td>\n",
              "      <td>England s decade of disasters</td>\n",
              "      <td>Revenge is sweet for jubilant Cronje</td>\n",
              "      <td>Our choice  not theirs</td>\n",
              "      <td>Profile of former US Nazi Party officer Willia...</td>\n",
              "      <td>New evidence shows record of war crimes suspec...</td>\n",
              "      <td>The rise of the supernerds</td>\n",
              "      <td>Written on the body</td>\n",
              "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
              "      <td>BBC worst hit as digital TV begins to bite</td>\n",
              "      <td>How much can you pay for</td>\n",
              "      <td>Christmas glitches</td>\n",
              "      <td>Upending a table  Chopping a line and Scoring ...</td>\n",
              "      <td>Scientific evidence  unreliable   defence claims</td>\n",
              "      <td>Fusco wins judicial review in extradition case</td>\n",
              "      <td>Rebels thwart Russian advance</td>\n",
              "      <td>Blair orders shake up of failing NHS</td>\n",
              "      <td>Lessons of law s hard heart</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Hitches and Horlocks</td>\n",
              "      <td>Beckham off but United survive</td>\n",
              "      <td>Breast cancer screening</td>\n",
              "      <td>Alan Parker</td>\n",
              "      <td>Guardian readers  are you all whingers</td>\n",
              "      <td>Hollywood Beyond</td>\n",
              "      <td>Ashes and diamonds</td>\n",
              "      <td>Whingers   a formidable minority</td>\n",
              "      <td>Alan Parker   part two</td>\n",
              "      <td>Thuggery  Toxins and Ties</td>\n",
              "      <td>Met faces fresh attack on race crime</td>\n",
              "      <td>Everton fans top racist  league of shame</td>\n",
              "      <td>Our breasts  ourselves</td>\n",
              "      <td>Russia s new boss has an extremely strange his...</td>\n",
              "      <td>Always and forever</td>\n",
              "      <td>Most everywhere   UDIs</td>\n",
              "      <td>Most wanted   Chloe lunettes</td>\n",
              "      <td>Return of the cane  completely off the agenda</td>\n",
              "      <td>From Sleepy Hollow to Greeneland</td>\n",
              "      <td>Blunkett outlines vision for over   s</td>\n",
              "      <td>Embattled Dobson attacks  play now  pay later ...</td>\n",
              "      <td>Doom and the Dome</td>\n",
              "      <td>What is the north south divide</td>\n",
              "      <td>Aitken released from jail</td>\n",
              "      <td>Gone aloft</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0  ...                            24\n",
              "0  A  hindrance to operations   extracts from the...  ...            Recovering a title\n",
              "1                                          Scorecard  ...  Millennium bug fails to bite\n",
              "2                  Coventry caught on counter by Flo  ...                  Useful links\n",
              "3                      Pilgrim knows how to progress  ...   Lessons of law s hard heart\n",
              "4                               Hitches and Horlocks  ...                    Gone aloft\n",
              "\n",
              "[5 rows x 25 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aSFLggjSIBM_",
        "outputId": "916d6dbb-3c0a-47ff-bab1-76aee196d987",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        }
      },
      "source": [
        "# Convertng headlines to lower case\n",
        "for index in new_Index:\n",
        "    data[index]=data[index].str.lower()\n",
        "data.head(1)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a  hindrance to operations   extracts from the...</td>\n",
              "      <td>scorecard</td>\n",
              "      <td>hughes  instant hit buoys blues</td>\n",
              "      <td>jack gets his skates on at ice cold alex</td>\n",
              "      <td>chaos as maracana builds up for united</td>\n",
              "      <td>depleted leicester prevail as elliott spoils e...</td>\n",
              "      <td>hungry spurs sense rich pickings</td>\n",
              "      <td>gunners so wide of an easy target</td>\n",
              "      <td>derby raise a glass to strupar s debut double</td>\n",
              "      <td>southgate strikes  leeds pay the penalty</td>\n",
              "      <td>hammers hand robson a youthful lesson</td>\n",
              "      <td>saints party like it s</td>\n",
              "      <td>wear wolves have turned into lambs</td>\n",
              "      <td>stump mike catches testy gough s taunt</td>\n",
              "      <td>langer escapes to hit</td>\n",
              "      <td>flintoff injury piles on woe for england</td>\n",
              "      <td>hunters threaten jospin with new battle of the...</td>\n",
              "      <td>kohl s successor drawn into scandal</td>\n",
              "      <td>the difference between men and women</td>\n",
              "      <td>sara denver  nurse turned solicitor</td>\n",
              "      <td>diana s landmine crusade put tories in a panic</td>\n",
              "      <td>yeltsin s resignation caught opposition flat f...</td>\n",
              "      <td>russian roulette</td>\n",
              "      <td>sold out</td>\n",
              "      <td>recovering a title</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0  ...                  24\n",
              "0  a  hindrance to operations   extracts from the...  ...  recovering a title\n",
              "\n",
              "[1 rows x 25 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "63PObsPQIBNG",
        "outputId": "fa0792d7-a2d5-42d3-cb91-ca3b315e1d83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        }
      },
      "source": [
        "' '.join(str(x) for x in data.iloc[1,0:25])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'scorecard the best lake scene leader  german sleaze inquiry cheerio  boyo the main recommendations has cubie killed fees  has cubie killed fees  has cubie killed fees  hopkins  furious  at foster s lack of hannibal appetite has cubie killed fees  a tale of two tails i say what i like and i like what i say elbows  eyes and nipples task force to assess risk of asteroid collision how i found myself at last on the critical list the timing of their lives dear doctor irish court halts ira man s extradition to northern ireland burundi peace initiative fades after rebels reject mandela as mediator pe points the way forward to the ecb campaigners keep up pressure on nazi war crimes suspect jane ratcliffe yet more things you wouldn t know without the movies millennium bug fails to bite'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wAogOXDD5Jai",
        "outputId": "f315ec71-bd3c-476b-af03-dcc95fbc9f19",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#len(data.index)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3975"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uct3mk0HIBNV"
      },
      "source": [
        "headlines = []\n",
        "for row in range(0,len(data.index)):\n",
        "    headlines.append(' '.join(str(x) for x in data.iloc[row,0:25]))"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A9Y26Ua3IBNd",
        "outputId": "269d3b9e-e764-4d12-c81d-8f4fb41cfee2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 137
        }
      },
      "source": [
        "headlines[0]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'a  hindrance to operations   extracts from the leaked reports scorecard hughes  instant hit buoys blues jack gets his skates on at ice cold alex chaos as maracana builds up for united depleted leicester prevail as elliott spoils everton s party hungry spurs sense rich pickings gunners so wide of an easy target derby raise a glass to strupar s debut double southgate strikes  leeds pay the penalty hammers hand robson a youthful lesson saints party like it s      wear wolves have turned into lambs stump mike catches testy gough s taunt langer escapes to hit     flintoff injury piles on woe for england hunters threaten jospin with new battle of the somme kohl s successor drawn into scandal the difference between men and women sara denver  nurse turned solicitor diana s landmine crusade put tories in a panic yeltsin s resignation caught opposition flat footed russian roulette sold out recovering a title'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wLo5DLSIIBNk"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.ensemble import RandomForestClassifier"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5XoFGs9x5-3a"
      },
      "source": [
        "#Implement BAG OF WORDS\n",
        "\n",
        "For more details on Countvectorizer https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\n",
        "\n",
        "***ngram_rangetuple (min_n, max_n), default=(1, 1) :***\n",
        "The lower and upper boundary of the range of n-values for different word n-grams or char n-grams to be extracted. All values of n such such that min_n <= n <= max_n will be used. For example an ngram_range of (1, 1) means only unigrams, (1, 2) means unigrams and bigrams, and (2, 2) means only bigrams. Only applies if analyzer is not callable."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e4YofpA_IBNp"
      },
      "source": [
        "## implement BAG OF WORDS\n",
        "countvector=CountVectorizer(ngram_range=(2,2))\n",
        "traindataset=countvector.fit_transform(headlines)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FG8ueyQ97Izk",
        "outputId": "904ef789-0f96-4e1f-f039-2aedfc1d3e92",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(traindataset[0:10])"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  (0, 232447)\t1\n",
            "  (0, 521422)\t1\n",
            "  (0, 366553)\t1\n",
            "  (0, 171227)\t1\n",
            "  (0, 200339)\t1\n",
            "  (0, 505997)\t1\n",
            "  (0, 291575)\t1\n",
            "  (0, 425161)\t1\n",
            "  (0, 447703)\t1\n",
            "  (0, 240194)\t1\n",
            "  (0, 255230)\t1\n",
            "  (0, 234167)\t1\n",
            "  (0, 77743)\t1\n",
            "  (0, 66371)\t1\n",
            "  (0, 271121)\t1\n",
            "  (0, 208233)\t1\n",
            "  (0, 233562)\t1\n",
            "  (0, 463800)\t1\n",
            "  (0, 361122)\t1\n",
            "  (0, 40926)\t1\n",
            "  (0, 241959)\t1\n",
            "  (0, 107274)\t1\n",
            "  (0, 14037)\t1\n",
            "  (0, 94946)\t1\n",
            "  (0, 37228)\t1\n",
            "  :\t:\n",
            "  (9, 559187)\t1\n",
            "  (9, 445998)\t1\n",
            "  (9, 250762)\t1\n",
            "  (9, 255448)\t1\n",
            "  (9, 23533)\t1\n",
            "  (9, 254174)\t1\n",
            "  (9, 181391)\t1\n",
            "  (9, 509059)\t1\n",
            "  (9, 517171)\t1\n",
            "  (9, 161364)\t1\n",
            "  (9, 492373)\t1\n",
            "  (9, 463615)\t1\n",
            "  (9, 363289)\t1\n",
            "  (9, 413132)\t1\n",
            "  (9, 459084)\t1\n",
            "  (9, 183194)\t1\n",
            "  (9, 129979)\t1\n",
            "  (9, 315287)\t1\n",
            "  (9, 441233)\t1\n",
            "  (9, 338407)\t1\n",
            "  (9, 100517)\t1\n",
            "  (9, 166438)\t1\n",
            "  (9, 144904)\t1\n",
            "  (9, 447555)\t1\n",
            "  (9, 40602)\t1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IaN3eOZY7qYO"
      },
      "source": [
        "#Implement RandomForest Classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "msJNGqKKIBNv",
        "outputId": "7d9cbf4f-f499-4155-a80f-c829727b408e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# implement RandomForest Classifier\n",
        "randomclassifier=RandomForestClassifier(n_estimators=200,criterion='entropy')\n",
        "randomclassifier.fit(traindataset,train['Label'])"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='entropy', max_depth=None, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=200,\n",
              "                       n_jobs=None, oob_score=False, random_state=None,\n",
              "                       verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0BeuPmZt71Oz"
      },
      "source": [
        "#Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXnFeJf0IBN1"
      },
      "source": [
        "## Predict for the Test Dataset\n",
        "test_transform= []\n",
        "for row in range(0,len(test.index)):\n",
        "    test_transform.append(' '.join(str(x) for x in test.iloc[row,2:27]))\n",
        "test_dataset = countvector.transform(test_transform)\n",
        "predictions = randomclassifier.predict(test_dataset)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xJZUoYVe8B3s"
      },
      "source": [
        "#Accuracy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FOobOzCDIBN6"
      },
      "source": [
        "## Import library to check accuracy\n",
        "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wZCC7CSCIBOB",
        "outputId": "af3b1b48-adea-4434-d335-6e9996e727ac",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "matrix=confusion_matrix(test['Label'],predictions)\n",
        "print(matrix)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[137  49]\n",
            " [  8 184]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PiB6G_Ox8H-M",
        "outputId": "6788f589-7078-4e22-92f2-418f5968735d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "score=accuracy_score(test['Label'],predictions)\n",
        "print(score)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8492063492063492\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eksieqnk8KJ9",
        "outputId": "10f6210d-f1a5-4566-fa69-4916804d157a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "report=classification_report(test['Label'],predictions)\n",
        "print(report)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.74      0.83       186\n",
            "           1       0.79      0.96      0.87       192\n",
            "\n",
            "    accuracy                           0.85       378\n",
            "   macro avg       0.87      0.85      0.85       378\n",
            "weighted avg       0.87      0.85      0.85       378\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}