Test Dataset Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Dataset Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "import pandas as pd\n",
    "import requests\n",
    "import scrapy\n",
    "from scrapy import Selector\n",
    "from scrapy.crawler import CrawlerProcess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('date_dict.pickle', 'rb') as handle:\n",
    "    date_dict = pickle.load(handle)\n",
    "with open('place_dict.pickle', 'rb') as handle:\n",
    "    place_dict = pickle.load(handle)\n",
    "with open('url_dict.pickle', 'rb') as handle:\n",
    "    url_dict = pickle.load(handle)    \n",
    "with open('text_dict.pickle', 'rb') as handle:\n",
    "    text_dict = pickle.load(handle)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_url = 'https://archivepmo.nic.in/drmanmohansingh/all-speeches.php'\n",
    "html = requests.get(source_url).content\n",
    "sel = Selector( text = html )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21,\n",
       " ['https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=734',\n",
       "  'https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=922',\n",
       "  'https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1016',\n",
       "  'https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=664',\n",
       "  'https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=873'])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Collecting urls which couldn't be scraped\n",
    "url_prefix = 'https://archivepmo.nic.in/drmanmohansingh/'\n",
    "url_suffixes = sel.xpath('//div[@class = \"speechPan\"]/ul//li').xpath('./a/@href').extract()\n",
    "urls = [url_prefix +url_suffix for url_suffix in url_suffixes]\n",
    "\n",
    "missed_urls = list(set(urls) - set(url_dict.values()))\n",
    "len(missed_urls),missed_urls[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Something went wrong. Removing the url\n",
      "Something went wrong. Removing the url\n",
      "Something went wrong. Removing the url\n"
     ]
    }
   ],
   "source": [
    "# Checking if speeches can be scraped, and if scraped are they of sufficent length. if not urls dropped \n",
    "for url_speech in missed_urls:\n",
    "    html_speech = requests.get( url_speech ).content\n",
    "    sel = Selector(text = html_speech)\n",
    "    \n",
    "    try:\n",
    "        ((sel.xpath('//*[@class = \"innerHead\"]/text()')).extract_first())\n",
    "        ((sel.xpath('//div[@class = \"contentInner\"]//h2[@class = \"date\"]/text()'))[0].extract())\n",
    "        ((sel.xpath('//div[@class = \"contentInner\"]//h2[@class = \"date\"]/text()'))[1].extract())\n",
    "        \n",
    "        text = \"\".join((sel.css('div.contentInner div.rt')).css('p::text').extract())\n",
    "        if len(text)<500:\n",
    "            missed_urls.remove(url_speech)\n",
    "            print(\"Something went wrong. Removing the url\")\n",
    "    except:\n",
    "        print(\"Something went wrong. Removing the url\")\n",
    "        missed_urls.remove(url_speech)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(missed_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating another spider to collect the missed speeches so that they can be made into a test set\n",
    "class MMS_Missed_Speech_Spider( scrapy.Spider ):\n",
    "    name = 'mms_missed_speeches_spider'\n",
    "    def start_requests( self ):\n",
    "        for url in missed_urls:\n",
    "            yield scrapy.Request( url = url, callback = self.parse )\n",
    "    def parse( self, response ):\n",
    "        #Extracting url\n",
    "        speech_url = response.url\n",
    "        \n",
    "        #Extracting title of speech\n",
    "        title = (response.xpath('//*[@class = \"innerHead\"]/text()')).extract_first()\n",
    "        \n",
    "        #Extracting date of speech\n",
    "        date = (response.xpath('//div[@class = \"contentInner\"]//h2[@class = \"date\"]/text()'))[0].extract()\n",
    "        \n",
    "        #Extracting place of speech\n",
    "        place = (response.xpath('//div[@class = \"contentInner\"]//h2[@class = \"date\"]/text()'))[1].extract()\n",
    "        \n",
    "        #Extracting speech text\n",
    "        text  = \"\".join((response.css('div.contentInner div.rt')).css('p::text').extract())\n",
    "        \n",
    "        #Storing in respective dict_testionaries\n",
    "        place_dict_test[title] = place\n",
    "        date_dict_test[title] = date\n",
    "        url_dict_test[title] = speech_url\n",
    "        text_dict_test[title] = text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-04-09 12:17:13 [scrapy.utils.log] INFO: Scrapy 1.8.0 started (bot: scrapybot)\n",
      "2020-04-09 12:17:13 [scrapy.utils.log] INFO: Versions: lxml 4.2.1.0, libxml2 2.9.8, cssselect 1.1.0, parsel 1.5.2, w3lib 1.21.0, Twisted 19.10.0, Python 3.6.5 |Anaconda custom (64-bit)| (default, Mar 29 2018, 13:32:41) [MSC v.1900 64 bit (AMD64)], pyOpenSSL 17.5.0 (OpenSSL 1.1.1d  10 Sep 2019), cryptography 2.4.2, Platform Windows-10-10.0.18362-SP0\n",
      "2020-04-09 12:17:13 [scrapy.crawler] INFO: Overridden settings: {}\n",
      "2020-04-09 12:17:13 [scrapy.extensions.telnet] INFO: Telnet Password: b26f59253623eb65\n",
      "2020-04-09 12:17:14 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.logstats.LogStats']\n",
      "2020-04-09 12:17:16 [scrapy.middleware] INFO: Enabled downloader middlewares:\n",
      "['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',\n",
      " 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',\n",
      " 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',\n",
      " 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',\n",
      " 'scrapy.downloadermiddlewares.retry.RetryMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',\n",
      " 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',\n",
      " 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',\n",
      " 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',\n",
      " 'scrapy.downloadermiddlewares.stats.DownloaderStats']\n",
      "2020-04-09 12:17:16 [scrapy.middleware] INFO: Enabled spider middlewares:\n",
      "['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',\n",
      " 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',\n",
      " 'scrapy.spidermiddlewares.referer.RefererMiddleware',\n",
      " 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',\n",
      " 'scrapy.spidermiddlewares.depth.DepthMiddleware']\n",
      "2020-04-09 12:17:16 [scrapy.middleware] INFO: Enabled item pipelines:\n",
      "[]\n",
      "2020-04-09 12:17:16 [scrapy.core.engine] INFO: Spider opened\n",
      "2020-04-09 12:17:16 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)\n",
      "2020-04-09 12:17:16 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023\n",
      "2020-04-09 12:17:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=608> (referer: None)\n",
      "2020-04-09 12:17:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=926> (referer: None)\n",
      "2020-04-09 12:17:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=734> (referer: None)\n",
      "2020-04-09 12:17:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=664> (referer: None)\n",
      "2020-04-09 12:17:17 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=873> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1016> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1002> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=517> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=339> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=922> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1146> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=994> (referer: None)\n",
      "2020-04-09 12:17:18 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=585> (referer: None)\n",
      "2020-04-09 12:17:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=542> (referer: None)\n",
      "2020-04-09 12:17:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=637> (referer: None)\n",
      "2020-04-09 12:17:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=305> (referer: None)\n",
      "2020-04-09 12:17:19 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=846> (referer: None)\n",
      "2020-04-09 12:17:20 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://archivepmo.nic.in/drmanmohansingh/speech-details.php?nodeid=1445> (referer: None)\n",
      "2020-04-09 12:17:20 [scrapy.core.engine] INFO: Closing spider (finished)\n",
      "2020-04-09 12:17:20 [scrapy.statscollectors] INFO: Dumping Scrapy stats:\n",
      "{'downloader/request_bytes': 4702,\n",
      " 'downloader/request_count': 18,\n",
      " 'downloader/request_method_count/GET': 18,\n",
      " 'downloader/response_bytes': 118394,\n",
      " 'downloader/response_count': 18,\n",
      " 'downloader/response_status_count/200': 18,\n",
      " 'elapsed_time_seconds': 3.930545,\n",
      " 'finish_reason': 'finished',\n",
      " 'finish_time': datetime.datetime(2020, 4, 9, 6, 47, 20, 420485),\n",
      " 'log_count/DEBUG': 18,\n",
      " 'log_count/INFO': 10,\n",
      " 'response_received_count': 18,\n",
      " 'scheduler/dequeued': 18,\n",
      " 'scheduler/dequeued/memory': 18,\n",
      " 'scheduler/enqueued': 18,\n",
      " 'scheduler/enqueued/memory': 18,\n",
      " 'start_time': datetime.datetime(2020, 4, 9, 6, 47, 16, 489940)}\n",
      "2020-04-09 12:17:20 [scrapy.core.engine] INFO: Spider closed (finished)\n"
     ]
    }
   ],
   "source": [
    "date_dict_test = {}\n",
    "place_dict_test = {}\n",
    "text_dict_test = {}\n",
    "url_dict_test = {}\n",
    "process = CrawlerProcess()\n",
    "process.crawl(MMS_Missed_Speech_Spider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13, 13, 13, 13)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(date_dict_test),len(place_dict_test),len(text_dict_test),len(url_dict_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "      <th>place</th>\n",
       "      <th>url</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PM's address to IPS Probationers</td>\n",
       "      <td>January 5, 2010</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/spee...</td>\n",
       "      <td>From your introductions, I am very encouraged ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Excerpts of address by the PM at the Combined ...</td>\n",
       "      <td>September 13, 2010</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/spee...</td>\n",
       "      <td>The Nation is proud of the selfless devotion t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PM's opening remarks at the All Party Meeting</td>\n",
       "      <td>November 30, 2008</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/spee...</td>\n",
       "      <td>\"Esteemed Chairperson UPA, respected colleague...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PM's address to the Nation</td>\n",
       "      <td>May 17, 2014</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/spee...</td>\n",
       "      <td>\\r\\n\\tMy Fellow Citizens,\\r\\n\\tI address you t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PM's opening remarks at the Full Planning Comm...</td>\n",
       "      <td>April 21, 2011</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>https://archivepmo.nic.in/drmanmohansingh/spee...</td>\n",
       "      <td>\"This meeting of the Planning Commission has b...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title                date  \\\n",
       "0                   PM's address to IPS Probationers     January 5, 2010   \n",
       "1  Excerpts of address by the PM at the Combined ...  September 13, 2010   \n",
       "2      PM's opening remarks at the All Party Meeting   November 30, 2008   \n",
       "3                         PM's address to the Nation        May 17, 2014   \n",
       "4  PM's opening remarks at the Full Planning Comm...      April 21, 2011   \n",
       "\n",
       "       place                                                url  \\\n",
       "0  New Delhi  https://archivepmo.nic.in/drmanmohansingh/spee...   \n",
       "1  New Delhi  https://archivepmo.nic.in/drmanmohansingh/spee...   \n",
       "2  New Delhi  https://archivepmo.nic.in/drmanmohansingh/spee...   \n",
       "3  New Delhi  https://archivepmo.nic.in/drmanmohansingh/spee...   \n",
       "4  New Delhi  https://archivepmo.nic.in/drmanmohansingh/spee...   \n",
       "\n",
       "                                                text  \n",
       "0  From your introductions, I am very encouraged ...  \n",
       "1  The Nation is proud of the selfless devotion t...  \n",
       "2  \"Esteemed Chairperson UPA, respected colleague...  \n",
       "3  \\r\\n\\tMy Fellow Citizens,\\r\\n\\tI address you t...  \n",
       "4  \"This meeting of the Planning Commission has b...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Creating test dataset\n",
    "date_df = pd.DataFrame.from_dict(date_dict_test, orient='index',columns= ['date'])\n",
    "place_df = pd.DataFrame.from_dict(place_dict_test, orient='index', columns= ['place'])\n",
    "text_df = pd.DataFrame.from_dict(text_dict_test, orient='index', columns= ['text'])\n",
    "url_df = pd.DataFrame.from_dict(url_dict_test, orient='index', columns= ['url'])\n",
    "\n",
    "df_combined_test = pd.concat([date_df, place_df,url_df,text_df], axis=1, sort=False, join = 'inner')\n",
    "df_combined_test.index.rename('title',inplace=True)\n",
    "df_combined_test.reset_index(drop = False,inplace= True)\n",
    "\n",
    "df_combined_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Storing test dataset into a file\n",
    "df_combined_test.to_excel('PM_MMS_Speech_test.xlsx',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}