Initial_Model_Run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np \n",
    "import re\n",
    "% matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\gensim\\utils.py:865: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Phrases\n",
    "from gensim.models.phrases import Phraser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Update puncuation list in spacy\n",
    "nlp.vocab[\"$\"].is_punct = True\n",
    "nlp.vocab[\"|\"].is_punct = True\n",
    "nlp.vocab[\"+\"].is_punct = True\n",
    "nlp.vocab[\"<\"].is_punct = True\n",
    "nlp.vocab[\">\"].is_punct = True\n",
    "nlp.vocab[\"=\"].is_punct = True\n",
    "nlp.vocab[\"^\"].is_punct = True\n",
    "nlp.vocab[\"`\"].is_punct = True\n",
    "nlp.vocab[\"~\"].is_punct = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clean_final = pd.read_csv(\"~/Documents/Capstone/code/clean_final_news.csv\", encoding = \"utf8\", index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clean_final[\"binary_response\"] = np.where(clean_final[\"response\"] == \"Not fake\", 0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>published</th>\n",
       "      <th>text</th>\n",
       "      <th>domain_rank</th>\n",
       "      <th>site_url</th>\n",
       "      <th>spam_score</th>\n",
       "      <th>title</th>\n",
       "      <th>response</th>\n",
       "      <th>length</th>\n",
       "      <th>binary_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAM TANENHAUS</td>\n",
       "      <td>2016-11-20T03:51:00.000+02:00</td>\n",
       "      <td>Privacy Policy Eisenhowers two terms bore this...</td>\n",
       "      <td>98.0</td>\n",
       "      <td>nytimes.com</td>\n",
       "      <td>0.000</td>\n",
       "      <td>Opinion: Donald Trump’s Art of the New Deal?</td>\n",
       "      <td>Not fake</td>\n",
       "      <td>949</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-11-20T00:52:00.000+02:00</td>\n",
       "      <td>Can Trump Save Their Jobs?  by Nelson D. Sc...</td>\n",
       "      <td>98.0</td>\n",
       "      <td>nytimes.com</td>\n",
       "      <td>0.000</td>\n",
       "      <td>Carrier Workers for Trump</td>\n",
       "      <td>Not fake</td>\n",
       "      <td>372</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kathleen Elkins</td>\n",
       "      <td>2016-11-20T06:18:00.000+02:00</td>\n",
       "      <td>Tuesday, 18 Oct 2016 | 10:25 AM ET CNBC.com Ed...</td>\n",
       "      <td>767.0</td>\n",
       "      <td>cnbc.com</td>\n",
       "      <td>0.008</td>\n",
       "      <td>Tennis star Caroline Wozniacki shares the mone...</td>\n",
       "      <td>Not fake</td>\n",
       "      <td>386</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JEFF SOMMER</td>\n",
       "      <td>2016-11-20T00:04:00.000+02:00</td>\n",
       "      <td>Continue reading the main story Yet it is poss...</td>\n",
       "      <td>98.0</td>\n",
       "      <td>nytimes.com</td>\n",
       "      <td>0.000</td>\n",
       "      <td>Strategies: It’s Not Just the White House. Cha...</td>\n",
       "      <td>Not fake</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>James Rufus Koren</td>\n",
       "      <td>2016-11-20T02:42:00.000+02:00</td>\n",
       "      <td>Wells Fargo hit with new sanctions following f...</td>\n",
       "      <td>609.0</td>\n",
       "      <td>latimes.com</td>\n",
       "      <td>0.264</td>\n",
       "      <td>Wells Fargo hit with new sanctions following f...</td>\n",
       "      <td>Not fake</td>\n",
       "      <td>809</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              author                      published  \\\n",
       "0      SAM TANENHAUS  2016-11-20T03:51:00.000+02:00   \n",
       "1                NaN  2016-11-20T00:52:00.000+02:00   \n",
       "2    Kathleen Elkins  2016-11-20T06:18:00.000+02:00   \n",
       "3        JEFF SOMMER  2016-11-20T00:04:00.000+02:00   \n",
       "4  James Rufus Koren  2016-11-20T02:42:00.000+02:00   \n",
       "\n",
       "                                                text  domain_rank  \\\n",
       "0  Privacy Policy Eisenhowers two terms bore this...         98.0   \n",
       "1     Can Trump Save Their Jobs?  by Nelson D. Sc...         98.0   \n",
       "2  Tuesday, 18 Oct 2016 | 10:25 AM ET CNBC.com Ed...        767.0   \n",
       "3  Continue reading the main story Yet it is poss...         98.0   \n",
       "4  Wells Fargo hit with new sanctions following f...        609.0   \n",
       "\n",
       "      site_url  spam_score                                              title  \\\n",
       "0  nytimes.com       0.000       Opinion: Donald Trump’s Art of the New Deal?   \n",
       "1  nytimes.com       0.000                          Carrier Workers for Trump   \n",
       "2     cnbc.com       0.008  Tennis star Caroline Wozniacki shares the mone...   \n",
       "3  nytimes.com       0.000  Strategies: It’s Not Just the White House. Cha...   \n",
       "4  latimes.com       0.264  Wells Fargo hit with new sanctions following f...   \n",
       "\n",
       "   response  length  binary_response  \n",
       "0  Not fake     949                0  \n",
       "1  Not fake     372                0  \n",
       "2  Not fake     386                0  \n",
       "3  Not fake    1005                0  \n",
       "4  Not fake     809                0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert text and response to array \n",
    "x_text = clean_final.text.values\n",
    "y_response = clean_final.binary_response.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove twitter handles from text \n",
    "for idx in range(len(x_text)):\n",
    "    x_text[idx] = re.sub(r'@([A-Za-z0-9_]+)', \"\", str(x_text[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove hyperlinks from text\n",
    "for idx in range(len(x_text)):\n",
    "    x_text[idx] = re.sub(r\"(https|http)\\S+\", \"\", str(x_text[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# helper function to eliminate tokens that are pure punctuation, whitespace, or stopword\n",
    "# can be updated based on desired filtering \n",
    "\n",
    "def process_txt(token):\n",
    "    return token.is_punct or token.is_space or token.is_stop or token.like_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to take array of articles and turn them into nested list of tokens\n",
    "\n",
    "def lemmatize_txt(array):\n",
    "    lemma = []\n",
    "    \n",
    "    for doc in nlp.pipe(array, batch_size=50,\n",
    "                        n_threads=-1):\n",
    "        if doc.is_parsed:\n",
    "            lemma.append([n.lemma_ for n in doc if not process_txt(n)])\n",
    "        \n",
    "        else:\n",
    "            lemma.append(None)\n",
    "    \n",
    "    return lemma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to recombine nested list of tokens into full articles \n",
    "\n",
    "def lemma_combine(lis):\n",
    "    parsed_articles = []\n",
    "    \n",
    "    for i in range(len(lis)):\n",
    "        concat_art = ' '.join(lis[i])\n",
    "        parsed_articles.append(concat_art)\n",
    "    \n",
    "    return parsed_articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to match \"cleaned\" text back up with response variable\n",
    "\n",
    "def zip_response(observations, response):\n",
    "    response = response.tolist()\n",
    "    \n",
    "    return list(zip(observations, response))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create nested list of tokens for each article\n",
    "lem = lemmatize_txt(x_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create bi-grams for our text \n",
    "phrases = Phrases(lem)\n",
    "bigram = Phraser(phrases)\n",
    "bigram_lem = list(bigram[lem])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create tri-grams for our text\n",
    "phrases2 = Phrases(bigram_lem)\n",
    "trigram = Phraser(phrases2)\n",
    "trigram_lem = list(trigram[bigram_lem])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Recombine full article text for unigrams, bigrams, and trigrams\n",
    "uni_lem_comb = lemma_combine(lem)\n",
    "bi_lem_comb = lemma_combine(bigram_lem)\n",
    "tri_lem_comb = lemma_combine(trigram_lem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "token_uni = \"token_unigram_text\"\n",
    "fileObject = open(token_uni,'wb') \n",
    "pickle.dump(lem,fileObject)   \n",
    "fileObject.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "token_trigrams = \"token_trigram_text\"\n",
    "fileObject = open(token_trigrams,'wb') \n",
    "pickle.dump(trigram_lem,fileObject)   \n",
    "fileObject.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save processed text so it can be loaded later \n",
    "unigrams = \"unigram_text\"\n",
    "fileObject = open(unigrams,'wb') \n",
    "pickle.dump(uni_lem_comb,fileObject)   \n",
    "fileObject.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bigrams = \"bigram_text\"\n",
    "fileObject = open(bigrams,'wb') \n",
    "pickle.dump(bi_lem_comb,fileObject)   \n",
    "fileObject.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trigrams = \"trigram_text\"\n",
    "# open the file for writing\n",
    "fileObject = open(trigrams,'wb') '\n",
    "pickle.dump(tri_lem_comb,fileObject)   \n",
    "fileObject.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reload processed text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fileObject = open('unigram_text','rb')  \n",
    "uni_lem_comb2 = pickle.load(fileObject)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fileObject = open('bigram_text','rb')  \n",
    "bi_lem_comb2 = pickle.load(fileObject)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fileObject = open('trigram_text','rb')  \n",
    "tri_lem_comb2 = pickle.load(fileObject)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"trump save jobs nelson d. schwartz nov. look worker carrier_plant indianapolis soon close production move_mexico worker interview speak support donald_j._trump article attract comment follow edit_condense mr._trump not save job nafta_free_trade agreement keep economy_grow underlying dynamic change underlying dynamic have_get better automation b big era american growth doug boulder heartbreaking read people people victim change victim technological revolution victim global_economy victim economy people want pay money possible good company produce product want profit possible trump con single job not come claudia piepenburg vista calif. mr._trump save carrier worker job willing work wage company pay mexican worker scott new_york worker sadly disappointed twice carrier move mr._trump anticipate confusion grief buyer_remorse work force advertisement susan michigan not mr._trump set good example c.e.o.s order company start make product u.s. right lead example mr._trump rita california defeatist germany manufacturing priority not steve minneapolis bernie_sanders voter hold_nose vote hillary_clinton want_hear factory_worker explain situation feeling hate display trump_supporter reporter find people record story respectfully annie dooley georgia letters_sunday business send sunbiz.com version_article_appear_print november_page bu6 new_york_edition_headline comment order_reprint_today 's_paper_subscribe\",\n",
       " \"tuesday oct 10:25 et cnbc.com eduardo_munoz_alvarez_afp getty_images caroline wozniacki denmark celebrate_defeat madison keys united_states open women 's singles match usta billie_jean_king national tennis center new_york september shares letter younger_self world tennis_player caroline wozniacki grant_permission buy car be dream kid red ferrari mega yacht future jetpack faberg egg danish tennis_star write promise easy handbag will allow ferrari continue rothko original sailboat random expensive thing have_get love spend ton_money caroline wozniacki earn prize_money course career believe treat occasion have think work long_term damage finance be o.k. reward love car buy appear earlier_year 26-year_old post picture red ferrari twitter nd roll amfar gala supercar tweet course splurge frugal note wozniacki 's philosophy similar personal_finance expert jim_cramer 's host cnbc_'s mad_money tell farnoosh torabi podcast big_believer find like be expensive money frugal determine extravagance important aim earn income require spend guilt live frugally cramer tell torabi stanley frugal box philadelphia_eagles game say wife ilk be crazy spending big\",\n",
       " \"continue_read_main_story possible election remember pivotal_moment market politic new republican_control government likely enact different policy arguable election give market impetus excuse solidify deep trend underway advertisement_continue_read_main story election shock accelerate change begin evident small way mr. paulsen say think trend likely stick deny election_outcome take market surprise overwhelming assumption financial circle poll hillary_clinton win mr._trump lose expectation unlikely_event trump victory major stock_market decline occur hour powerful stock_market_rally slow stop s story revaluation wide_range asset security take_place recent star fade market transitory judgment new investment vogue trader vote dollar state street sector exchange_trade_fund proxy broad_market mr. mazza report unusually_high inflow sector e.t.f.s financial_service health_care industrial less inflow material energy_stock trader stampede away utility technology consumer_staple share neutral slightly negative consumer_discretionary real_estate share financial stock utility recently investor shun bank celebrate utility steady eddie dividend_payout shift radically day election jpmorgan_chase goldman_sachs_morgan_stanley bank_america gain_percent lead utility exelon firstenergy nextera_energy entergy lose percent election utility share ahead financial year day_later trail nearly inflow financial_select_sector spdr_fund example big week e.t.f.s inception mr. mazza say cash_flow generally correlate market price trend big_weekly outflow occur week jan. financial_crisis investor find bank_stock scary find attractive utility hand move election week avoidance list net outflow utilities select_sector_spdr_fund mr. mazza say advertisement_continue_read_main story trader shift_gear hastily rational basis movement problem lie assumption market shift prediction politic risky base underlie economic bit reliable example bond_market shift appear partly base judgment inflation_rise mr._trumps policy inchoate call tax_cut spend increase nation infrastructure military inflationary widen budget_deficit conservative congress increase deficit_spending clear trump_administration try_accomplish assumption bear fact similarly technology share fall_sharply partly concern possibly negative_effect mr._trumps policy say larry puglia portfolio_manager t._rowe_price possible say mr._trumps approach immigration difficult highly_skilled foreigner receive visa work american tech_company trade_policy logistic global tech manufacturing cumbersome expensive people not jump_conclusion mr. puglia say not_know happen say change investment_portfolio prefer wait hand sea change underway economy mr. paulsen say election give trader excuse act recent datum_indicate economic_growth accelerate election aside say make_sense bond_yield_rise circumstance federal_reserve increase short_term_interest_rate janet_l. yellen_fed chairwoman say rate_increase relatively likely occur soon central_bank meeting december rise_interest_rate generally help bank_stock hurt utility dividend longer appealing furthermore rise_rate support strong_dollar justify decline price_gold favor election addition labor_market improve long_time american economy good shape year_ago reasonable investor aggressive mr. paulsen say risk geopolitical domestic shock real acknowledge trader make educate_guess turn wrong market turn_upside advertisement_continue_read_main story long_term investor argument broad diversification despite turmoil market simple balanced portfolio stock_bond barely move election period extreme swerve shift count achievement twitter_version_article appear_print_november_page bu6 new_york_edition_headline white_house change air wall_st. order_reprint_today 's_paper_subscribe\",\n",
       " 'wells_fargo hit new sanction follow fake_account scandal_wells_fargo branch frederic_j._brown_afp getty_images pedestrians_walk_past branch_wells_fargo bank_downtown_los angeles_oct. pedestrian_walk past branch_wells_fargo bank_downtown_los angeles_oct. frederic_j._brown_afp getty_images james_rufus_koren co. require_approval replace hire new executive change late fallout san_francisco bank fake_account_scandal late_friday office_comptroller_currency regulator reach_settlement bank creation_unauthorized customer_account issue_brief_statement say revoke term sept. deal specifically occ say cancel part agreement shield wells_fargo oversight usually_reserve trouble bank occ issue_statement explain cancellation wade francis occ bank examiner say amount regulatory vote_confidence bank leadership occ say not trust run business say francis president long_beach bank consultancy unicon financial_services question judgment management wells_fargo provide occ write notice plan replace board_member bank executive change appear apply wells_fargo_bank separate board corporate parent wells_fargo_co. clear change impact wells_fargos appointment new chief_executive month bank longer entitle expedited regulatory review application basic practice include opening relocate bank_branch regulatory order prohibit bank make call golden_parachute_payment compensation pay executive step fire ban typically impose bank troubled insolvent occ say not trust bank prudent francis say wells_fargo respond_request_comment statement wall_street_journal bank spokeswoman_jennifer dunn say new order inhibit_ability executive strategy caption_second_story not_miss_trump_supporter be_fair_equate white_pride_white_supremacist rams_break_ground new_stadium_trump move_forward_name_position administration_los_angeles_practice major_earthquake_caption_second story_not_miss trump_supporter_be fair_equate_white_pride white_supremacist_rams_break ground_new_stadium trump_move_forward name_position_administration_los angeles_practice_major_earthquake caption_los_angeles rams_break_ground new_stadium_groundbreaking_ceremony hold_inglewood_construction_los angeles_rams_new_2.6-billion stadium_officially_get_underway genaro_molina_los_angeles times_groundbreaking_ceremony_hold inglewood_construction_los_angeles rams_new_2.6-billion stadium_officially_get_underway genaro_molina_los_angeles times_caption_richard_spencer chairman_national_policy_institute richard_spencer_chairman_national policy_institute_prominent_alt right_organization_joshua roberts_times_richard_spencer chairman_national_policy_institute prominent_alt_right organization_joshua_roberts_times caption_vigilant_guard_california national_guard_train_earthquake response_weeklong_earthquake_response train_exercise_call_vigilant guard_host_california_california national_guard_participation_member local state_federal_agency include los_angeles_county_fire official_los_angeles county_sheriffs_department_al seib_los_angeles_times weeklong_earthquake_response_train exercise_call_vigilant_guard host_california_california_national guard_participation_member_local state_federal_agency include los_angeles_county_fire official_los_angeles county_sheriffs_department_al seib_los_angeles_times caption_weekend_roundup_story not_miss_l.a._auto begin_spot_fake_news facebook_rural_teen_baby indias_large_bill_ban have_negative_effect_poor twitter_begin_ban bully_northern_californians_feel forget_rams_break_ground new_stadium_l.a._auto begin_spot_fake_news facebook_rural_teen_baby indias_large_bill_ban have_negative_effect_poor twitter_begin_ban bully_northern_californians_feel forget_rams_break_ground new_stadium']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tri_lem_comb2[1:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_model(clf, actual, predicted):\n",
    "    report = metrics.classification_report(actual, predicted, target_names = [\"0\", \"1\"])\n",
    "    matrix = confusion_matrix(actual, predicted)\n",
    "    train_loss = metrics.log_loss(y_train, clf.predict_proba(x_train))\n",
    "    test_loss = metrics.log_loss(y_test, clf.predict_proba(x_test))\n",
    "    \n",
    "    print(\"Train score:\", train_loss)\n",
    "    print(\"Test score:\", test_loss)\n",
    "    print(\"Confusion Matrix\\n\", matrix)\n",
    "    print(\"Classification Report:\\n\", report)\n",
    "    \n",
    "    return test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Split test and train data using trigram text\n",
    "x_train, x_test, y_train, y_test = train_test_split(tri_lem_comb2, y_response, test_size = 0.2, stratify = y_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"let_friend_social_network know_read_facebook email search resume miss indonesia_boat sink people boat miss post_facebook search resume miss indonesia_boat sink people boat miss check_story_usatoday.com_cancel send_link_send_friend 's_email_address post_link_post_facebook feed_join_nation_'s conversation_find_facebook_comment read_conversation_guidelines_faq search resume miss indonesia_boat sinking associated_press 2:59 a.m._edt_november indonesian rescuer evacuate body victim sink boat mata ikan bay batam riau islands indonesia nov._photo m urip epa jakarta_indonesia_indonesian rescue authority thursday resume search people miss sinking overcrowded speedboat day_earlier search effort suspend nightfall wednesday start daylight personnel involve say sutopo purwo nugroho head countrys disaster mitigation agency eighteen body identify people rescue accord agency people boat miss sutopo say boat carry indonesian worker home malaysia capsize indonesian_island batam a.m. wednesday stormy_weather hardin nafiih chief riau islands province disaster mitigation agency say boat_sink kilometer_mile land police suspect trip illegal worker_undocumented high fare_passenger say pay board include crew_member batam police_chief sambudi gusdian say indonesian woman believe responsible voyage arrest wednesday try leave batam nearby singapore haryanto 51-year_old survivor say boat_capsize heavy_rain high wave hour leave johor bahru malaysia crowd sit say haryanto save fisherman like indonesians go speedboat ferry common form transport indonesia sprawl_archipelago sinking common poorly_enforce safety regulation bad ferry sinking recent_year occur sulawesi island kill_people\"]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Bayes Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up pipeline to run TFIDF and Naive Bayes\n",
    "text_pipe_nb = Pipeline([('tfidf', TfidfVectorizer()), ('nb', MultinomialNB())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tfidf', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,\n",
       " ...rue,\n",
       "        vocabulary=None)), ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_pipe_nb.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83021525570644328"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use pipeline to predict on train data and find accuracy - no grid search\n",
    "#predicted_train = text_pipe.predict(x_train)\n",
    "#np.mean(predicted_train == y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.79806442293803259"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use pipeline to predict on test data and find accuracy - no grid search \n",
    "#predicted = text_pipe.predict(x_test)\n",
    "#np.mean(predicted == y_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   Not fake       0.77      1.00      0.87      4715\n",
      "       fake       0.99      0.37      0.54      2208\n",
      "\n",
      "avg / total       0.84      0.80      0.76      6923\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Get more detailed performance metrics - no grid search\n",
    "print(metrics.classification_report(y_test, predicted, target_names = [\"Not fake\", \"fake\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up grid search to find best parameters for TFIDF and MNB\n",
    "parameters = {'tfidf__max_df': (0.75, 0.80, 0.85, 0.90, 0.95, 1.0),\\\n",
    "              'tfidf__min_df': (0.001,0.01,0.1),\\\n",
    "              'nb__alpha': (1, 0.1, 0.01, 0.001, 0.0001, 0.00001)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Establish grid search instance\n",
    "gs_nb = GridSearchCV(text_pipe_nb, parameters, n_jobs=-1, scoring = 'f1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.172519814274\n",
      "Test score: 0.212836566076\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.21283656607574009"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "# Fit grid search to train data and find best parameters\n",
    "gs_nb = gs_nb.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clf__alpha': 1e-05, 'tfidf__max_df': 0.75, 'tfidf__min_df': 0.001}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check best parameters\n",
    "gs_nb.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90423226924743605"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Predict on test data\n",
    "nb_predicted = gs_nb.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   Not fake       0.91      0.95      0.93      4715\n",
      "       fake       0.88      0.81      0.84      2208\n",
      "\n",
      "avg / total       0.90      0.90      0.90      6923\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Produce confusion matrix and classification report for nb model\n",
    "evaluate_model(gs_nb, y_test, nb_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'gs_nb' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-68-3644c443b954>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mfilename\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'nb_model.sav'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgs_nb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'gs_nb' is not defined"
     ]
    }
   ],
   "source": [
    "filename = 'nb_model.sav'\n",
    "pickle.dump(gs_nb, open(filename, 'wb'))\n",
    "filename.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_pipe_lr = Pipeline([('tfidf', TfidfVectorizer()), ('lr', LogisticRegression())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tfidf', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,\n",
       " ...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_pipe_lr.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up grid search to find best parameters for TFIDF and LogReg\n",
    "lr_parameters = {'tfidf__max_df': (0.75, 0.80, 0.85, 0.90, 0.95, 1.0),\\\n",
    "              'tfidf__min_df': (0.001,0.01,0.1),\\\n",
    "              'lr__penalty': ('l1','l2'),\\\n",
    "                'lr__C': (0.01, 0.1, 1),\\\n",
    "                'lr__class_weight': (None, 'balanced')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Establish grid search instance for log reg\n",
    "gs_lr = GridSearchCV(text_pipe_lr, lr_parameters, n_jobs=-1, scoring='f1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1h 13min 21s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# Fit grid search to train data and find best parameters\n",
    "gs_lr = gs_lr.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lr__C': 1,\n",
       " 'lr__class_weight': 'balanced',\n",
       " 'lr__penalty': 'l2',\n",
       " 'tfidf__max_df': 0.75,\n",
       " 'tfidf__min_df': 0.001}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get best parameters for log reg\n",
    "gs_lr.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predict on test data \n",
    "lr_predicted = gs_lr.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.199665811636\n",
      "Test score: 0.237676017458\n",
      "Confusion Matrix\n",
      " [[4373  342]\n",
      " [ 202 2006]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.96      0.93      0.94      4715\n",
      "          1       0.85      0.91      0.88      2208\n",
      "\n",
      "avg / total       0.92      0.92      0.92      6923\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.23767601745789554"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Produce confusion matrix and classification report for lr model\n",
    "evaluate_model(gs_lr, y_test, lr_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.955096607469\n",
      "Test score: 0.917990014139\n"
     ]
    }
   ],
   "source": [
    "train_loss = metrics.roc_auc_score(y_train, gs_lr.predict(x_train))\n",
    "test_loss = metrics.roc_auc_score(y_test, gs_lr.predict(x_test))\n",
    "print(\"Train score:\", train_loss)\n",
    "print(\"Test score:\", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1e85fdd0588>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuQXHWd9/H3t+c+yWRmQoYQciEB2UDIZYAAK/qo6KoJ\nXtB9rEXQRS1WHlyzrJYlULL1uN4odEtEilvxeME75Sor7D5RFB4WtFiUoJFcIBiDKzNcEsJ0z6Un\n093T3+eP7pn0zHTPdGZOnzPT83lVTaX7nDOnv4dJPvzmd36/3zF3R0REqkss6gJERCR4CncRkSqk\ncBcRqUIKdxGRKqRwFxGpQgp3EZEqpHAXEalCCncRkSqkcBcRqUK1UX3wkiVLfPXq1VF9vIjInPTE\nE0+87O4dUx0XWbivXr2aHTt2RPXxIiJzkpn9dznHqVtGRKQKKdxFRKqQwl1EpAop3EVEqpDCXUSk\nCincRUSqkMJdRKQKKdxFRKqQwl1EpAop3EVEqpDCXUSkCincRUSqkMJdRKQKTRnuZvYNMztoZrtL\n7Dczu9nM9pvZk2Z2VvBliojIsSin5X4XsGWS/VuBU/NfVwC3z7wsERGZiSnXc3f3R8xs9SSHXAR8\n290deMzM2sxsmbu/EFCNIiKRymad1HCWoXSWocwwQ5ksQ0NHyCR7GE72kB3ogSO9bHjD/4y61FFB\nPKxjOfBcwfuu/LYJ4W5mV5Br3bNq1aoAPlpEqp27kx72o6GayTKUzr1OjbzPDOeDN/e61PYJ35Ma\nhnSSunSChnSChnQfjcO9NGUSNA/3sSDbz8JsHy300cYArTZAm/XTxgAtNjimzqwb/rp3YbGaiP5L\njRXqk5jc/U7gToDNmzd7mJ8tItOTGc4WtFoLWq7pLKnhieE5lM4yNHw0gMeHbKrw2AlBXWT7cBaf\nIi1iZGkhmQ/eftpsgFYGaM2/X2wDtMeStMf6abcBWumnhQEWeR91ZEqedzhWy2D9IoZqW0nVtZKu\nX8VQfSsvNLTR3diG57+saTE0t3NawP/tZyKIcO8GVha8X5HfJiIzNNodMFUrtEiQTuhGKLl9XGt3\n3L7h7MzbYfW1MRpqYzTU1uT+rItRXxOjoS73vqWxlobaGhbUZGi3flrzAbzI+2nJ9rHA+1mQ7aN5\nuJemTC8NmT4aMgkaUgnq0glqUn0Ypev0+hasqQ2a2qDpeGhaC41t0NRe8DX+fTs1dc0sNGPhjP8L\nhC+IcL8P2GZmdwPnAQn1t0s1KOwOOPqr/GSt0FJdA2PDd8z3pIsEdeH5h7Mzvo7amOUDNR+s+ZAd\nCdzGuhitTXVF9zXUFQRybYz6gnAu3N5QV5MP6xgNNUZjNpkL33SCulQvNtgDI19H4vnXBX/25/dl\nBktfiMWOBm9jG7Qug6Z1xYN5THC3YTV1M/7vONdMGe5m9gPgDcASM+sCPg3UAbj7HcB24EJgP5AE\nPlSpYmV+Gc76lK3QMYE7WX9sQZCW2l7se6bqDpiKGRNarCOvRwK0rbl+QvjWT/E9DbU1+e3jQnbc\n9vraGDUxm+YPIF0QwC+PDebennEBXRjeCfDh0uetbRobxIvXQNOZJYK5ILjrWyCmqTnlKme0zCVT\n7Hfgo4FVJLOCu5f1K/tQevjY+2Mzk33P0XDOBNUdUFMkIOtqaKiJsbChluMWTAzSoyE6NjDHhG7J\n7UeDujZmmE0zXIPgDkP941rMJYJ5fIs61T/JiQ0aW8e2mNtPmrqro7EN6hpDu/z5LNQbqlIedyeT\n9WP8db68X/PH9rPmw7lI6zeVCa47oL5ES7O+oK91fGuzVOu0flx4TtaNUF8TIzbdVutskx3OtYin\nDOYi+7Pp0uetqR8bvItWwNINRYK5DRoLtjW2wiwZFSLFKdwj8OBTL3Hbf/6RI+mJXQwjQT3TRmth\nd0CpVmhrUx0NLQ1lt0LHnKNod8HYm2W1NfoVeoL04LEF88j7ocTk561vGRvIx582SR90QWDXNef+\nskjVUbhH4P/ueoE9zyd4zSlLSt+YGn/TaqS7oKbE9nHhW1cTcXdANctmYah3imCOFw/vzJHS57Wa\nsQG98HjoKGNUR2MrzMMbhjI5hXsEEsk0p3Qs5OsfPCfqUua3TGpcOE/Vos4fcyQOPkm3VV3z2Bbz\n4pMn74Meed3Qola0BEbhHoGeZIq2ZrW0AuGeu/FXbjAXHpcemOTEIzcMC4K4fXV5XR21DWFdvUhJ\nCvcIxAfTLGttirqM2WU4c/SG4ZR90OOOyZaeYZi7Ybj4aIu5bSUs21gQzG3FW9UNrRp2J3Oawj0C\niWS6elvu6cEyh9sVfiWmvmHYsKhgpEYbLDpx0pmFo+Fd16SuDpmXFO4hc3fig7M83LPZXNhONQ66\nWHgPD5U+b6x2bFfGwhOg47TSwTzmhqH+qoocC/2LCVnfUIbhrNPWVF/5D8sMHQ3fY+rqiMMk63RQ\nt2Bsi3nJqybvgx55Xb9QrWiRkCjcQxYfyE0oKbvlPnrDsJxgHteinuqG4fjujMlGdYyGt24YiswF\nCveQxQdTALQ1F2m59/w33P8p6D84NrQnvWHYMDaA206CZZsmBvT4VnXDIt0wFKliCveQxZO5lnt7\nsZb7H34OT/8HrP4fcML68ro66jTqRkQmUriHrCc50nIvEu6JLojVwWX3qVUtIjOiBAlZYjDXcm8t\ndkO1txtalyvYRWTGlCIhG+mWKd5y786tyiciMkMK95D1JFMsbKilrtiKib1duZa7iMgMKdxDlkim\naW0q0mrPDkPv87BI4S4iM6dwD1l8ME37giLh3n8wN+RRLXcRCYDCPWQ9yVTx2am93bk/1ecuIgFQ\nuIcskUzTWmoYJKjlLiKBULiHLD6YLj6BabTlrnAXkZlTuIcom3XipbplEt1Hn+AjIjJDCvcQ9Q1l\nyHqJMe69XblWu1ZNFJEAKNxDFE9OsmhYolv97SISGIV7iEZnpxYb596r2akiEhyFe4ji+XVlJoxz\nH05D34tquYtIYBTuIRrplpmwaFjfC4BrpIyIBKascDezLWa2z8z2m9m1Rfa3mtm/m9nvzWyPmX0o\n+FLnvpKLhiXywyDVcheRgEwZ7mZWA9wKbAXWAZeY2bpxh30U2Ovum4A3AF82sxAeEjq3lOxz1+xU\nEQlYOS33c4H97n7A3VPA3cBF445xoMXMDFgIvAJM8my4+aknmaKloZba8StCanaqiASsnHBfDjxX\n8L4rv63QLcDpwPPALuAf3T0bSIVVJDFYYumB3m5oaIWGlvCLEpGqFNQN1bcCO4ETgU7gFjNbNP4g\nM7vCzHaY2Y5Dhw4F9NFzRzyZol1j3EUkBOWEezewsuD9ivy2Qh8C7vGc/cCzwGnjT+Tud7r7Znff\n3NHRMd2a56yeZHry2akiIgEpJ9wfB041szX5m6TvBe4bd8yfgTcBmNlSYC1wIMhCq0FisMSDOhLd\n0KqbqSISnNqpDnD3jJltA+4HaoBvuPseM7syv/8O4HPAXWa2CzDgGnd/uYJ1z0lFu2XSg5B8Wd0y\nIhKoKcMdwN23A9vHbbuj4PXzwFuCLa26ZLNOYrBIt0zv87k/NQxSRAKkGaoh6TuSWxFyQreMhkGK\nSAUo3EPSk196YEK3jB7SISIVoHAPyciiYSWXHlC4i0iAFO4hKbmWe28XNC+BusYIqhKRaqVwD8mk\ni4apv11EAqZwD8loy73YomEaKSMiAVO4h2Skz33iaBm13EUkeAr3kMSTaVoax60IOdQHQwndTBWR\nwCncQxJPpiZ5SIe6ZUQkWAr3kMQH00XGuOcnMKnlLiIBU7iHpCdZZNEwPV5PRCpE4R6SRDJVZIx7\nN2DQsiySmkSkeincQ5LrlinScm85AWqKLAMsIjIDCvcQDI+sCDlhjLse0iEilaFwD0HfkTTuRZYe\n0Bh3EakQhXsIeootPeCu2akiUjEK9xAcXTSsINwHeyCdVMtdRCpC4R6Co8v9FnTL9GoCk4hUjsI9\nBEUXDRtdx13hLiLBU7iH4OhyvwUt98RzuT/VLSMiFaBwD8FIuI+ZodrbDbE6WHB8RFWJSDVTuIcg\nnkyxqLGWmpgd3ZjohkXLIKYfgYgET8kSgvhguvjSA+pvF5EKUbiHIJ4stvRAl/rbRaRiFO4hiCdT\ntBa23LNZ6H1eSw+ISMUo3EMQH7+uzMAhyKY1xl1EKkbhHoIJ3TJ6SIeIVFhZ4W5mW8xsn5ntN7Nr\nSxzzBjPbaWZ7zOzhYMucu4azTu+R9NhuGT2kQ0QqrHaqA8ysBrgVeDPQBTxuZve5+96CY9qA24At\n7v5nM9Pg7bzewdyKkGNb7pqdKiKVVU7L/Vxgv7sfcPcUcDdw0bhjLgXucfc/A7j7wWDLnLuOritT\nuPRAF9Q2QvPiiKoSkWpXTrgvB54reN+V31boL4B2M/tPM3vCzC4LqsC5rmd0XZlxi4YtWg5mJb5L\nRGRmpuyWOYbznA28CWgC/svMHnP3ZwoPMrMrgCsAVq1aFdBHz26JYmu56yEdIlJh5bTcu4GVBe9X\n5LcV6gLud/cBd38ZeATYNP5E7n6nu292980dHR3TrXlOGW25j1/uV/3tIlJB5YT748CpZrbGzOqB\n9wL3jTvmXuC1ZlZrZs3AecBTwZY6N42uCDkyzn04A30vqOUuIhU1ZbeMu2fMbBtwP1ADfMPd95jZ\nlfn9d7j7U2b2M+BJIAt8zd13V7LwuSI+mMYMFo2Ee/+L4FmNcReRiiqrz93dtwPbx227Y9z7fwH+\nJbjSqkNuRci6oytCJvQEJhGpPM1QrbB4Mj32Zqpmp4pICBTuFTZhuV+13EUkBAr3CosnU2MXDevt\nhoZF0LgouqJEpOop3CtsQrdMoktdMiJScQr3CosnU7SP6ZbRQzpEpPIU7hWUGc7SeyQz8cHYarmL\nSIUp3Cuo90gGKFgRMjOUe1CHbqaKSIUp3CsoPn7pgdGlftVyF5HKUrhXUE9+6YHWkZa7HtIhIiFR\nuFdQYjDXcm+f0HJXt4yIVJbCvYJ6BsYtGpYYmZ16YkQVich8oXCvoAlPYerthqbFUN8cYVUiMh8o\n3CsokUzlVoRsLOhzV3+7iIRA4V5BPck0rU11xEZWhNRDOkQkJAr3CooPpseuK6PZqSISEoV7BcWT\nqaNj3FMDcCSuMe4iEgqFewWNWTRMS/2KSIgU7hUUHyxY7lcP6RCRECncKyjXcs93y2h2qoiESOFe\nIZnhLH1HMmPHuGPQoglMIlJ5CvcKSeQnMI0uPZDogoXHQ239JN8lIhIMhXuFFJ2dqpupIhIShXuF\njCz3O/qgjoQe0iEi4VG4V0g8WdAt456fwKSWu4iEQ+FeISNrubc11+UmL6UH1HIXkdAo3Ctk9ClM\nTfUaBikioVO4V0hiME3MoKWxVg/pEJHQKdwrpCeZOroi5MhDOtRyF5GQlBXuZrbFzPaZ2X4zu3aS\n484xs4yZvSe4EuemMbNTe7shVgsLl0ZblIjMG1OGu5nVALcCW4F1wCVmtq7EcV8Efh50kXNRYnDc\nomEtyyBWE21RIjJvlNNyPxfY7+4H3D0F3A1cVOS4fwB+DBwMsL45qydZuGiYxriLSLjKCfflwHMF\n77vy20aZ2XLg3cDtk53IzK4wsx1mtuPQoUPHWuucEk+mxy49oP52EQlRUDdUbwKucffsZAe5+53u\nvtndN3d0dAT00bNTIpmmtbkuN4Gp93m13EUkVLVlHNMNrCx4vyK/rdBm4G4zA1gCXGhmGXf/SSBV\nzjHp4Sx9Q5ncGPeBl2F4SLNTRSRU5YT748CpZraGXKi/F7i08AB3XzPy2szuAv5jvgY7FKwIuaBO\nD+kQkUhMGe7unjGzbcD9QA3wDXffY2ZX5vffUeEa55yRdWVam+o0O1VEIlFOyx133w5sH7etaKi7\n+wdnXtbcNrr0QHM99Gh2qoiETzNUK+DoipB1uZEyNQ2wYEnEVYnIfKJwr4DRB3U01efHuJ8IuZvN\nIiKhULhXwOiDOprzfe4aKSMiIVO4V0A8maYmZiwaWRFSI2VEJGQK9woYWRHSPJubwKSWu4iETOFe\nAfHBdG5dmf6XwIc1DFJEQqdwr4BEMr8i5Mg67hoGKSIhU7hXQE8ylRvjrod0iEhEFO4VEB9puY8+\nXk/hLiLhUrhXQGIwffTB2PULobE16pJEZJ5RuAcslcnSP5TJt9y7cq12TWASkZAp3AM2uiLk6AQm\ndcmISPgU7gFLDI7MTq3XBCYRiYzCPWA9I4uG1Tv0H9QEJhGJhMI9YCMrQnbwCuBquYtIJBTuARtZ\nNGxx5mBug/rcRSQCCveAjbTcF6Zeym3Q7FQRiYDCPWDxwRQ1MaMp+WJug1ruIhIBhXvAepK5RcOs\ntxsa26B+QdQlicg8pHAPWCKZ1kM6RCRyCveAxQdTtDfXH52dKiISAYV7wHoG8mu5a3aqiERI4R6w\nxGCajsYsDL6ilruIRKY26gKqTTyZYmVtbqy7+txFJCoK9wClMlkGUsOcSDy3QeEuIhFRt0yA4vlF\nw5bycm6DumVEJCIK9wAl8rNTFw/nlx5YdGKE1YjIfFZWuJvZFjPbZ2b7zezaIvvfZ2ZPmtkuM3vU\nzDYFX+rsN7IiZGvqICw4HmobIq5IROarKcPdzGqAW4GtwDrgEjNbN+6wZ4HXu/sG4HPAnUEXOheM\nLBq2cOglDYMUkUiV03I/F9jv7gfcPQXcDVxUeIC7P+ruPfm3jwHz8k5iPP8UpsbkC+pvF5FIlRPu\ny4HnCt535beVcjnw02I7zOwKM9thZjsOHTpUfpVzxEjLvbb/eY2UEZFIBXpD1cwuIBfu1xTb7+53\nuvtmd9/c0dER5EfPCvFkmrbYIJbqV8tdRCJVzjj3bmBlwfsV+W1jmNlG4GvAVnc/HEx5c0tPMs3a\npgQMoz53EYlUOS33x4FTzWyNmdUD7wXuKzzAzFYB9wB/6+7PBF/m3JAYTHFyfX4Ckx7SISIRmrLl\n7u4ZM9sG3A/UAN9w9z1mdmV+/x3A/waOA24zM4CMu2+uXNmzUzyZZmNt/r6yWu4iEqGylh9w9+3A\n9nHb7ih4/XfA3wVb2tzTk0yzvOYVsBgsPCHqckRkHtMM1QAlkilO4DC0LIMaLdsjItFRuAcoPphm\nyfAhjZQRkcgp3AMylBkmmRqmPXNI/e0iEjmFe0Byi4Z5bukBtdxFJGIK94DEB9O000dtdkizU0Uk\ncgr3gPQMpDjRXsm9UctdRCKmcA9IfDDNMstPzFWfu4hETOEekESyINw1O1VEIqZwD0hPMsWJdhiv\nqYcF1bcomojMLQr3gMQH06yIHc49Wi+m/6wiEi2lUEDiyRQrYj2YumREZBbQHPmAxEf63Fs3Rl2K\nSNnS6TRdXV0cOXIk6lJknMbGRlasWEFdXd20vl/hHpDEwBGW+GENg5Q5pauri5aWFlavXk1+RVeZ\nBdydw4cP09XVxZo1a6Z1DnXLBGXgILUMaxikzClHjhzhuOOOU7DPMmbGcccdN6PfqBTuAWkafDH3\nQn3uMsco2Genmf5cFO4BWXjkpdwLtdxFjklNTQ2dnZ2sX7+ed7zjHcTj8dF9e/bs4Y1vfCNr167l\n1FNP5XOf+xzuPrr/pz/9KZs3b2bdunWceeaZfOITn5hw/rvuuotYLMaTTz45um39+vX86U9/AmD1\n6tVs2LCBzs5ONmzYwL333jvhHOeddx6dnZ2sWrWKjo4OOjs76ezsHD1HOa677joeeuihso+fMXeP\n5Ovss8/2ajGYyvhnP/X37p9e5D5wOOpyRMq2d+/eqEvwBQsWjL6+7LLL/POf/7y7uyeTST/55JP9\n/vvvd3f3gYEB37Jli99yyy3u7r5r1y4/+eST/amnnnJ390wm47fddtuE83/zm9/0lStX+t/8zd+M\nbjvjjDP82WefdXf3k046yQ8dOuTu7k8//bSvWrWqZK3f/OY3/aMf/WjJ/ZlMppxLLluxnw+ww8vI\nWLXcA5DILz2QqWmEpvaoyxGZs1796lfT3d0NwPe//31e85rX8Ja3vAWA5uZmbrnlFm644QYAvvSl\nL3Hddddx2mmnAbnfAD7ykY8UPe/b3/529uzZw759+yb9/N7eXtrby/83nMlkaGtr42Mf+xgbN27k\nN7/5DZ/+9Kc555xzWL9+PVdeeeXobxrvf//7+clPfgLAihUr+Od//mfOPPNMNm7cyDPPBP/oaY2W\nCcDIMMgjTctYqP5LmaM+8+972Pt8b6DnXHfiIj79jjPKOnZ4eJgHH3yQyy+/HMh1yZx99tljjjnl\nlFPo7++nt7eX3bt3F+2GKSYWi3H11Vdz/fXX861vfWvC/gsuuAB358CBA/zwhz8s65wjEokEr3vd\n67jpppsAWLt2LZ/5zGdwdy699FJ+9rOfsXXr1gnft3TpUn73u99x8803c+ONN3LHHXdMOGYm1HIP\nQG7pgVfILDwx6lJE5pzBwUE6Ozs54YQTeOmll3jzm99ckc+59NJLeeyxx3j22Wcn7HvooYfYvXs3\nu3btYtu2bfT395d93vr6et797nePvn/wwQc599xz2bRpEw8//DB79uwp+n1//dd/DcDZZ599TH33\n5VLLPQDxZJpOO4wvOivqUkSmrdwWdtCamprYuXMnyWSSt771rdx6661cddVVrFu3jkceeWTMsQcO\nHGDhwoUsWrSIM844gyeeeIJNmzaV9Tm1tbV84hOf4Itf/GLJY0455RSWLl3K3r17Offcc8uuf2Rk\nSzKZZNu2bfz2t79l+fLl/NM//VPJ4YwNDQ1Arjspk8mU9VnHQi33APQODHA8cWraNQxSZLqam5u5\n+eab+fKXv0wmk+F973sfv/rVr3jggQeAXAv/qquu4uqrrwbgk5/8JNdff/1of3U2m52ya+ODH/wg\nDzzwAIcOHSq6/+DBgzz77LOcdNJJ07qGwcFBYrEYS5Ysoa+vjx//+MfTOk8QFO4BSMWfJ2ZOw+JV\nUZciMqeN3GD8wQ9+QFNTE/feey+f//znWbt2LRs2bOCcc85h27ZtAGzcuJGbbrqJSy65hNNPP531\n69dz4MCBSc9fX1/PVVddxcGDB8dsv+CCC+js7OSCCy7ghhtuYOnSpdOq/7jjjuMDH/gA69atY+vW\nrZx33nnTOk8QbORObtg2b97sO3bsiOSzg/bdH97N+/f+L/x9P8ZO/auoyxEp21NPPcXpp58edRlS\nQrGfj5k94e6bp/petdwDUNv3PACmZ6eKyCyhcA9A/cALuReanSois4TCPQDNQy8yYAugoSXqUkRE\ngDLD3cy2mNk+M9tvZtcW2W9mdnN+/5NmNq/GBC4aeome2uOjLkNEZNSU4W5mNcCtwFZgHXCJma0b\nd9hW4NT81xXA7QHXOastHj5EX8P07q6LiFRCOS33c4H97n7A3VPA3cBF4465CPh2fl2bx4A2M1sW\ncK2z1vHZQww2zZvLFZE5oJxwXw48V/C+K7/tWI+pSkeS/Sy2PlILFO4i0zFflvwFuOeee3j66aeP\n6XumK9QbqmZ2hZntMLMdpWaIzT3Or0+/jiVnvi3qQkTmpJHlB3bv3s3ixYu59dZbgdxsz3e+851c\ne+217Nu3j9///vc8+uij3HbbbQDs3r2bbdu28d3vfpe9e/eyY8cOXvWqVxX9jBUrVvCFL3yhZA0P\nPfQQO3fu5Ec/+hFXXXXVhP2//vWv2blzJ5/97Ge5+OKL2blzJzt37mT16tXHdK2zLdy7gZUF71fk\ntx3rMbj7ne6+2d03d3R0HGuts1JjcwvnXXw1r9r02qhLEZnz5tqSv5D77eHVr341Z511FhdffDED\nAwNAbnmEdevWsXHjRq655hp++ctfsn37dj7+8Y9Pq9V/rMpZOOxx4FQzW0MusN8LXDrumPuAbWZ2\nN3AekHD3FwKtVEQq66fXwou7gj3nCRtg6w1lHToXl/w9ePAgN9xwAw8++CDNzc184Qtf4Ktf/SqX\nX34527dvZ8+ePZgZ8XictrY2LrzwQt7znvfwrne9q+zPmK4pW+7ungG2AfcDTwE/dPc9ZnalmV2Z\nP2w7cADYD/wf4O8rVK+IVJm5vOTvo48+yt69ezn//PPp7Ozke9/7Hn/6059YvHgxsViMD3/4w/zb\nv/0bCxYsCPpyplTWkr/uvp1cgBduu6PgtQMfDbY0EQlVmS3soM3lJX/dnS1btvCd73xnwr4dO3bw\ni1/8gn/913/l9ttv5+c//3lZdQZFM1RFZFaYi0v+nn/++Tz88MOjq1EODAzwhz/8gb6+Pnp7e3n7\n29/OV77yFX73u98B0NLSQl9fX1nnnimFu4jMGnNtyd+lS5fy9a9/nYsvvphNmzZx/vnn88wzz5BI\nJHjb297Gpk2beP3rX8+NN94IwCWXXML1118fyg1VLfkrMo9pyd/ZTUv+iojIGAp3EZEqpHAXEalC\nCneReS6q+24yuZn+XBTuIvNYY2Mjhw8fVsDPMu7O4cOHaWxsnPY5yprEJCLVacWKFXR1dZUc9y3R\naWxsZMWK6T+XWeEuMo/V1dWxZs2aqMuQClC3jIhIFVK4i4hUIYW7iEgVimz5ATPrAyZfOX9uWQK8\nHHURAamma4Hqup5quhaorusJ61pOcvcpn3YU5Q3VfeWsjzBXmNmOarmearoWqK7rqaZrgeq6ntl2\nLeqWERGpQgp3EZEqFGW43xnhZ1dCNV1PNV0LVNf1VNO1QHVdz6y6lshuqIqISOWoW0ZEpAqFFu5m\nttjMfmFmf8j/2V7kmJVm9pCZ7TWzPWb2j2HVVw4z22Jm+8xsv5ldW2S/mdnN+f1PmtlZUdRZrjKu\n533569hlZo+aWXlPIo7AVNdScNw5ZpYxs/eEWd+xKud6zOwNZrYz/2/l4bBrLFcZf89azezfzez3\n+Wv5UBR1lsPMvmFmB81sd4n9sycD3D2UL+BLwLX519cCXyxyzDLgrPzrFuAZYF1YNU5Rfw3wR+Bk\noB74/fjagAuBnwIG/CXw66jrnuH1nA+0519vna3XU861FBz3/4DtwHuirnuGP5s2YC+wKv/++Kjr\nnsG1fGokD4AO4BWgPuraS1zP64CzgN0l9s+aDAizW+Yi4Fv5198C3jX+AHd/wd1/m3/dBzwFLA+t\nwsmdC+yos97PAAAC3klEQVR39wPungLuJndNhS4Cvu05jwFtZrYs7ELLNOX1uPuj7t6Tf/sYMP0l\n6iqrnJ8NwD8APwYOFtk3m5RzPZcC97j7nwHcfbZeUznX4kCLmRmwkFy4Z8Itszzu/gi5+kqZNRkQ\nZrgvdfcX8q9fBCZ9vLiZrQbOBH5d2bLKthx4ruB9FxP/x1POMbPFsdZ6ObkWyWw05bWY2XLg3cDt\nIdY1XeX8bP4CaDez/zSzJ8zsstCqOzblXMstwOnA88Au4B/dPRtOeYGbNRkQ6AxVM3sAOKHIrusK\n37i7m1nJYTpmtpBcC+tj7t4bZI1y7MzsAnLh/tqoa5mBm4Br3D2bayDOebXA2cCbgCbgv8zsMXd/\nJtqypuWtwE7gjcApwC/M7Jf6tz8zgYa7u/9VqX1m9pKZLXP3F/K/phT9NdLM6sgF+/fc/Z4g65uh\nbmBlwfsV+W3HesxsUVatZrYR+Bqw1d0Ph1TbsSrnWjYDd+eDfQlwoZll3P0n4ZR4TMq5ni7gsLsP\nAANm9giwidx9qtmknGv5EHCD5zqt95vZs8BpwG/CKTFQsyYDwuyWuQ/4QP71B4B7xx+Q73P7OvCU\nu98YYm3leBw41czWmFk98F5y11ToPuCy/B3zvwQSBV1Rs82U12Nmq4B7gL+d5S3CKa/F3de4+2p3\nXw38CPj7WRrsUN7ftXuB15pZrZk1A+eRu0c125RzLX8m9xsIZrYUWAscCLXK4MyeDAjxLvNxwIPA\nH4AHgMX57ScC2/OvX0vu5sqT5H5N2wlcGNXd5iLXcCG5ltEfgevy264Ersy/NuDW/P5dwOaoa57h\n9XwN6Cn4WeyIuubpXsu4Y+9iFo+WKfd6gE+SGzGzm1wXZuR1T/Pv2YnAz/P/ZnYD74+65kmu5QfA\nC0Ca3G9Pl8/WDNAMVRGRKqQZqiIiVUjhLiJShRTuIiJVSOEuIlKFFO4iIlVI4S4iUoUU7iIiVUjh\nLiJShf4/qX3oU7NBe7YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1e87ff99cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "A=metrics.roc_curve(y_train,gs_lr.predict(x_train),pos_label=1)\n",
    "B=metrics.roc_curve(y_test,gs_lr.predict(x_test),pos_label=1)\n",
    "\n",
    "plt.plot(A[0],A[1],label='ROC NB Train')\n",
    "plt.plot(B[0],B[1],label='ROC NB Test')\n",
    "plt.xlim((-0.2,1.1))\n",
    "plt.ylim((-0.1,1.1))\n",
    "plt.legend(loc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = 'lr_model.sav'\n",
    "pickle.dump(gs_lr, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_pipe_rf = Pipeline([('tfidf', TfidfVectorizer()), ('rf', RandomForestClassifier())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tfidf', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,\n",
       " ...imators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_pipe_rf.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up grid search to find best parameters for TFIDF and RF\n",
    "rf_parameters = {'tfidf__max_df': (0.75, 0.80, 0.85, 0.90, 0.95, 1.0),\\\n",
    "              'tfidf__min_df': (0.001,0.01,0.1),\\\n",
    "              'rf__n_estimators': (50, 100, 200),\\\n",
    "                'rf__criterion': ('gini', 'entropy'),\\\n",
    "                'rf__class_weight': (None, 'balanced'),\\\n",
    "                'rf__min_samples_split': (10,20,30)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Establish grid search instance for log reg\n",
    "gs_rf = GridSearchCV(text_pipe_rf, rf_parameters, n_jobs=-1, scoring='f1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-28-4805c3722710>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'# Fit grid search to train data and find best parameters\\ngs_rf = gs_rf.fit(x_train, y_train)'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[1;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[0;32m   2113\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2114\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2115\u001b[1;33m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2116\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2117\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<decorator-gen-60>\u001b[0m in \u001b[0;36mtime\u001b[1;34m(self, line, cell, local_ns)\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\IPython\\core\\magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(f, *a, **k)\u001b[0m\n\u001b[0;32m    186\u001b[0m     \u001b[1;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    187\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 188\u001b[1;33m         \u001b[0mcall\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    189\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    190\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\IPython\\core\\magics\\execution.py\u001b[0m in \u001b[0;36mtime\u001b[1;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[0;32m   1183\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1184\u001b[0m             \u001b[0mst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1185\u001b[1;33m             \u001b[0mexec\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1186\u001b[0m             \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1187\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups)\u001b[0m\n\u001b[0;32m    943\u001b[0m             \u001b[0mtrain\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mtest\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    944\u001b[0m         \"\"\"\n\u001b[1;32m--> 945\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    946\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    947\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, groups, parameter_iterable)\u001b[0m\n\u001b[0;32m    562\u001b[0m                                   \u001b[0mreturn_times\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_parameters\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m                                   error_score=self.error_score)\n\u001b[1;32m--> 564\u001b[1;33m           \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mparameter_iterable\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    565\u001b[0m           for train, test in cv_iter)\n\u001b[0;32m    566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    766\u001b[0m                 \u001b[1;31m# consumption.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    767\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 768\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    769\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    770\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    717\u001b[0m                     \u001b[0mensure_ready\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_managed_backend\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    718\u001b[0m                     \u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mabort_everything\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mensure_ready\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mensure_ready\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 719\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    720\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    721\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    680\u001b[0m                 \u001b[1;31m# check if timeout supported in backend future implementation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    681\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[1;34m'timeout'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mgetfullargspec\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 682\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    683\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    684\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\multiprocessing\\pool.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    600\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    601\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 602\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    603\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mready\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    604\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\multiprocessing\\pool.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    597\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    598\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 599\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    600\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    601\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    547\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    548\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 549\u001b[1;33m                 \u001b[0msignaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    550\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    551\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\jgoldste\\Anaconda2\\envs\\py35\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    291\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 293\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    294\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    295\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# Fit grid search to train data and find best parameters\n",
    "gs_rf = gs_rf.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'rf__class_weight': 'balanced',\n",
       " 'rf__criterion': 'entropy',\n",
       " 'rf__n_estimators': 200,\n",
       " 'tfidf__max_df': 0.85,\n",
       " 'tfidf__min_df': 0.001}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get best parameters for log reg\n",
    "gs_rf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predict on test data \n",
    "rf_predicted = gs_rf.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.0927214376178\n",
      "Test score: 0.296447461285\n",
      "Confusion Matrix\n",
      " [[4582  133]\n",
      " [ 467 1741]]\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.91      0.97      0.94      4715\n",
      "          1       0.93      0.79      0.85      2208\n",
      "\n",
      "avg / total       0.91      0.91      0.91      6923\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.29644746128478461"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_model(gs_rf, y_test, rf_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 1.0\n",
      "Test score: 0.880144264758\n"
     ]
    }
   ],
   "source": [
    "train_loss = metrics.roc_auc_score(y_train, gs_rf.predict(x_train))\n",
    "test_loss = metrics.roc_auc_score(y_test, gs_rf.predict(x_test))\n",
    "print(\"Train score:\", train_loss)\n",
    "print(\"Test score:\", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1e34d23a470>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHf1JREFUeJzt3X1wVfW97/H3NwnhWYEEUQkxCAGNPESJUG2PLW2tYLXa\nXm8RtLYdTxlaOZx2nCpz7Vz7oA5tb611RBlvq/aZ6YNVewZr1etRO1ZrrFQICMRgNWgFgkDCQ0LI\n9/6xNmQnBLIS1t5r7c3nNcNk771+2fu7DHz85bv2/v3M3RERkfxSEHcBIiISPYW7iEgeUriLiOQh\nhbuISB5SuIuI5CGFu4hIHlK4i4jkIYW7iEgeUriLiOShorheuLS01CsqKuJ6eRGRnPTKK69sd/fR\nvY2LLdwrKiqora2N6+VFRHKSmf0zzDi1ZURE8pDCXUQkDyncRUTykMJdRCQPKdxFRPKQwl1EJA8p\n3EVE8pDCXUQkDyncRUTykMJdRCQPKdxFRPKQwl1EJA8p3EVE8lCv4W5mD5jZVjNbe5TjZmZ3m1m9\nmb1mZudFX6aIiPRFmJn7Q8CcYxyfC1Sm/iwE7jv+skRE5Hj0up67uz9nZhXHGHIF8DN3d+BFMxth\nZqe5+7sR1Zh43/pjHeve2R13GSISgyJv49T2dzh3xF6uvfb6uMs5LIrNOsYCb6fdb0w9dkS4m9lC\ngtk95eXlEby0iEgWuDOiYwentzdyensjpx1sPHz7lIPvUUAHHdsLof1zUFQcd7VAlndicvf7gfsB\nampqPJuvnUm3Xn5O3CWISBQO7Icdb8D2jbC9Hpo2wfZNsKMeWtN+Oy8aDCUTofQDUFIJpZUUlFZC\nQWF8tXcTRbhvAcal3S9LPSYikjzu0PxuENqHwvvQ7Z1vA2nzzpPGQmklTJsXfC2ZGHw9qQwKkv1m\nwyjC/TFgsZmtBGYBu06kfruIJFTbXmg6NPs+9HUjNL0BbS2d4wYMhZIJUHY+VF/TGeAlE6F4aHz1\nH6dew93Mfg18BCg1s0bgVmAAgLuvAFYBlwL1wF7gi5kqVkSki44OaH7nyDZKUz3sSr8UaHDyOCid\nCOM+EIR3aWXQUjnpdDCL7RQyJcy7Zeb3ctyBGyKrSESku9aW1Cy8PhXkqTZK0xtwYG/nuOJhQWiX\nXwCl13XOwkdNgOIh8dUfg6xeUBUROaqOjmC23b2Nsr0+mJ0fZjCiPAjtin9La6NUwvBT83IW3h8K\ndxHJrv27uwX4ps5Zefv+znEDTw7aKOMvCr6WTgoCfNSZMGBQfPXnCIW7iESv4yDsfKtbG6U++Nry\nr85xVgAjK4LQPvMjnbPw0kkwdLRm4cdB4S4i/bdvZ2dop7dRdjTAwdbOcYNGBKE98WNd2yijxkPR\nwPjqz2MKdxE5toPtsPOfXd8XfmhGvmdb5zgrDMK6pBIqP97ZRimthCElmoVnmcJdRAJ7d/TcRtnR\nAB0HOscNKQlCe9IlneFdOilorxQOiK186UrhLnIiOXgA3n/zyDZK0ybY29Q5rmBAMAsvnQST53a2\nUUorYcio2MqX8BTuIvnGPQjqntoo778JHe2dY4eODkL7rE92baOMOAMKFQ+5TD89kVzV3gbvbz6y\njdK0Cfa93zmusDj4EM8pZ8PZn+pso5RMhMEj4qtfMkrhLpJk7sFFy54WuXr/n+AHO8cOGxOEdtWV\naW2UicEsPEGrFUp2KNxFkuDA/uDCZfc2yvZ6aN3VOa5oUDALP3UqTPkfnQFeMhEGnRxf/ZI4CneR\nbHGHlvd6bqPsfAu8o3Ps8NOD0J56VddFrk4el/ilZiUZFO4iUTuwL1jQqnsbZXs9tDV3jisaHAT4\n6ecF64Wnz8IHDo+vfskLCneR/nCH3e/03EbZ1X3Dh7IgtKvnpwV4ZbARhGbhkiEKd5FjaduT1j6p\n7zoLP7Cnc9yAoam1wmfCudd0tlFKJuT0hg+SuxTuIh0dsHtLz22U3Y1pAw1GjAtC+7wLui5yNfw0\nfbxeEkXhLieO1ubU7LvbWuFN9dC+r3Nc8fBgFl7xwa5tlJIJMGBwfPWL9IHCXfJLx8Gg591lrfDU\n1+a0rX2tINjwoaQSxv9b14/XDxujWbjkPIW75KbDGz50a6PseKPrhg+DTu55rfCR47Xhg+Q1hbsk\nV8fB1FKz3dsom4L3ix9ihTDyjCC0J8zuOgvXhg9yglK4S/z2vd9DGyU1Cz/Y1jlu8MggtCd+vGuA\njxwPRcXx1S+SQAp3yY7DGz5s7BrgTZu6bvhQUBSEdWklVF6ctshVJQwtia9+kRyjcJdo7d3R81rh\nOzYfueFD6SSYNCdtFj4paK9owweR46Zwl747eCAI6+7vRtm+Cfbt6BxXMCDYqb50UrBe+KE2SslE\nbfggkmEKd+lZ+oYP2zd2baPs2Nx1qdmhpwShffblXdcK14YPIrHRv7wTXXtr11l4+kx8/87OcYXF\nQWCfUgVVV3SdhWvDB5HEUbifCNyhZWu3Ra5SM/Kd/+y61OywU4PQnvKZrgE+olwbPojkkFDhbmZz\ngB8BhcCP3X1Zt+MnA78AylPP+X/c/cGIa5XeHNgfvH2w+7tRetrwoWQinDY9tV54qo1SMhEGnRRf\n/SISmV7D3cwKgeXAxUAj8LKZPebu69KG3QCsc/fLzWw0sMHMfunubT08pRwPd2j+15HvRtme2vCh\ny1KzY4PAnvY/O2fhpZXBErRaalYkr4WZuc8E6t29AcDMVgJXAOnh7sBwMzNgGLADaO/+RNIHbXvT\nZuH1ae8Pf6Prhg8DhgQLWpXVwPT5nW2UkokwcFh89YtIrMKE+1jg7bT7jcCsbmPuAR4D3gGGA/Pc\n0xu50qNDGz5s33jkWuG73uo69uRxQWBXz+9so5RWBtuxaRYuIt1EdUH1EmA18FFgAvCkmT3v7rvT\nB5nZQmAhQHl5eUQvnQPSN3xIfzdK0xtdN3woHhaEdvksKLk2WGq2dFKwIXLxkPjqF5GcEybctwDj\n0u6XpR5L90Vgmbs7UG9mm4GzgL+lD3L3+4H7AWpqapwTwX99DWofSHsgteFD6SQ444Oda4WXVmrD\nBxGJTJhwfxmoNLPxBKF+NbCg25i3gI8Bz5vZGGAy0BBloTnrrZdgzBS46OtBgI86Uxs+iEjG9Rru\n7t5uZouBJwjeCvmAu9eZ2aLU8RXAd4CHzGwNYMDN7r49g3XnjrZmKL8Azrky7kpE5AQSqufu7quA\nVd0eW5F2+x3gE9GWlidaW4JeuohIFultFpnW1qK3JIpI1incM6m9Ldhsonh43JWIyAlG4Z5JbS3B\nV83cRSTLFO6Z1Jr6JKl67iKSZQr3TNLMXURionDPpNZUuKvnLiJZpnDPpEMLfGnmLiJZpnDPpMMz\nd4W7iGSXwj2T1HMXkZgo3DNJPXcRiYnCPZPUcxeRmCjcM6m1BQoGQNHAuCsRkROMwj2TtK6MiMRE\n4Z5JrS3qt4tILBTumaSZu4jEROGeSa3Neo+7iMRC4Z5JmrmLSEwU7pmkXZhEJCYK90xqa4GBuqAq\nItmncM8kzdxFJCYK90xxDz6hqp67iMRA4Z4pB/aBd2jmLiKxULhnyuEVIdVzF5HsU7hnivZPFZEY\nKdwzRWu5i0iMFO6Zol2YRCRGCvdMUc9dRGIUKtzNbI6ZbTCzejNbepQxHzGz1WZWZ2bPRltmDlLP\nXURiVNTbADMrBJYDFwONwMtm9pi7r0sbMwK4F5jj7m+Z2SmZKjhnqOcuIjEKM3OfCdS7e4O7twEr\ngSu6jVkAPOzubwG4+9Zoy8xB6rmLSIzChPtY4O20+42px9JNAkaa2X+b2Stmdl1UBeasNoW7iMSn\n17ZMH55nBvAxYDDwVzN70d03pg8ys4XAQoDy8vKIXjqhWpuhaDAURvWfWEQkvDAz9y3AuLT7ZanH\n0jUCT7j7HnffDjwHTO/+RO5+v7vXuHvN6NGj+1tzbtBa7iISozDh/jJQaWbjzawYuBp4rNuYR4EP\nmVmRmQ0BZgHroy01x2hFSBGJUa89A3dvN7PFwBNAIfCAu9eZ2aLU8RXuvt7M/gS8BnQAP3b3tZks\nPPE0cxeRGIVqCLv7KmBVt8dWdLv/feD70ZWW41pboFgfYBKReOgTqpmitdxFJEYK90xRz11EYqRw\nzxT13EUkRgr3TFHPXURipHDPhI4OOLBHM3cRiY3CPRO09ICIxEzhnglay11EYqZwz4RWhbuIxEvh\nnglt2qhDROKlcM+EVm3UISLxUrhngi6oikjMFO6ZoJ67iMRM4Z4J6rmLSMwU7pmgnruIxEzhnglt\nLWAFMGBI3JWIyAlK4Z4Jh1aENIu7EhE5QSncM6GtWf12EYmVwj0TWrXcr4jES+GeCW3aqENE4qVw\nzwTN3EUkZgr3TGjTRh0iEi+Feya0anNsEYmXwj0T1HMXkZgp3DNBPXcRiZnCPWoHD8DBVvXcRSRW\nCveotaYWDdPMXURipHCPmtZyF5EEULhHTStCikgChAp3M5tjZhvMrN7Mlh5j3Plm1m5mV0VXYo45\nPHNXz11E4tNruJtZIbAcmAtUAfPNrOoo474L/DnqInOKeu4ikgBhZu4zgXp3b3D3NmAlcEUP4/4D\n+D2wNcL6co967iKSAGHCfSzwdtr9xtRjh5nZWODTwH3HeiIzW2hmtWZWu23btr7WmhvUcxeRBIjq\ngupdwM3u3nGsQe5+v7vXuHvN6NGjI3rphFHPXUQSoCjEmC3AuLT7ZanH0tUAKy3YeagUuNTM2t39\nkUiqzCXquYtIAoQJ95eBSjMbTxDqVwML0ge4+/hDt83sIeC/TshghyDcCwZA0cC4KxGRE1iv4e7u\n7Wa2GHgCKAQecPc6M1uUOr4iwzXmljatKyMi8Qszc8fdVwGruj3WY6i7+xeOv6wc1qq13EUkfvqE\natQ0cxeRBFC4R621We9xF5HYKdyjppm7iCSAwj1qrdqFSUTip3CPWlsLDNQFVRGJl8I9apq5i0gC\nKNyj5A5tzeq5i0jsFO5ROrAPvEMzdxGJncI9SocWDVPPXURipnCP0qFFwzRzF5GYKdyj1Ka13EUk\nGRTuUWrVLkwikgwK9yip5y4iCaFwj5J67iKSEAr3KKnnLiIJoXCPknruIpIQCvcotSncRSQZFO5R\nam2GosFQGGqDKxGRjFG4R0lruYtIQijco6QVIUUkIRTuUdLMXUQSQuEepdYWGHhS3FWIiCjcI9Wm\nzbFFJBkU7lFqVVtGRJJB4R6lNl1QFZFkULhHqVWbY4tIMijco9LRAQf2aOYuIokQKtzNbI6ZbTCz\nejNb2sPxa8zsNTNbY2YvmNn06EtNOC0aJiIJ0mu4m1khsByYC1QB882sqtuwzcCH3X0q8B3g/qgL\nTTytKyMiCRJm5j4TqHf3BndvA1YCV6QPcPcX3P391N0XgbJoy8wBrdqoQ0SSI0y4jwXeTrvfmHrs\naK4HHu/pgJktNLNaM6vdtm1b+CpzQZs26hCR5Ij0gqqZzSYI95t7Ou7u97t7jbvXjB49OsqXjl+r\neu4ikhxh1qbdAoxLu1+WeqwLM5sG/BiY6+5N0ZSXQ9RzF5EECTNzfxmoNLPxZlYMXA08lj7AzMqB\nh4HPufvG6MvMAeq5i0iC9Dpzd/d2M1sMPAEUAg+4e52ZLUodXwH8b6AEuNfMANrdvSZzZSeQeu4i\nkiChtgxy91XAqm6PrUi7/e/Av0dbWo5Rz11EEkSfUI1KWwtYAQwYEnclIiIK98gc2oUpaEuJiMRK\n4R4VreUuIgmicI+K1nIXkQRRuEdFa7mLSIIo3KOimbuIJIjCPSptLVCsDzCJSDIo3KPS2qyZu4gk\nhsI9Kuq5i0iCKNyjop67iCSIwj0KBw/AwVb13EUkMUKtLSO9aE0tGqaZu+SYAwcO0NjYyP79++Mu\nRboZNGgQZWVlDBgwoF/fr3CPgtZylxzV2NjI8OHDqaiowLR0RmK4O01NTTQ2NjJ+/Ph+PYfaMlHQ\nipCSo/bv309JSYmCPWHMjJKSkuP6jUrhHoXDM3f13CX3KNiT6Xh/Lgr3KKjnLtJvhYWFVFdXM2XK\nFC6//HJ27tx5+FhdXR0f/ehHmTx5MpWVlXznO9/B3Q8ff/zxx6mpqaGqqopzzz2XG2+88Yjnf+ih\nhygoKOC11147/NiUKVN48803AaioqGDq1KlUV1czdepUHn300SOeY9asWVRXV1NeXs7o0aOprq6m\nurr68HOEccstt/DMM8+EHn/c3D2WPzNmzPC8UfeI+60nub+7Ju5KRPpk3bp1cZfgQ4cOPXz7uuuu\n89tuu83d3ffu3etnnnmmP/HEE+7uvmfPHp8zZ47fc8897u6+Zs0aP/PMM339+vXu7t7e3u733nvv\nEc//4IMP+rhx4/yzn/3s4cfOOecc37x5s7u7n3HGGb5t2zZ3d3/99de9vLz8qLU++OCDfsMNNxz1\neHt7e5hTDq2nnw9Q6yEyVjP3KKjnLhKJCy64gC1btgDwq1/9ig9+8IN84hOfAGDIkCHcc889LFu2\nDIDvfe973HLLLZx11llA8BvAl7/85R6f97LLLqOuro4NGzYc8/V3797NyJEjQ9fb3t7OiBEj+OpX\nv8q0adP429/+xq233sr555/PlClTWLRo0eHfNK699loeeeQRAMrKyvjmN7/Jueeey7Rp09i4Mfqt\np/VumSio5y554Ft/rGPdO7sjfc6q00/i1svPCTX24MGDPP3001x//fVA0JKZMWNGlzETJkygpaWF\n3bt3s3bt2h7bMD0pKCjgpptu4o477uCnP/3pEcdnz56Nu9PQ0MBvfvObUM95yK5du7jooou46667\nAJg8eTLf+ta3cHcWLFjAn/70J+bOnXvE940ZM4ZXX32Vu+++mzvvvJMVK1YcMeZ4aOYeBfXcRfpt\n3759VFdXc+qpp/Lee+9x8cUXZ+R1FixYwIsvvsjmzZuPOPbMM8+wdu1a1qxZw+LFi2lpaQn9vMXF\nxXz6058+fP/pp59m5syZTJ8+nWeffZa6uroev+8zn/kMADNmzOhT7z4szdyj0NYCBQOgaGDclYj0\nW9gZdtQGDx7M6tWr2bt3L5dccgnLly9nyZIlVFVV8dxzz3UZ29DQwLBhwzjppJM455xzeOWVV5g+\nfXqo1ykqKuLGG2/ku9/97lHHTJgwgTFjxrBu3TpmzpwZuv5D72zZu3cvixcv5u9//ztjx47lG9/4\nxlHfzjhwYJAXhYWFtLe3h3qtvtDMPQpaV0bkuA0ZMoS7776bH/zgB7S3t3PNNdfwl7/8haeeegoI\nZvhLlizhpptuAuDrX/86d9xxx+F+dUdHR6+tjS984Qs89dRTbNu2rcfjW7duZfPmzZxxxhn9Ood9\n+/ZRUFBAaWkpzc3N/P73v+/X80RB4R4FreUuEolDFxh//etfM3jwYB599FFuu+02Jk+ezNSpUzn/\n/PNZvHgxANOmTeOuu+5i/vz5nH322UyZMoWGhoZjPn9xcTFLlixh69atXR6fPXs21dXVzJ49m2XL\nljFmzJh+1V9SUsLnP/95qqqqmDt3LrNmzerX80TBDl3Jzbaamhqvra2N5bUjt/Ia2NEAX/lr3JWI\n9Mn69es5++yz4y5DjqKnn4+ZveLuNb19r2buUdBa7iKSMAr3KKjnLiIJo3CPgmbuIpIwocLdzOaY\n2QYzqzezpT0cNzO7O3X8NTM7L/pSE6y1BQbqgqqIJEev4W5mhcByYC5QBcw3s6puw+YClak/C4H7\nIq4z2dqaNXMXkUQJM3OfCdS7e4O7twErgSu6jbkC+FlqXZsXgRFmdlrEtSaTu3ruIpI4YcJ9LPB2\n2v3G1GN9HZOf2veDH9TMXaSfTpQlfwEefvhhXn/99T59T39l9YKqmS00s1ozqz3aJ8RyjhXA3O/D\nhNlxVyKSkw4tP7B27VpGjRrF8uXLgeDTnp/61KdYunQpGzZs4B//+AcvvPAC9957LwBr165l8eLF\n/OIXv2DdunXU1tYyceLEHl+jrKyM22+//ag1PPPMM6xevZrf/e53LFmy5IjjL730EqtXr+bb3/42\n8+bNY/Xq1axevZqKioo+nWvSwn0LMC7tflnqsb6Owd3vd/cad68ZPXp0X2tNpqKBMGshnH5u3JWI\n5LxcW/IXgt8eLrjgAs477zzmzZvHnj17gGB5hKqqKqZNm8bNN9/M888/z6pVq/ja177Wr1l/X4VZ\nOOxloNLMxhME9tXAgm5jHgMWm9lKYBawy93fjbRSEcmsx5fCv9ZE+5ynToW5y0INzcUlf7du3cqy\nZct4+umnGTJkCLfffjs/+tGPuP7661m1ahV1dXWYGTt37mTEiBFceumlXHXVVVx55ZWhX6O/ep25\nu3s7sBh4AlgP/Mbd68xskZktSg1bBTQA9cD/Bb6SoXpFJM/k8pK/L7zwAuvWrePCCy+kurqaX/7y\nl7z55puMGjWKgoICvvSlL/GHP/yBoUOHRn06vQq15K+7ryII8PTHVqTdduCGaEsTkawKOcOOWi4v\n+evuzJkzh5///OdHHKutreXJJ5/kt7/9Lffddx9//vOfQ9UZFX1CVUQSIReX/L3wwgt59tlnD69G\nuWfPHjZt2kRzczO7d+/msssu44c//CGvvvoqAMOHD6e5uTnUcx8vhbuIJEauLfk7ZswYfvKTnzBv\n3jymT5/OhRdeyMaNG9m1axef/OQnmT59Oh/+8Ie58847AZg/fz533HFHVi6oaslfkROYlvxNNi35\nKyIiXSjcRUTykMJdRCQPKdxFTnBxXXeTYzven4vCXeQENmjQIJqamhTwCePuNDU1MWjQoH4/R6gP\nMYlIfiorK6OxsfGo7/uW+AwaNIiysrJ+f7/CXeQENmDAAMaPHx93GZIBasuIiOQhhbuISB5SuIuI\n5KHYlh8ws2bg2Cvn55ZSYHvcRUQkn84F8ut88ulcIL/OJ1vncoa797rbUZwXVDeEWR8hV5hZbb6c\nTz6dC+TX+eTTuUB+nU/SzkVtGRGRPKRwFxHJQ3GG+/0xvnYm5NP55NO5QH6dTz6dC+TX+STqXGK7\noCoiIpmjtoyISB7KWrib2Sgze9LMNqW+juxhzDgze8bM1plZnZn9Z7bqC8PM5pjZBjOrN7OlPRw3\nM7s7dfw1MzsvjjrDCnE+16TOY42ZvWBm4XYijkFv55I27nwzazezq7JZX1+FOR8z+4iZrU79W3k2\n2zWGFeLv2clm9kcz+0fqXL4YR51hmNkDZrbVzNYe5XhyMsDds/IH+B6wNHV7KfDdHsacBpyXuj0c\n2AhUZavGXuovBN4AzgSKgX90rw24FHgcMOADwEtx132c53MhMDJ1e25SzyfMuaSN+3/AKuCquOs+\nzp/NCGAdUJ66f0rcdR/HufyvQ3kAjAZ2AMVx136U87kIOA9Ye5TjicmAbLZlrgB+mrr9U+DK7gPc\n/V13/3vqdjOwHhibtQqPbSZQ7+4N7t4GrCQ4p3RXAD/zwIvACDM7LduFhtTr+bj7C+7+furui0D/\nl6jLrDA/G4D/AH4PbO3hWJKEOZ8FwMPu/haAuyf1nMKciwPDzcyAYQTh3p7dMsNx9+cI6juaxGRA\nNsN9jLu/m7r9L+CY24ubWQVwLvBSZssKbSzwdtr9Ro78H0+YMUnR11qvJ5iRJFGv52JmY4FPA/dl\nsa7+CvOzmQSMNLP/NrNXzOy6rFXXN2HO5R7gbOAdYA3wn+7ekZ3yIpeYDIj0E6pm9hRwag+Hbkm/\n4+5uZkd9m46ZDSOYYX3V3XdHWaP0nZnNJgj3D8Vdy3G4C7jZ3TuCCWLOKwJmAB8DBgN/NbMX3X1j\nvGX1yyXAauCjwATgSTN7Xv/2j0+k4e7uHz/aMTN7z8xOc/d3U7+m9PhrpJkNIAj2X7r7w1HWd5y2\nAOPS7pelHuvrmKQIVauZTQN+DMx196Ys1dZXYc6lBliZCvZS4FIza3f3R7JTYp+EOZ9GoMnd9wB7\nzOw5YDrBdaokCXMuXwSWedC0rjezzcBZwN+yU2KkEpMB2WzLPAZ8PnX788Cj3Qekem4/Ada7+51Z\nrC2Ml4FKMxtvZsXA1QTnlO4x4LrUFfMPALvSWlFJ0+v5mFk58DDwuYTPCHs9F3cf7+4V7l4B/A74\nSkKDHcL9XXsU+JCZFZnZEGAWwTWqpAlzLm8R/AaCmY0BJgMNWa0yOsnJgCxeZS4BngY2AU8Bo1KP\nnw6sSt3+EMHFldcIfk1bDVwa19XmHs7hUoKZ0RvALanHFgGLUrcNWJ46vgaoibvm4zyfHwPvp/0s\nauOuub/n0m3sQyT43TJhzwf4OsE7ZtYStDBjr7uff89OB/6c+jezFrg27pqPcS6/Bt4FDhD89nR9\nUjNAn1AVEclD+oSqiEgeUriLiOQhhbuISB5SuIuI5CGFu4hIHlK4i4jkIYW7iEgeUriLiOSh/w/f\nCUGklsV1AwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1e34d479a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "A=metrics.roc_curve(y_train,gs_rf.predict(x_train),pos_label=1)\n",
    "B=metrics.roc_curve(y_test,gs_rf.predict(x_test),pos_label=1)\n",
    "\n",
    "plt.plot(A[0],A[1],label='ROC NB Train')\n",
    "plt.plot(B[0],B[1],label='ROC NB Test')\n",
    "plt.xlim((-0.2,1.1))\n",
    "plt.ylim((-0.1,1.1))\n",
    "plt.legend(loc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = 'rf_model.sav'\n",
    "pickle.dump(gs_rf, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_pipe_svm = Pipeline([('tfidf', TfidfVectorizer()), ('svm', SVC())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_pipe_svm.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up grid search to find best parameters for TFIDF and SVM\n",
    "svm_parameters = {'tfidf__max_df': (0.75, 0.80, 0.85, 0.90, 0.95, 1.0),\\\n",
    "              'tfidf__min_df': (0.001,0.01,0.1),\\\n",
    "              'svm__C': (1, 10, 100, 1000),\\\n",
    "                'svm__degree': (1 , 2, 3),\\\n",
    "                'svm__class_weight': (None, 'balanced')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Establish grid search instance for SVM\n",
    "gs_svm = GridSearchCV(text_pipe_svm, svm_parameters, n_jobs=-1, scoring='f1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time \n",
    "# Fit grid search to train data and find best parameters\n",
    "gs_svm = gs_svm.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get best parameters for SVM\n",
    "gs_svm.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predict on test data \n",
    "svm_predicted = gs_svm.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluate_model(gs_svm, y_test, svm_predicted)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}