Dev_Ram_portland_NAICS_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Naive Bayes\n",
    "2. Logistic Regression\n",
    "3. Ada Boosting\n",
    "4. XG Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_2017 = pd.read_csv(\"Portland_NAICS_2017.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(338, 25)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_2017.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>geo_id</th>\n",
       "      <th>year</th>\n",
       "      <th>pop</th>\n",
       "      <th>households</th>\n",
       "      <th>income</th>\n",
       "      <th>edut</th>\n",
       "      <th>total_college</th>\n",
       "      <th>college_ratio</th>\n",
       "      <th>ratio_65</th>\n",
       "      <th>ratio_22_50</th>\n",
       "      <th>college_ratio_pop</th>\n",
       "      <th>bus_ratio</th>\n",
       "      <th>bus_ratio_total</th>\n",
       "      <th>public_ratio</th>\n",
       "      <th>car_ratio</th>\n",
       "      <th>bicy_ratio</th>\n",
       "      <th>walked_ratio</th>\n",
       "      <th>maxNaics_est</th>\n",
       "      <th>maxNaics_emp</th>\n",
       "      <th>Empratio</th>\n",
       "      <th>estabratio</th>\n",
       "      <th>emp_label</th>\n",
       "      <th>est_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>97009</td>\n",
       "      <td>41051010402</td>\n",
       "      <td>2017</td>\n",
       "      <td>6981.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>2303.0</td>\n",
       "      <td>4633.0</td>\n",
       "      <td>884.0</td>\n",
       "      <td>0.190805</td>\n",
       "      <td>0.355966</td>\n",
       "      <td>0.133792</td>\n",
       "      <td>0.126629</td>\n",
       "      <td>0.867188</td>\n",
       "      <td>0.032118</td>\n",
       "      <td>0.037037</td>\n",
       "      <td>0.882523</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.035880</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Low</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1352</td>\n",
       "      <td>97014</td>\n",
       "      <td>41051010500</td>\n",
       "      <td>2017</td>\n",
       "      <td>3912.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>1466.0</td>\n",
       "      <td>2732.0</td>\n",
       "      <td>551.0</td>\n",
       "      <td>0.201684</td>\n",
       "      <td>0.224949</td>\n",
       "      <td>0.214468</td>\n",
       "      <td>0.140849</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037785</td>\n",
       "      <td>0.833876</td>\n",
       "      <td>0.005863</td>\n",
       "      <td>0.016938</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>63.888889</td>\n",
       "      <td>6.388889</td>\n",
       "      <td>Low</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1551</td>\n",
       "      <td>97019</td>\n",
       "      <td>41051010500</td>\n",
       "      <td>2017</td>\n",
       "      <td>3912.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>1466.0</td>\n",
       "      <td>2732.0</td>\n",
       "      <td>551.0</td>\n",
       "      <td>0.201684</td>\n",
       "      <td>0.224949</td>\n",
       "      <td>0.214468</td>\n",
       "      <td>0.140849</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037785</td>\n",
       "      <td>0.833876</td>\n",
       "      <td>0.005863</td>\n",
       "      <td>0.016938</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>292.622951</td>\n",
       "      <td>29.262295</td>\n",
       "      <td>Medium Low</td>\n",
       "      <td>Medium High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1608</td>\n",
       "      <td>97024</td>\n",
       "      <td>41051009603</td>\n",
       "      <td>2017</td>\n",
       "      <td>4044.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>1229.0</td>\n",
       "      <td>2533.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>0.116857</td>\n",
       "      <td>0.370425</td>\n",
       "      <td>0.116716</td>\n",
       "      <td>0.073195</td>\n",
       "      <td>0.792683</td>\n",
       "      <td>0.036806</td>\n",
       "      <td>0.046433</td>\n",
       "      <td>0.852208</td>\n",
       "      <td>0.024915</td>\n",
       "      <td>0.018120</td>\n",
       "      <td>23</td>\n",
       "      <td>42</td>\n",
       "      <td>2.129630</td>\n",
       "      <td>0.190741</td>\n",
       "      <td>Low</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1610</td>\n",
       "      <td>97024</td>\n",
       "      <td>41051010100</td>\n",
       "      <td>2017</td>\n",
       "      <td>8755.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>3728.0</td>\n",
       "      <td>6112.0</td>\n",
       "      <td>914.0</td>\n",
       "      <td>0.149542</td>\n",
       "      <td>0.382524</td>\n",
       "      <td>0.145060</td>\n",
       "      <td>0.104397</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.081324</td>\n",
       "      <td>0.081324</td>\n",
       "      <td>0.879169</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009492</td>\n",
       "      <td>23</td>\n",
       "      <td>42</td>\n",
       "      <td>920.000000</td>\n",
       "      <td>82.400000</td>\n",
       "      <td>Medium High</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  zipcode       geo_id  year     pop  households  income    edut  \\\n",
       "0           7    97009  41051010402  2017  6981.0       115.0  2303.0  4633.0   \n",
       "1        1352    97014  41051010500  2017  3912.0       122.0  1466.0  2732.0   \n",
       "2        1551    97019  41051010500  2017  3912.0       122.0  1466.0  2732.0   \n",
       "3        1608    97024  41051009603  2017  4044.0       119.0  1229.0  2533.0   \n",
       "4        1610    97024  41051010100  2017  8755.0       190.0  3728.0  6112.0   \n",
       "\n",
       "   total_college  college_ratio  ratio_65  ratio_22_50  college_ratio_pop  \\\n",
       "0          884.0       0.190805  0.355966     0.133792           0.126629   \n",
       "1          551.0       0.201684  0.224949     0.214468           0.140849   \n",
       "2          551.0       0.201684  0.224949     0.214468           0.140849   \n",
       "3          296.0       0.116857  0.370425     0.116716           0.073195   \n",
       "4          914.0       0.149542  0.382524     0.145060           0.104397   \n",
       "\n",
       "   bus_ratio  bus_ratio_total  public_ratio  car_ratio  bicy_ratio  \\\n",
       "0   0.867188         0.032118      0.037037   0.882523    0.000000   \n",
       "1   0.000000         0.000000      0.037785   0.833876    0.005863   \n",
       "2   0.000000         0.000000      0.037785   0.833876    0.005863   \n",
       "3   0.792683         0.036806      0.046433   0.852208    0.024915   \n",
       "4   1.000000         0.081324      0.081324   0.879169    0.000000   \n",
       "\n",
       "   walked_ratio  maxNaics_est  maxNaics_emp    Empratio  estabratio  \\\n",
       "0      0.035880            23            23    0.000000    0.000000   \n",
       "1      0.016938            42            42   63.888889    6.388889   \n",
       "2      0.016938            23            23  292.622951   29.262295   \n",
       "3      0.018120            23            42    2.129630    0.190741   \n",
       "4      0.009492            23            42  920.000000   82.400000   \n",
       "\n",
       "     emp_label    est_label  \n",
       "0          Low          Low  \n",
       "1          Low          Low  \n",
       "2   Medium Low  Medium High  \n",
       "3          Low          Low  \n",
       "4  Medium High         High  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_2017.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_2017 = data_2017.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(336, 25)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_2017.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_2016 = pd.read_csv(\"Portland_NAICS_before_2017.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2376, 25)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_2016.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_2016 = data_2016.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2368, 25)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_2016.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NAIVE BAYES\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = data_2016.loc[ : , \"pop\":\"walked_ratio\"]\n",
    "y_train = data_2016.loc[ : , \"maxNaics_est\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = data_2017.loc[ : , \"pop\":\"walked_ratio\"]\n",
    "y_test = data_2017.loc[ : , \"maxNaics_est\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using the GridSearchCV to find optimal estimators and leaf nodes\n",
    "param_grid = {'max_depth':range(1,51), 'max_leaf_nodes':range(2,51)}\n",
    "dt = DecisionTreeClassifier()\n",
    "gr = GridSearchCV(dt, param_grid=param_grid, scoring='accuracy')\n",
    "rs = gr.fit(X_train, y_train_emp)\n",
    "print(rs.best_params_)\n",
    "print(rs.score(X_test, y_test_emp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb = GaussianNB()\n",
    "y_pred = gnb.fit(X_train, y_train).predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 336 points : 190\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of mislabeled points out of a total %d points : %d\"\n",
    "...       % (X_test.shape[0], (y_test != y_pred).sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      23\n",
      "1      42\n",
      "2      23\n",
      "3      23\n",
      "4      23\n",
      "       ..\n",
      "333    72\n",
      "334    72\n",
      "335    72\n",
      "336    72\n",
      "337    72\n",
      "Name: maxNaics_est, Length: 336, dtype: int64 [23 23 23 23 23 23 23 62 23 23 23 23 62 23 23 23 23 23 23 44 23 54 54 54\n",
      " 54 54 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 23 54 54\n",
      " 54 54 54 54 54 54 81 54 54 62 54 62 62 72 72 54 54 72 54 54 72 54 72 54\n",
      " 62 72 62 62 62 62 72 44 54 54 54 54 54 62 72 72 72 62 72 72 72 54 54 72\n",
      " 62 44 62 44 81 54 54 54 54 81 54 54 23 54 54 54 54 54 54 54 54 62 54 54\n",
      " 54 72 72 72 54 72 62 72 62 72 44 42 72 54 72 54 54 54 54 54 62 54 54 54\n",
      " 72 54 62 62 54 54 62 72 62 62 62 54 62 54 72 54 54 54 72 54 54 54 54 54\n",
      " 72 62 62 62 54 54 62 62 62 62 62 23 62 44 72 54 54 54 62 72 72 72 54 62\n",
      " 72 23 44 54 62 54 72 42 44 62 44 81 54 54 54 54 72 54 62 72 54 62 54 72\n",
      " 62 62 62 62 62 42 44 23 62 62 23 44 62 23 54 54 72 54 62 54 54 62 44 62\n",
      " 54 72 54 54 54 23 62 42 62 44 62 62 23 23 23 23 23 62 23 62 23 23 72 72\n",
      " 72 72 23 23 23 23 62 62 62 62 54 54 54 54 54 54 54 54 54 54 54 62 62 23\n",
      " 44 62 23 23 62 62 53 23 62 44 23 44 44 23 23 53 23 23 44 53 23 23 44 23\n",
      " 54 81 54 54 54 62 72 62 62 62 62 72 72 62 23 62 44 44 44 44 44 23 62 23]\n"
     ]
    }
   ],
   "source": [
    "print(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.43452380952380953"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnb.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:667: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), UserWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Solver newton-cg supports only 'l2' or 'none' penalties, got l1 penalty.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Solver sag supports only 'l2' or 'none' penalties, got l1 penalty.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:466: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:314: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:466: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:314: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:466: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:314: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:466: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:314: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:466: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:314: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:212: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Solver newton-cg supports only 'l2' or 'none' penalties, got elasticnet penalty.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got elasticnet penalty.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Only 'saga' solver supports elasticnet penalty, got solver=liblinear.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: Solver sag supports only 'l2' or 'none' penalties, got elasticnet penalty.\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "ValueError: l1_ratio must be between 0 and 1; got (l1_ratio=None)\n",
      "\n",
      "  FitFailedWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:466: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:314: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:426: LineSearchWarning: Rounding errors prevent the line search from converging\n",
      "  warn(msg, LineSearchWarning)\n",
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:204: UserWarning: Line Search failed\n",
      "  warnings.warn('Line Search failed')\n"
     ]
    }
   ],
   "source": [
    "#Using the GridSearchCV to find optimal estimators and leaf nodes\n",
    "param_grid = {'penalty':['l1', 'l2', 'elasticnet'], 'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'], \n",
    "             'max_iter':[1000]}\n",
    "clf = LogisticRegression()\n",
    "gr = GridSearchCV(clf, param_grid=param_grid, scoring='accuracy')\n",
    "rs = gr.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_iter': 1000, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
      "0.5089285714285714\n"
     ]
    }
   ],
   "source": [
    "print(rs.best_params_)\n",
    "print(rs.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adaptive Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:667: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), UserWarning)\n"
     ]
    }
   ],
   "source": [
    "#Using the GridSearchCV to find optimal estimators and leaf nodes\n",
    "param_grid = {'n_estimators':[50, 100, 150, 200]}\n",
    "clf = AdaBoostClassifier()\n",
    "gr = GridSearchCV(clf, param_grid=param_grid, scoring='accuracy')\n",
    "rs = gr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': 50}\n",
      "0.3958333333333333\n"
     ]
    }
   ],
   "source": [
    "print(rs.best_params_)\n",
    "print(rs.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XG Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ram\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:667: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), UserWarning)\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'n_estimators':[50, 100, 150, 200], 'max_depth':[5,10,25,50,100]}\n",
    "clf = XGBClassifier()\n",
    "gr = GridSearchCV(clf, param_grid=param_grid, scoring='accuracy')\n",
    "rs = gr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 10, 'n_estimators': 150}\n",
      "0.5625\n"
     ]
    }
   ],
   "source": [
    "print(rs.best_params_)\n",
    "print(rs.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}