roc_auc_ml_classification_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  ROC (Receiver Operating Characteristic )  Curve\n",
    "# AUC (Area Under the Curve ) Score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(569, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst texture  worst perimeter  worst area  \\\n",
       "0                 0.07871  ...          17.33           184.60      2019.0   \n",
       "1                 0.05667  ...          23.41           158.80      1956.0   \n",
       "2                 0.05999  ...          25.53           152.50      1709.0   \n",
       "3                 0.09744  ...          26.50            98.87       567.7   \n",
       "4                 0.05883  ...          16.67           152.20      1575.0   \n",
       "\n",
       "   worst smoothness  worst compactness  worst concavity  worst concave points  \\\n",
       "0            0.1622             0.6656           0.7119                0.2654   \n",
       "1            0.1238             0.1866           0.2416                0.1860   \n",
       "2            0.1444             0.4245           0.4504                0.2430   \n",
       "3            0.2098             0.8663           0.6869                0.2575   \n",
       "4            0.1374             0.2050           0.4000                0.1625   \n",
       "\n",
       "   worst symmetry  worst fractal dimension  outcome  \n",
       "0          0.4601                  0.11890        1  \n",
       "1          0.2750                  0.08902        1  \n",
       "2          0.3613                  0.08758        1  \n",
       "3          0.6638                  0.17300        1  \n",
       "4          0.2364                  0.07678        1  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv('breast_cancer.csv')\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "smoothness error          -0.067016\n",
       "mean fractal dimension    -0.012838\n",
       "texture error             -0.008303\n",
       "symmetry error            -0.006522\n",
       "fractal dimension error    0.077972\n",
       "concavity error            0.253730\n",
       "compactness error          0.292999\n",
       "worst fractal dimension    0.323872\n",
       "mean symmetry              0.330499\n",
       "mean smoothness            0.358560\n",
       "concave points error       0.408042\n",
       "mean texture               0.415185\n",
       "worst symmetry             0.416294\n",
       "worst smoothness           0.421465\n",
       "worst texture              0.456903\n",
       "area error                 0.548236\n",
       "perimeter error            0.556141\n",
       "radius error               0.567134\n",
       "worst compactness          0.590998\n",
       "mean compactness           0.596534\n",
       "worst concavity            0.659610\n",
       "mean concavity             0.696360\n",
       "mean area                  0.708984\n",
       "mean radius                0.730029\n",
       "worst area                 0.733825\n",
       "mean perimeter             0.742636\n",
       "worst radius               0.776454\n",
       "mean concave points        0.776614\n",
       "worst perimeter            0.782914\n",
       "worst concave points       0.793566\n",
       "outcome                    1.000000\n",
       "Name: outcome, dtype: float64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.corr().outcome.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst radius</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>25.38</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>24.99</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>23.57</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>22.54</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst radius  worst texture  worst perimeter  \\\n",
       "0                 0.07871  ...         25.38          17.33           184.60   \n",
       "1                 0.05667  ...         24.99          23.41           158.80   \n",
       "2                 0.05999  ...         23.57          25.53           152.50   \n",
       "3                 0.09744  ...         14.91          26.50            98.87   \n",
       "4                 0.05883  ...         22.54          16.67           152.20   \n",
       "\n",
       "   worst area  worst smoothness  worst compactness  worst concavity  \\\n",
       "0      2019.0            0.1622             0.6656           0.7119   \n",
       "1      1956.0            0.1238             0.1866           0.2416   \n",
       "2      1709.0            0.1444             0.4245           0.4504   \n",
       "3       567.7            0.2098             0.8663           0.6869   \n",
       "4      1575.0            0.1374             0.2050           0.4000   \n",
       "\n",
       "   worst concave points  worst symmetry  worst fractal dimension  \n",
       "0                0.2654          0.4601                  0.11890  \n",
       "1                0.1860          0.2750                  0.08902  \n",
       "2                0.2430          0.3613                  0.08758  \n",
       "3                0.2575          0.6638                  0.17300  \n",
       "4                0.1625          0.2364                  0.07678  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = data.iloc[:,:-1]\n",
    "y = data.outcome\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    1\n",
       "3    1\n",
       "4    1\n",
       "Name: outcome, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3,random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ashokveda/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "model = LogisticRegression(C=100)\n",
    "model.fit(X_train,y_train)\n",
    "y_predict= model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9532163742690059\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outcome</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>106</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0      0   1\n",
       "outcome         \n",
       "0        106   6\n",
       "1          2  57"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(y_test,y_predict))\n",
    "pd.crosstab(y_test,y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.98995300e-01, 2.34298465e-05, 8.56930664e-05, 9.99999742e-01,\n",
       "       4.67891673e-04, 3.17454707e-01, 2.49781497e-05, 3.68270134e-01,\n",
       "       9.99999972e-01, 9.80798039e-01, 1.23315150e-05, 1.19532548e-05,\n",
       "       9.66618325e-01, 4.83016826e-05, 9.99974499e-01, 9.99552792e-01,\n",
       "       2.78116705e-04, 3.89930098e-02, 1.10381411e-02, 9.57174253e-01,\n",
       "       9.90681339e-01, 1.99273192e-01, 7.82507079e-03, 2.09677032e-03,\n",
       "       1.00000000e+00, 1.00000000e+00, 1.94285397e-01, 1.00000000e+00,\n",
       "       6.89321741e-05, 3.29788376e-01, 1.74556886e-04, 9.46851631e-04,\n",
       "       1.00000000e+00, 1.73876604e-04, 3.17678229e-02, 3.84216885e-05,\n",
       "       8.16849576e-04, 1.77543216e-06, 1.48393313e-04, 2.70525126e-03,\n",
       "       3.52587098e-05, 8.96489228e-05, 9.48050502e-02, 9.95748778e-01,\n",
       "       4.40614315e-04, 1.00000000e+00, 9.98595526e-01, 9.99966331e-01,\n",
       "       9.99176796e-01, 8.24312783e-01, 2.99019915e-02, 5.38044846e-03,\n",
       "       9.99986271e-01, 4.75431429e-04, 4.04273956e-03, 1.00000000e+00,\n",
       "       9.99997702e-01, 4.70531990e-05, 6.50085272e-02, 8.50861136e-07,\n",
       "       1.00000000e+00, 3.79023716e-05, 9.98531397e-01, 6.58811425e-01,\n",
       "       1.67293087e-05, 9.73315858e-01, 4.12149037e-02, 4.64410138e-07,\n",
       "       9.99928135e-01, 9.99922656e-01, 4.98502340e-06, 1.28890317e-05,\n",
       "       1.35763129e-03, 8.34703019e-06, 1.50768247e-03, 9.97898817e-01,\n",
       "       4.12519909e-04, 1.21324135e-03, 9.99998177e-01, 9.99794248e-01,\n",
       "       1.30848841e-04, 1.43457039e-03, 1.00000000e+00, 3.01163688e-03,\n",
       "       1.72454346e-06, 2.99771312e-04, 3.06024022e-02, 9.21845284e-01,\n",
       "       4.28479877e-04, 9.05093330e-01, 1.11982483e-03, 1.18273473e-02,\n",
       "       9.99996279e-01, 5.58009114e-03, 3.13710815e-06, 2.03895848e-04,\n",
       "       9.99999999e-01, 1.57243039e-04, 1.00000000e+00, 5.64993231e-04,\n",
       "       1.00000000e+00, 9.99962964e-01, 9.71353160e-09, 2.45647269e-06,\n",
       "       7.33657246e-05, 6.67858159e-07, 9.99999999e-01, 1.00000000e+00,\n",
       "       4.39018154e-05, 1.00578066e-04, 5.38163970e-05, 5.11442974e-03,\n",
       "       1.00000000e+00, 9.99999923e-01, 1.00000000e+00, 1.02061912e-05,\n",
       "       1.07087905e-01, 1.00000000e+00, 9.99569301e-01, 1.24802712e-04,\n",
       "       2.98180727e-03, 9.99999999e-01, 3.07589073e-04, 1.00000000e+00,\n",
       "       1.00074559e-07, 2.52461114e-02, 1.00000000e+00, 9.99716202e-01,\n",
       "       1.66528019e-04, 1.01098413e-03, 2.44712229e-02, 1.00559899e-04,\n",
       "       9.50842470e-01, 2.25077204e-06, 6.17131163e-05, 9.99999999e-01,\n",
       "       9.31101291e-01, 1.00000000e+00, 8.70914222e-02, 9.99999999e-01,\n",
       "       9.14401646e-04, 6.10636497e-06, 1.00000000e+00, 2.34274541e-05,\n",
       "       2.15540160e-05, 1.00000000e+00, 5.16372922e-05, 1.05252011e-03,\n",
       "       8.17159370e-03, 3.61693544e-06, 9.99962778e-01, 6.05044810e-06,\n",
       "       1.00000000e+00, 2.79218188e-01, 4.07430928e-06, 1.65318636e-03,\n",
       "       1.00000000e+00, 2.67743059e-04, 4.20001778e-04, 9.99947199e-01,\n",
       "       5.60371804e-03, 1.53358388e-05, 8.05252052e-05, 2.38456623e-04,\n",
       "       7.10846569e-04, 9.99999560e-01, 1.00000000e+00, 9.59553424e-03,\n",
       "       1.69013511e-07, 7.31447525e-04, 2.13048840e-03])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_proba(X_test)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "y_predict_th3 = np.where(model.predict_proba(X_test)[:,1]>0.3,1,0) # Threshold is 0.3\n",
    "y_predict_th4 = np.where(model.predict_proba(X_test)[:,1]>0.4,1,0) # Threshold is 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outcome</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>106</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0      0   1\n",
       "outcome         \n",
       "0        106   6\n",
       "1          2  57"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(y_test,y_predict_th4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_threshold (model,X_test,thresholds):\n",
    "    import numpy as np\n",
    "    return np.where(model.predict_proba(X_test)[:,1]>thresholds,1,0) # Threshold is 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold : 0.0\n",
      "[[  0 112]\n",
      " [  0  59]]\n",
      "Threshold : 0.1\n",
      "[[101  11]\n",
      " [  0  59]]\n",
      "Threshold : 0.2\n",
      "[[102  10]\n",
      " [  2  57]]\n",
      "Threshold : 0.30000000000000004\n",
      "[[103   9]\n",
      " [  2  57]]\n",
      "Threshold : 0.4\n",
      "[[106   6]\n",
      " [  2  57]]\n",
      "Threshold : 0.5\n",
      "[[106   6]\n",
      " [  2  57]]\n",
      "Threshold : 0.6000000000000001\n",
      "[[106   6]\n",
      " [  2  57]]\n",
      "Threshold : 0.7000000000000001\n",
      "[[106   6]\n",
      " [  3  56]]\n",
      "Threshold : 0.8\n",
      "[[106   6]\n",
      " [  3  56]]\n",
      "Threshold : 0.9\n",
      "[[107   5]\n",
      " [  3  56]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "for thr in np.arange(0,1.0,0.1):\n",
    "    y_predict = predict_threshold(model,X_test,thr)\n",
    "    print(\"Threshold :\",thr)\n",
    "    print(confusion_matrix(y_test,y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tpr,fpr,thresholds = roc_curve(y_test,model.predict_proba(X_test)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.00000000e+00, 1.00000000e+00, 9.80798039e-01, 9.73315858e-01,\n",
       "       9.66618325e-01, 9.57174253e-01, 9.50842470e-01, 8.24312783e-01,\n",
       "       6.58811425e-01, 2.79218188e-01, 1.99273192e-01, 1.94285397e-01,\n",
       "       1.07087905e-01, 9.71353160e-09])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm8HFWZ//HPNwmEPQgJCgkQVhEY1hjBwZFFNEQlDKMYBDSIRphBfyjygxFnBhlxkBlcEBSjQAAJm7JEAREjizCEJOwkgAQIJICEJSAgIIRn/jjnmk6nu27dS6q7b+73/Xr1K13706dv6ulzTtUpRQRmZmbNDGh3AGZm1tmcKMzMrJAThZmZFXKiMDOzQk4UZmZWyInCzMwKOVFYx1FyjqRFkmYsh/2NlBSSBi2P+KwcSTdI+nxF+95I0suSBubpd0q6SdJLkk6V9HVJP6vi2P2RE0WHkDRP0qv5j/9PkiZLWqNunfdL+n3+z/CipF9J2rpunbUkfV/S43lfc/P00NZ+ordlN2BvYEREjG53MMuLpN0lLWh3HCuCiHg8ItaIiMV51kTgWWCtiDg6Ir4dEZUkqf7IiaKzfDwi1gB2AHYE/rVrgaRdgd8CVwIbAJsAdwO3SNo0r7MyMA3YBhgDrAW8H3gOqOyEW8Ev9Y2BeRHxSjticc2jsQ4vl42BOfE27yDOtVmfF+tFhF8d8ALmAR+qmT4FuKpm+g/Ajxpsdw1wXn7/eeBpYI0eHHcb4Drg+bzt1/P8ycC3atbbHVhQF++xwD3A68A3gF/U7fsHwGn5/RDgLOAp4AngW8DABvEcBrwGLAZeBr6Z538BmJvjnApsULNNAP8CPAQ82mCfI/M6E4EncwxH1yw/AfgF8HPgz7kcBwDHAQ+TEu0lwDo121wK/Al4EbgJ2KZm2VhgDvBS/qxfA1YHXgXeyp/r5drPULPtqsCpwGN53zcDq5Y45mTgDOCqfNzbgM1KfM9NP2dNuR0GPA7c1ORvaBxwVy67h4Exef4NwOfz+82A3+djPAtcAKxds49jc1m9BDwI7JXnjwZm5X0/DXy3LrZB+bO/Afw1l+uH8nf685r97wL8L/AC6QfW7jXLbgBOAm7J39Hm7T4fdNqr7QH4lb+ImkQBjADuBX6Qp1cjnTj3aLDdocBT+f1FwLk9OOaa5JMmsEqefl9eNpnuE8VdwIakk9vGwF9IVX+AgXnfu+TpK4CfkE6Y6wEzgC82iWsCcHPN9J755LITMBj4Ye1JK58wrgPWIZ9U6/bXdVK5MB//74Bnasr7hHyi2Y904lwVOAqYnr+LwTn2C2v2+blcXoOB7wN31Sx7CvhAfv8OYKdGZdjks5+RT1zDcxm+Hxhc4piTSUlgNOnkeQFwUYnvuennrCm383K5NSrb0aTEtXcuu+HAVnnZDSxJFJvndQYDw0iJ7vt52buB+eTEmY+7WX5/K3BIfr8GS/6eumIb1OTv9QRyosgxPUdK4ANyHM8Bw2rifJyUTAcBK7X7fNBpr7YH4Ff+ItKJ92XSL6ogNSGtnZeNyPO2arDdGOCN/P464OQeHPNA4M4my+r/4+3Osonic3Xb3Ax8Jr/fG3g4v38nqdaxat2xr29y7AksnSjOAk6pmV6DdGIfmacD2LPgc3adVLaqmXcKcFZ+fwJ1v5aB+8m/avP0+vmYgxrsf+28/yF5+nHgi+Sk2awMG+xnAOkX7fYlvrv6Y04GflazfCzwQInvuennrCm3TQvi+AnwvSbLbiAnigbL9uuKiZREFpJqAivVrXcT8E1gaJPvtEyiOBY4v277a4HP1sR5Ytn/N/3x5ba4zrJfRKxJOqFsBXR1QC8iNVms32Cb9Um/tiH9Smq0TjMbkpoKemt+3fQU0kkJ4NN5GlJtYyXgKUkvSHqBdIJZr+RxNiA1xQAQES+TPuvwgli6i/exvN9m228MXF4T7/2kWt07JQ2UdLKkhyX9mZQ0Ycn39U+kE/Vjkm7M/UtlDCX94l/mOylxTEjNUl3+QkqoUPw9N/2cNesUlW2pvyFJ60m6SNITOf6fd8UeEXNJNZsTgIV5va7v5jBgS+ABSTMlfay7YzWwMfDJrs+YP+duLP1/pczfT7/lRNGBIuJG0i+k/8nTr5Cq4J9ssPoBpNoHwO+Aj0haveSh5pPajht5hdTk1eVdjUKtm74U2F3SCOAfWZIo5pNqFEMjYu38WisitikZ55Ok/+wA5M+3LqlNu1ksjWxY836jvN9m288H9qmJd+2IWCUiniAlwXGkX8BDSL9uAQQQETMjYhwpEV5BavcvE+OzpP6ZRt9J4TG7UfQ9F33OLkVxF+271n/l/WwXEWsBB9fGHhFTImI30vccwHfy/Ici4kBSWX4H+EUP/r5rYzy/7jOuHhEn16xT5u+n33Ki6FzfB/aWtEOePg74rKQvS1pT0jskfQvYlVQ1Bzif9J/il5K2kjRA0rr5mvKxDY7xa+Bdko6SNDjv93152V3AWEnrSHoX6RdfoYh4hlSNP4fUqXx/nv8U6YqtU/PluwMkbSbpgyXLYgpwqKQdJA0Gvg3cFhHzSm7f5d8krSZpG1LfzsUF654JnCRpYwBJwySNy8vWJCW+50jJ9NtdG0laWdJBkoZExBukTtiuSzifBtaVNKTRASPiLeBs4LuSNsi1iF3zZ256zBKKvueiz1nGWaTvZq/8vQ6XtFWD9dYkNa2+IGk4cEzXAknvlrRn/pyvkZrfFudlB0salsvmhbzJYnrm58DHJX0kl+kq+VLlET3cT7/lRNGh8kn3PODf8vTNwEeA/Ukdk4+RLqHdLSIeyuu8TvrF+QCpv+LPpE7joaSrYOqP8RKpL+HjpGaLh4A98uLzSVeHzCOd5ItOqrWm5Bim1M3/DLAy6WqgRaSrjEo1k0XENFI5/JL02TcDxpeMp9aNpCunpgH/ExG/LVj3B6Srq34r6SVSh2/XyfU8Uvk/Qfo80+u2PQSYl5tYDif9eiYiHiB1qD+Sm0A2YFlfI13IMJPUOf0d0v/T7o7ZVDffc9HnLLPvGaSk+z1Sp/aN1NT+anyTdDHCi6Qrsy6rWTYYOJlUo/oTqfbw9bxsDDBb0ss51vER8VrZ+HKM80m1sa+TLmKYT0pUPv+VpNyZY2Zm1pAzqpmZFaosUUg6W9JCSfc1WS5Jp+UhJu6RtFNVsZiZWe9VWaOYTGpfbGYfYIv8mgj8uMJYzMyslypLFBFxE6kzrplxpKEnIiKmA2tL6sk9AGZm1gLtHORrOEvf5LIgz3uqfkVJE0m1DlZfffWdt9qq0dV3LXDHHeDOfzPrg26HZyNiWG+2bWeiaHSjUMOzcERMAiYBjBo1KmbNmlVlXM2pzL1NZmadRzWjG/RUO696WsDSd8qOYOk7ZTvDwoUwdiysskq7IzEza4t21iimAkdKuoh0g8+L+Q7ezjJhAlxzTfPlbooys77gbbSIVJYoJF1IGtxuaH6q13+QBoYjIs4EriYNnDaXNIDZoVXF8rb8/vfNlw0e3Lo4zMzapLJEkQfyKloepIfNdLbXX2++bM89WxeHmVmb9L87s2v7HKTuX40MHgz77AOTJ7c0dDOzdujkZ+BWo7s+hzJe69GYZGZmfVr/q1EU9TmU4X4JM+tn+n6i6GlTUlGfQxnulzCzfqbvNz0tj6akMgYPTknC/RJm1s/0/USxPJqS3OdgZtZU3296clOSmVml+maiWB7DavgSVzOzUvpm05OH1TAza5m+WaPwsBpmZi3TNxOFh9UwM2uZvtf0dMcdjef78lUzs0r0vUTRrP/Bl7iamVWibzY91XO/hJlZZVaMROF+CTOzyvS9pqda7pcwM6tc304U7pcwM6vcitH0ZGZmlXGiMDOzQk4UZmZWyInCzMwKOVGYmVkhJwozMyvkRGFmZoWcKMzMrJAThZmZFXKiMDOzQk4UZmZWyInCzMwKFQ4KKOm9wMHAB4D1gVeB+4CrgCkR8VLlEZqZWVs1TRSSfg08B1wJnAosBFYBtgT2AK6SdEpE/LoVgZqZWXsomjxaVNI7I+Lpwo2l9SJiYSWRNTFKilldE80ei2pmZkuRdHtEjOrNtk37KLqShKTDJQ1psk5Lk4SZmbVemc7skcAdkqZI+lDF8ZiZWYfpNlFExHHAFsAFwOGSHpJ0oqSRFcdmZmYdoNTlsRHxFjAvv94iXQF1paT/KtpO0hhJD0qaK+m4Bss3knS9pDsl3SNpbI8/gZmZVarbRCHpnyXNAH4A3A5sFxFfAHYEPlWw3UDgDGAfYGvgQElb1632DeCSiNgRGA/8qFefwszMKlN4H0U2AhgfEY/UzoyItyTtW7DdaGBu13aSLgLGAXNqdwOsld8PAZ4sG7iZmbVGmaanDeqThKTJABFxX8F2w4H5NdML8rxaJwAHS1oAXA18qdGOJE2UNEvSrEbLzcysOmUSxXa1E5IGAO8tsZ0azKu/8eFAYHJEjADGAufn/S+9UcSkiBjV22uAzcys95omCknHSloEbCfp+fxaBDxL+vXfnQXAhjXTI1i2aekw4BKAiLiVdOf30B7Eb2ZmFSuqUZwCDAO+l/8dBgyNiHUi4pgS+54JbCFpE0krkzqrp9at8ziwF4Ck95ASxTM9+whmZlalos7szSPiIUnnA9t0zZRSi1JE3FO044h4U9KRwLXAQODsiJgt6URgVkRMBY4GfirpK6RmqQnRbEwRMzNri6Kxns6KiMMk/aHB4oiIf6g2tMY81pOZWc+9nbGemtYoIuKw/O8HehuYmZn1fWVuuLtD0jGSNm5FQGZm1lnKXB77SWAlYKqkWyUdJan+fggzM1tBlRkU8OGI+HZEbA98DtgZeKzyyMzMrCOUGcIDSSOAA0hjOw0Cjq8yKDMz6xzdJgpJtwBrApcCh0TEHyuPyszMOkaZGsUXuxnTyczMVmBNE4WkAyPiQmBPSXvWL4+I0yqNzMzMOkJRjeId+d9hDZb5Tjczs36i6Ia7rocIXRUR02uXSdql0qjMzKxjlLmPotFT585Y3oGYmVlnKuqjGA3sCgyT9OWaRWuRbsAzM7N+oKiPYnXSsyEGsXQ/xUuku7XNzKwfKOqjuB64XtI59Y9CNTOz/qOo6enUiDgaOFXSMlc5RcT+lUZmZmYdoajp6eL87+mtCMTMzDpTUdPTjPzvtK55koYAwyNiTgtiMzOzDlDmeRTTJK0l6R3AvcAUSf9dfWhmZtYJytxHsU5E/BnYHzg3InYAPlJtWGZm1inKJIpBkoaRLon9VcXxmJlZhymTKE4CbgQej4gZkjYFHq02LDMz6xSK6Fvj+42SYlbXRB+L3cysXSTdHhGjerNtmQcXDSU9AnVk7foRMbE3BzQzs76lzIOLrgSmAzcDi6sNx8zMOk2ZRLF6vkPbzMz6oTKd2ddI+nDlkZiZWUcqkygOB34j6WVJz0taJOn5qgMzM7POUKbpaWjlUZiZWcfqtkYREYtJN9sdm9+vD+xQdWBmZtYZyoz1dDqwB3BInvUX4MwqgzIzs85Rpunp/RGxk6Q7ASLieUkrVxyXmZl1iDKd2W9IGgAEgKR1gbcqjcrMzDpGmURxBvBLYJikb5JuvPtOpVGZmVnH6LbpKSLOk3Q78KE865MRcV+1YZmZWadoWqOQtIqkgQARMRu4itTktGnZnUsaI+lBSXMlHddknQMkzZE0W9KUHsZvZmYVK2p6uhbYDEDSZsAMYGvgq5JO6m7HOcmcAeyTtztQ0tZ162wB/Cvw9xGxDXBUbz6EmZlVpyhRrBMRf8zvPwtcFBFHkJ5ut2+JfY8G5kbEIxHxV+AiYFzdOl8AzoiIRQARsbBH0ZuZWeWKEkXtwx72BK4DiIjXKXfV03Bgfs30gjyv1pbAlpJukTRd0phGO5I0UdIsSbMaLTczs+oUdWbPlnQy8ATphP5bAElDAJXYd6N16p80NAjYAtgdGAH8QdK2EfHCUhtFTAImQXpwUYljm5nZclJUo/g88DKwFTAmIl7J87cFvlti3wuADWumRwBPNljnyoh4IyIeBR4kJQ4zM+sQTWsUOTF8q8H8W4BbSux7JrCFpE1ItZLxwKfr1rkCOBCYnJ+ktyXwSLnQzcysFYouj71C0j6SlkkmkjaW9O+SPtds+4h4EziSdPXU/cAlETFb0omSujrDrwWekzQHuB44JiKeezsfyMzMli9FNG7ylzQcOBrYD3gaeAZYhXQfxeOkq5V+2aI4/2aUFH/r0W4Su5mZLU3S7RExqlfbNksUdQfYnDS8+KvAgxHxUm8Otjw4UZiZ9dzbSRRlRo8lIuYCc3tzADMz69vKDApoZmb9mBOFmZkVKpUoJK2c+ynMzKyfKfMo1I8C95KH8JC0g6TLqw7MzMw6Q5kaxYnA+4AXACLiLsC1CzOzfqLUo1Drx15i2TGbzMxsBVXm8tj7JR0ADMjDcfw/YHq1YZmZWacoU6M4EtiZNLT4ZcBrpGRhZmb9QJkaxUci4ljg2K4ZkvYnJQ0zM1vBlalRfKPBvOOXdyBmZtaZmtYoJH0EGAMMl1T7/Im1KPeEOzMzWwEUNT0tBO4j9UnMrpn/EnBclUGZmVnnKHpw0Z3AnZIuiIjXWhiTmZl1kDKd2cMlnQRsTXoeBQARsWVlUZmZWcco05k9GTgHELAPcAlwUYUxmZlZBymTKFaLiGsBIuLhiPgGsEe1YZmZWaco0/T0uiQBD0s6HHgCWK/asMzMrFOUSRRfAdYAvgycBAwBPldlUGZm1jm6TRQRcVt++xJwCICkEVUGZWZmnaOwj0LSeyXtJ2lont5G0nl4UEAzs36jaaKQ9F/ABcBBwG8kHQ9cD9wN+NJYM7N+oqjpaRywfUS8Kmkd4Mk8/WBrQjMzs05Q1PT0WkS8ChARzwMPOEmYmfU/RTWKTSV1DSUuYGTNNBGxf6WRmZlZRyhKFP9UN316lYGYmVlnKhoUcForAzEzs85UZggPMzPrx5wozMysUOlEIWlwlYGYmVln6jZRSBot6V7goTy9vaQfVh6ZmZl1hDI1itOAjwHPAUTE3XiYcTOzfqNMohgQEY/VzVtcRTBmZtZ5ygwzPl/SaCAkDQS+BPyx2rDMzKxTlKlRHAF8FdgIeBrYJc/rlqQxkh6UNFfScQXrfUJSSBpVZr9mZtY6ZWoUb0bE+J7uONc+zgD2BhYAMyVNjYg5deutSXoo0m3L7sXMzNqtTI1ipqSrJX02n9TLGg3MjYhHIuKvwEWkEWnr/SdwCvBaD/ZtZmYt0m2iiIjNgG8BOwP3SrpCUpkaxnBgfs30gjzvbyTtCGwYEb8u2pGkiZJmSZpV4rhmZrYclbrhLiL+NyK+DOwE/Jn0QKPuqNGu/rZQGgB8Dzi6xPEnRcSoiHAfhplZi5W54W4NSQdJ+hUwA3gGeH+JfS8ANqyZHkF6+FGXNYFtgRskzSN1kk91h7aZWWcp05l9H/Ar4JSI+EMP9j0T2ELSJsATwHjg010LI+JFYGjXtKQbgK9FhJuXzMw6SJlEsWlEvNXTHUfEm5KOBK4FBgJnR8RsSScCsyJiak/3aWZmraeIaLxAOjUijpZ0OTV9C13a9YS7UdKSKkeT2M3MbGmSbu9tP29RjeLi/K+fbGdm1o8VPeFuRn77nohYKlnkJiU/Ac/MrB8oc3ns5xrMO2x5B2JmZp2paY1C0qdIVyptIumymkVrAi9UHZiZmXWGoj6KGaRnUIwgjdnU5SXgziqDMjOzzlHUR/Eo8Cjwu9aFY2Zmnaao6enGiPigpEUsfXmsgIiIdSqPzszM2q6o6anrcadDC9YxM7MVXNOrnmruxt4QGBgRi4FdgS8Cq7cgNjMz6wBlLo+9gvQY1M2A84D3AFMqjcrMzDpGmUTxVkS8AewPfD8ivkTdcyXMzGzFVSZRvCnpk8AhQNcDhlaqLiQzM+skZe/M3oM0zPgjedjwC6sNy8zMOkXT0WOXWkkaBGyeJ+dGxJuVRlXAo8eamfVcVaPHdu38A8D5pIcPCXiXpEMi4pbeHNDMzPqWMg8u+h4wNiLmAEh6Dylx+JGlZmb9QJk+ipW7kgRARNwPrFxdSGZm1knK1CjukPQTUi0C4CA8KKCZWb9RJlEcDnwZ+P+kPoqbgB9WGZSZmXWOwkQh6e+AzYDLI+KU1oRkZmadpGkfhaSvk4bvOAi4TlKjJ92ZmdkKrqhGcRCwXUS8ImkYcDVwdmvCMjOzTlF01dPrEfEKQEQ80826Zma2giqqUWxa86xsAZvVPjs7IvavNDIzM+sIRYnin+qmT68yEDMz60xFz8ye1spAzMysM7nfwczMCjlRmJlZodKJQtLgKgMxM7PO1G2ikDRa0r3AQ3l6e0kewsPMrJ8oU6M4DfgY8BxARNxNeuKdmZn1A2USxYCIeKxu3uIqgjEzs85TZvTY+ZJGAyFpIPAl4I/VhmVmZp2iTI3iCOCrwEbA08AueZ6ZmfUD3SaKiFgYEeMjYmh+jY+IZ8vsXNIYSQ9KmivpuAbLvyppjqR7JE2TtHFvPoSZmVWn26YnST8Fon5+REzsZruBwBnA3sACYKakqbWPVSU9KW9URPxF0hHAKcCnehC/mZlVrEwfxe9q3q8C/CMwv8R2o4G5EfEIgKSLgHFA7fO3r69ZfzpwcIn9mplZC3WbKCLi4tppSecD15XY93CWTigLgPcVrH8YcE2jBZImAhMBdi5xYDMzW37K1CjqbQKU6UtQg3nLNGEBSDoYGAV8sNHyiJgETAIYJTXch5mZVaNMH8UilpzgBwDPA8t0TDewANiwZnoE8GSD/X8IOB74YES8XmK/ZmbWQoWJQpKA7YEn8qy3IqLsL/qZwBaSNsnbjwc+Xbf/HYGfAGMiYmFPAjczs9YovDw2J4XLI2JxfpVu9omIN4EjgWuB+4FLImK2pBMl7ZtX+29gDeBSSXdJmtq7j2FmZlVRd+d+ST8GfhoRd7QmpGKjpJjVNVE+b5mZ9WuSbo+IUb3ZtmnTk6RBuVawG/AFSQ8Dr5A6qSMidupVtGZm1qcU9VHMAHYC9mtRLGZm1oGKEoUAIuLhFsViZmYdqChRDJP01WYLI+K7FcRjZmYdpihRDCRdkdToxjkzM+snihLFUxFxYssiMTOzjlR0H4VrEmZmVpgo9mpZFGZm1rGaJoqIeL6VgZiZWWcq8yhUMzPrx5wozMyskBOFmZkVcqIwM7NCThRmZlbIicLMzAo5UZiZWSEnCjMzK+REYWZmhZwozMyskBOFmZkVcqIwM7NCThRmZlbIicLMzAo5UZiZWSEnCjMzK+REYWZmhZwozMyskBOFmZkVcqIwM7NCThRmZlbIicLMzAo5UZiZWSEnCjMzK+REYWZmhZwozMysUKWJQtIYSQ9KmivpuAbLB0u6OC+/TdLIKuMxM7OeqyxRSBoInAHsA2wNHChp67rVDgMWRcTmwPeA71QVj5mZ9U6VNYrRwNyIeCQi/gpcBIyrW2cccG5+/wtgL0mqMCYzM+uhQRXuezgwv2Z6AfC+ZutExJuSXgTWBZ6tXUnSRGBinnxdcF9esPyj7luGUldW/ZjLYgmXxRIuiyXe3dsNq0wUjc7i0Yt1iIhJwCQASbMiYtTbD6/vc1ks4bJYwmWxhMtiCUmzerttlU1PC4ANa6ZHAE82W0fSIGAI8HyFMZmZWQ9VmShmAltI2kTSysB4YGrdOlOBz+b3nwB+HxHL1CjMzKx9Kmt6yn0ORwLXAgOBsyNitqQTgVkRMRU4Czhf0lxSTWJ8iV1PqirmPshlsYTLYgmXxRIuiyV6XRbyD3gzMyviO7PNzKyQE4WZmRXq2ETh4T+WKFEWX5U0R9I9kqZJ2rgdcbZCd2VRs94nJIWkFfbSyDJlIemA/LcxW9KUVsfYKiX+j2wk6XpJd+b/J2PbEWfVJJ0taaGk+5osl6TTcjndI2mnUjuOiI57kTq/HwY2BVYG7ga2rlvnn4Ez8/vxwMXtjruNZbEHsFp+f0R/Lou83prATcB0YFS7427j38UWwJ3AO/L0eu2Ou41lMQk4Ir/fGpjX7rgrKot/AHYC7muyfCxwDeketl2A28rst1NrFB7+Y4luyyIiro+Iv+TJ6aR7VlZEZf4uAP4TOAV4rZXBtViZsvgCcEZELAKIiIUtjrFVypRFAGvl90NY9p6uFUJE3ETxvWjjgPMimQ6sLWn97vbbqYmi0fAfw5utExFvAl3Df6xoypRFrcNIvxhWRN2WhaQdgQ0j4tetDKwNyvxdbAlsKekWSdMljWlZdK1VpixOAA6WtAC4GvhSa0LrOD09nwDVDuHxdiy34T9WAKU/p6SDgVHAByuNqH0Ky0LSANIoxBNaFVAblfm7GERqftqdVMv8g6RtI+KFimNrtTJlcSAwOSJOlbQr6f6tbSPirerD6yi9Om92ao3Cw38sUaYskPQh4Hhg34h4vUWxtVp3ZbEmsC1wg6R5pDbYqStoh3bZ/yNXRsQbEfEo8CApcaxoypTFYcAlABFxK7AKacDA/qbU+aRepyYKD/+xRLdlkZtbfkJKEitqOzR0UxYR8WJEDI2IkRExktRfs29E9HowtA5W5v/IFaQLHZA0lNQU9UhLo2yNMmXxOLAXgKT3kBLFMy2NsjNMBT6Tr37aBXgxIp7qbqOObHqK6ob/6HNKlsV/A2sAl+b+/McjYt+2BV2RkmXRL5Qsi2uBD0uaAywGjomI59oXdTVKlsXRwE8lfYXU1DJhRfxhKelCUlPj0Nwf8x/ASgARcSapf2YsMBf4C3Boqf2ugGVlZmbLUac2PZmZWYdwojAzs0JOFGZmVsiJwszMCjlRmJlZISeKfkrSYkl31bxGFqw7stlolD085g15hM+787AS7+7FPg6X9Jn8foKkDWqW/UzS1ss5zpmSdiixzVGSVuvFsb4v6R/y+yPzqJ6R73vo6b7enWO/S9L9kpbr090k7ds1MqukYUqjNt8p6QOSrpa0dsG2Tb+3gm1+J+kdy+8TWK+1e7RDv9rzAl7uwbojaTIaZQ+PeQN5NFdgIjB1ee1vOZdNbZyHAteV2GYeMLSHx1kHmF4zvWMu6x7vK29/LTCuZvrvKvz7GQ+cW+X3Rrqh9viqPoNf5V+uUdjf5JrDHyTdkV/vb7DONpJm5F+t90jaIs8/uGb+TyQN7OZNdYRXAAAFFElEQVRwNwGb5233yr9M71UaT39wnn+yljxn43/yvBMkfU3SJ0jjWl2Qj7lq/jU9StIRkk6piXmCpB/2Ms5bqRk0TdKPJc1Ser7DN/O8LwMbANdLuj7P+7CkW3M5XippjQb7/gTwm66JiLgzIuZ1E0+R9UlDNHTt794cywRJV0r6Ta4p/UfN52lYHkrPd7gj16qm1ezn9FzDOgUYW1P287pqQZI+k7+zuyWdn+c1+94+Kunymnj2lnRZnpxKGqPJ2q3dmcqv9rxId+relV+X53mrAavk91uQ7mqFmhoF8EPgoPx+ZWBV4D3Ar4CV8vwfAZ9pcMwbWPJL/RjgYtJQCvOBLfP884CjSL+2H2TJTaFr539PAL5Wv7/aaWAYadjprvnXALv1Ms6jgG/XLFsn/zswr7ddnp5HrgWQxhC6CVg9Tx8L/HuD45wLfLzB/L/tq4ff6aGkUZSvAb5SU2YTgKdIoyuvCtyXy6lheeTymw9sUveZJwCn17+vjRnYJn9vQ+u2bfi9kQapewAYlqen1JYJ8BCwbrv/v/T3V0cO4WEt8WpE1Le9rwR0/WJcTBobqN6twPGSRgCXRcRDkvYCdgZmKg0hsirQbMypCyS9SjqxfAl4N/BoRPwxLz8X+BfgdNLzJH4m6Sqg9LDhEfGMpEeUxrJ5KB/jlrzfnsS5Oikh1D4F7ABJE0nD36xPegjOPXXb7pLn35KPszKp3Oqtz3IcbygizpF0LTCG9NyBL0raPi++LvLwHfkX+27AmzQuj12AmyINJEhE9GSwzT2BX0TEs2W2jYjItY6DJZ0D7EpKVl0WkmprK9zQI32JE4XV+grwNLA96UKHZR78ExFTJN0GfBS4VtLnSb8Kz42Ify1xjIOiZpA+SQ2fIRJp/J7RpIHcxgNHkk5CZV0MHED6tXp5PiH1KE7Sk9JOBs4A9pe0CfA14L0RsUjSZFKNqJ5IJ+bumk1ebbJ9U/lkuiPwZEQs8zjPiHgSOBs4W+kChG27FtWvSpPvTdK+DdYvHWIvtj2HVLN5Dbg00vNluqxCKidrI/dRWK0hwFORxug/hPRreimSNgUeiYjTSG3I2wHTgE9IWi+vs47KP7f7AWCkpM3z9CHAjblNf0hEXE1q/ml05dFLpKHFG7kM2I/Uxn1xntejOCPiDeAbwC5KI46uBbwCvCjpncA+TWKZDvx912eStJqkRrWz+8n9NGVFxKERsUOjJJH7FVbK799Famp6Ii/eO3/eVUnlcgvNy+NW4IM5MSJpnR6EOI1U61q3YNulvrec3J4klfXkms8j4F2k2qe1kROF1foR8FlJ00nNTq80WOdTwH2S7gK2Ij1WcQ7pP/lvJd0DXEdqVulWRLxGalu/VNK9wFvAmaQTya/z/m4k1XbqTQbO7OpQrdvvImAOsHFEzMjzehxnRLwKnEpqX7+b9Azq2aRf7bfUrDoJuEbS9RHxDKkN/8J8nOmksqp3FWmkTyB1iiuN+DkCuEfSz4pia+DDpO/mbtIVUMdExJ/yspuB80l9Ur+MiFnNyiPHPxG4LO/r4voDNRMRs4GTSMn+buC7DVabzLLf2wXA/BxTl51JV4W9Wb8Day2PHmvWRpJuBj4WFT51TtIEUufxkVUd4+2SdDpwZ0ScVTPvB6RLqKe1LzID1yjM2u1oYKN2B9FOkm4nNWH+vG7RfU4SncE1CjMzK+QahZmZFXKiMDOzQk4UZmZWyInCzMwKOVGYmVmh/wNxyOR6zs7AQQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_curve,roc_auc_score\n",
    "%matplotlib inline\n",
    "fpr, tpr, thresholds = roc_curve(y_test, model.predict_proba(X_test)[:,1],drop_intermediate=False)\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.0])\n",
    "plt.title('ROC curve for breast cancer classifier')\n",
    "plt.xlabel('False Positive Rate (1 - Specificity)')\n",
    "plt.ylabel('True Positive Rate (Sensitivity)')\n",
    "plt.plot(fpr, tpr,color='red',lw=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9954600484261501"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, model.predict_proba(X_test)[:,1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}