Conf_AUC_ROC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame, Series\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>predicted_prob</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.592837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.624829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.073848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.544891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0.015118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       class  predicted_prob\n",
       "index                       \n",
       "1          0        0.592837\n",
       "2          1        0.624829\n",
       "3          0        0.073848\n",
       "4          1        0.544891\n",
       "5          0        0.015118"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"model_outcome.csv\", index_col = 0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create 'predicted' column: 1 if predicted_prob >= 0.5, 0 else"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['predicted'] = df['predicted_prob'].apply(lambda x: 1 if x >= 0.5 else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion_matrix(truth_arr, pred_arr):\n",
    "    arr = [[0, 0], [0, 0]]\n",
    "\n",
    "    for i in range(len(truth_arr)):\n",
    "        if truth_arr[i] == 0:\n",
    "            if pred_arr[i] == 0:\n",
    "                # True Negative\n",
    "                arr[0][0] += 1\n",
    "            else:\n",
    "                # False Positive\n",
    "                arr[1][0] += 1 \n",
    "                    \n",
    "        elif truth_arr[i] == 1:\n",
    "            if pred_arr[i] == 1:\n",
    "                # True Positive\n",
    "                arr[1][1] += 1\n",
    "            else:\n",
    "                # False Negative\n",
    "                arr[0][1] += 1\n",
    "\n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix\n",
      " [[385, 90], [123, 402]]\n"
     ]
    }
   ],
   "source": [
    "conf_arr = confusion_matrix(df['class'].values, df['predicted'].values)\n",
    "print(\"confusion matrix\\n\", conf_arr)\n",
    "# [[TN FN], [FP TP]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Sensitivity and Specificity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.82\n",
      "Specificity: 0.76\n"
     ]
    }
   ],
   "source": [
    "# sensitivity = TP / (TP + FN)\n",
    "sensitivity = round(conf_arr[1][1] / (conf_arr[1][1] + conf_arr[0][1]), 2)\n",
    "\n",
    "# specificity = TN / (TN + FP)\n",
    "specificity = round(conf_arr[0][0] / (conf_arr[0][0] + conf_arr[1][0]), 2)\n",
    "\n",
    "print(\"Sensitivity: {0}\\nSpecificity: {1}\".format(sensitivity, specificity))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Area Under the Receiver Operating Characteristic Curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### step 1. calculate False Positive Rate(FPR) and True Positive Rate(TPR) vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>predicted_prob</th>\n",
       "      <th>predicted</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>705</th>\n",
       "      <td>1</td>\n",
       "      <td>0.994258</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>1</td>\n",
       "      <td>0.993762</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>630</th>\n",
       "      <td>1</td>\n",
       "      <td>0.993762</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>810</th>\n",
       "      <td>1</td>\n",
       "      <td>0.993454</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>1</td>\n",
       "      <td>0.993454</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>1</td>\n",
       "      <td>0.991934</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>1</td>\n",
       "      <td>0.991386</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>1</td>\n",
       "      <td>0.991386</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>1</td>\n",
       "      <td>0.990895</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>585</th>\n",
       "      <td>1</td>\n",
       "      <td>0.989595</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>1</td>\n",
       "      <td>0.989432</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930</th>\n",
       "      <td>1</td>\n",
       "      <td>0.989432</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>0</td>\n",
       "      <td>0.986416</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775</th>\n",
       "      <td>1</td>\n",
       "      <td>0.983400</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>850</th>\n",
       "      <td>1</td>\n",
       "      <td>0.981358</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       class  predicted_prob  predicted\n",
       "index                                  \n",
       "705        1        0.994258          1\n",
       "138        1        0.993762          1\n",
       "630        1        0.993762          1\n",
       "810        1        0.993454          1\n",
       "318        1        0.993454          1\n",
       "616        1        0.991934          1\n",
       "653        1        0.991386          1\n",
       "161        1        0.991386          1\n",
       "369        1        0.990895          1\n",
       "585        1        0.989595          1\n",
       "438        1        0.989432          1\n",
       "930        1        0.989432          1\n",
       "338        0        0.986416          1\n",
       "775        1        0.983400          1\n",
       "850        1        0.981358          1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sort df by predicted_prob in descending order\n",
    "df_roc = df.copy().sort_values(by = ['predicted_prob'], ascending=False)\n",
    "df_roc.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate FPR and TPR by decreasing threshold \n",
    "TPR, FPR = [], []\n",
    "TP, FP = 0, 0\n",
    "for i in range(len(df_roc)):\n",
    "    if df_roc.iloc[i]['class'] == 1:\n",
    "        TP, FP = TP+1, FP\n",
    "    else:\n",
    "        TP, FP = TP, FP+1\n",
    "    \n",
    "    TPR.append(TP/df_roc['class'].sum())\n",
    "    FPR.append(FP/(df_roc['class'].count()-df_roc['class'].sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store FPR and TPR in dataframe\n",
    "d = {'FPR': FPR, 'TPR': TPR}\n",
    "df_fpr_tpr = DataFrame(data=d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### step 2. calculate the area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+cXHV97/HXm00CCxECQVeygCDS9MZqiVlBitIFCgmoJG2DDXgR+pDGtpd7L7cPg6Qi2kgrSq1o4aKxIApCwIhpWlMjErbXWsQAgYSIKyFAshuE8mOBxcVNNp/7xzmTTIaZPbM/zsxs9v18PObBzPd8z5n3nizz2fM9Z75HEYGZmdlg9ql3ADMza3wuFmZmlsnFwszMMrlYmJlZJhcLMzPL5GJhZmaZXCxs3JJ0pKReSU05bPskSY+l25832ts3qzUXC8uVpA+nH5ilj5B0RT2zRcSWiJgcEQM5bH4JcG26/RWlCyU9Kakv3Re/knSTpMklfX5P0hpJr0h6SdK/SJpR0udASddI2pJua1P6+tAcfiYbx1wsLFcR8e30A3PXA7gEeAb4ep3j5ektwMaMPh9M98dxwExgcWGBpBOBHwL/DEwDjgYeBn4i6a1pn0nA3cDbgTnAgcDvAc8Dx4/mD1NM0oS8tm0NLCL88KNmD5IPxVeA9qK2acBK4AVgE/BnRcv2Ba4BtqWPa4B902XtQBdwKfAs8DQwDzgL+GW6vb8eJMtRQAAT0tcdwGeBn6QZfwgcOsj6f5bmfSHNPy1tfxzYCfQBvYW8Jes+CfxB0esvAN8vev1j4P+WWe/fgG+lzy8iKbqTh7D/3w7clWZ+prB/gJuAK4v6tQNdJXk/AawHfgNcDiwv2faXga+kzw8Cbkj/TbqBK4Gmev/++TH8h48srGYkTQGWk3wodRQtuo3kQ38aMB/4O0mnpcs+CbyH5K/v3yX5i/nyonXfDOwHtAJXkByt/HdgFvA+4IrCX+JVOg/4U+BNwCTg4xV+llOBzwEfAg4DngKWAUTEMcAW0iOHiPjNYG8o6XDgTJLCg6T9SY4QvlOm+x3A6enzPwB+EBG91fxgkt4A/Aj4Acm+fhvJkUm1zgXeD0wBbgbOknRguu0mkn1xa9r3m8CO9D1mAmeQFDcbo1wsrCYkieQD5BGSv6IL7UcA7wU+ERGvRcRDwD8B56ddPgwsiYhnI+K/gL8pWgawHfjbiNhO8mF9KPDliHglIjaSDAW9cwhRvxERv4yIPpIP5uMq9PswcGNEPJgWg8XAiZKOGsJ7rZD0CrCV5Mjo02n7IST/bz5dZp2nSX5GgKkV+lTyAeBXEfHFdF+/EhH3DWH9r0TE1ojoi4ingAdJjuQATgV+HRE/ldRCUvwuiYhXI+JZ4EvAgiG8lzUYFwurlU8AvwNcEBHFs1dOA16IiFeK2p4iOVIoLH+qZNm0otfPx+4T1H3pf58pWt4HTAYoOcF+ZIWcvyp6/uvCumXskSv96/75otzVmBcRbyAZ8vltdheBF0mGsQ4rs85hwHPp8+cr9KnkCJIhsuHaWvL6VpKjDUiOyApHFW8BJgJPS+qR1AN8jeRozcYoFwvLnaR2kuGk+RHRU7J4G3BIOkRScCTJOHdh+VtKlm0bTo7Y80T7luFso8geuSQdQPKXfnfFNSrn+neScwZ/n75+FbgXOKdM9w+xe+joR8Ds9L2rsRU4psKyV4H9i16/uVzUktffAdrTYbQ/ZHex2EpyXuPQiJiSPg6MiLdXmdMakIuF5UrSYSTDQ5dExLrS5RGxFfhP4HOS9pP0TuCjwLfTLrcBl0t6Y3o56BXALbVJP6hbgT+VdJykfYG/A+6LiCeHub1rgNMlFYa9LgMukPS/JL1B0sGSrgROJBmKg+S8wVbgu5J+W9I+kqZK+mtJZ5V5j38F3izpEkn7pts9IV32EMk5iEMkvZnkirVBpcOCHcA3gCci4tG0/WmSiwO+mF7au4+kYyT9/jD2izUIFwvL258BLcCXy3zX4qtpn3NJrkzaBnwP+HRE3JUuuxK4n+QqnA0k4+RX1vIHKCci7gY+BXyX5LzBMYxgTD794P1Wuk0i4j+A2cAfpdt/iuRE8Xsj4rG0z29ITnL/guQKp5eBn5EMZ73uXEQ61Hc68EGS4bbHgFPSxTeTXJr7JMkH/e1VRr81zXBrSftHSC4Q+DnJsNpyhjZkZg1Gew4fm5mZvZ6PLMzMLJOLhZmZZXKxMDOzTC4WZmaWaa+ZEOzQQw+No446atjrv/rqqxxwQLWXq9eXs+ZjrGQdKznBWfMymlkfeOCB5yLijZkd6z051Wg9Zs2aFSNxzz33jGj9WnLWfIyVrGMlZ4Sz5mU0swL3hycSNDOz0eBiYWZmmVwszMwsU67FQtIcSZ3prR4vK7P8ZEkPStohaX7JsiMl/VDSo5J+PsSpn83MbBTlVizSm6FcRzKv/Qzg3NL7B5PcIOZCXj+vDCTz5FwdEf+N5IY3z+aV1czMBpfnpbPHA5siYjOApGXAXJKJxQCIdIZOSTuLV0yLyoRIJ5OLKu8EZmZWCyvWdXP16k629fQxbUozi2ZPZ97ModzKZOzlyG0iwXRYaU5EXJS+Ph84ISIuLtP3JuBfI2J5+noeyS0Y+0luVP8j4LLYfZObwnoLgYUALS0ts5YtWzbsvL29vUyeXOk+N43FWfMxVrKOlZywd2b9z23buemRfvqL/sSdtA9c+DuT+L1pE3NMuFtvby/rX953VHKccsopD0REW1a/PI8sVKat2so0geT+yTNJhqpuJxmuumGPjUUsBZYCtLW1RXt7+zCjQkdHByNZv5acNR9jJetYyQl7Z9ZPXrVmjw9ogP6dcNPGHTz8Sm0KY09PH0+8vKNsju9vaeKvz2sf9ffM8wR3F8ltHAsOp/o7nHUB6yJic0TsAFYA7xrlfGZmQ7atp69se//AzrLtean0fpXyjVSeRxZrgWMlHU1yq8kFJPfprXbdgyW9MZKbwpxKcgMcM7O6mjalme4yH8itU5q5/WMn1iRDR0cHn/zpzrI5pk1pzuU9czuySI8ILgZWA48Cd0TERklLJJ0NIOndkrpI7jX8NUkb03UHgI8Dd0vaQDKk9fW8spqZVWvR7Ok0T2zao615YhOLZk/fq3PkOpFgRKwCVpW0XVH0fC3J8FS5de8C3plnPjOzoSpcbXTp8vX0D+yktU5XQxXer1ZXQ+01s86amdXKvJmt3PazLQA1G3qqlKNWRcrTfZiZWSYXCzMzy+RiYWZmmVwszMwsk09wm9mY0UhzMq3b0kP/wE5OumpN3XLUkouFmY0JK9Z1s/jODfRtT6aI6+7pY/GdGwBq+kFdyFH4BnW9ctSai4WZjQlXr+7cVSgK+rYPcOny9bsuYx2Jnp4+ru+8N7Nf4YiiNMfVqztdLMzM6j0ENF7nZGoULhZmlqkRhoCm7D+RF3+9/XXtB+8/cVS+GJfMOpu9nZOuWlPTOZkahYuFmWUa7hBQtUM71Xi57/WFAiCnW/JUtGj29D0KJ9Rnbqhac7Ews0yNMAQ0UKEovFShiOSl1nMyNQoXCzPLNNxpuasd2qlGIw3/1HJOpkbhL+WZWaZGmJa7ETKMZz6yMLNMjTAt93gd/mkULhZmVpVGmJZ7PA7/NAoPQ5mZWSYXCzMzy+RiYWZmmVwszMwsU67FQtIcSZ2SNkm6rMzykyU9KGmHpPlllh8oqVvStXnmNDOzweVWLCQ1AdcBZwIzgHMlzSjptgW4ELi1wmY+C/x7XhnNzKw6eR5ZHA9siojNEdEPLAPmFneIiCcjYj3wujkDJM0CWoAf5pjRzMyqkGexaAW2Fr3uStsySdoH+CKwKIdcZmY2RHl+KU9l2qqdH/IvgVURsVUqt5n0DaSFwEKAlpYWOjo6hppxl97e3hGtX0vOmo+xkrWeOXvSuZmqff+xsk/BWbPkWSy6gCOKXh8ObKty3ROB90n6S2AyMElSb0TscZI8IpYCSwHa2tqivb192GGTCc+Gv34tOWs+xkrWeuYsTDde7eSAY2WfgrNmybNYrAWOlXQ00A0sAM6rZsWI+HDhuaQLgbbSQmFmZrWT2zmLiNgBXAysBh4F7oiIjZKWSDobQNK7JXUB5wBfk7QxrzxmZjZ8uU4kGBGrgFUlbVcUPV9LMjw12DZuAm7KIZ6ZmVXJ3+A2M7NMLhZmZpbJxcLMzDK5WJiZWSYXCzMzy+RiYWZmmVwszMwsk4uFmZllcrEwM7NMuX6D28xGbsW6bq5e3cm2nj6mTWnm/UcO0F6nHOu29NA/sJOTrlrDotnTmTezqrsO2F7ARxZmDWzFum4W37mB7p4+Auju6eOmR/pZsa67Ljn6B5L7lHX39LH4zg01z2H14yMLswZ29epO+rYP7NHWvxMuXb6e2362pWY5CkcUxfq2D3D16k4fXYwTPrIwa2Db0psNlSr94M5bpferlM/2Pj6yMGtg06Y0013mA7l1SjO3f6y6GxCNhpOuWlM2x7QpzTXLYPXlIwuzBrZo9nSaJzbt0TZpn6S93jmaJzbVPIfVj48szBpY4XzApcvX0z+wk9b0aqhanycovF/xVVm+Gmp8cbEwa3DzZrbuOpl9+8dOpKOjo245XBzGLw9DmZlZJhcLMzPL5GJhZmaZXCzMzCxTrsVC0hxJnZI2SbqszPKTJT0oaYek+UXtx0m6V9JGSesl/UmeOc3MbHC5FQtJTcB1wJnADOBcSTNKum0BLgRuLWn/NfCRiHg7MAe4RtKUvLKamdng8rx09nhgU0RsBpC0DJgL/LzQISKeTJftMZdARPyy6Pk2Sc8CbwR6csxrZmYV5FksWoGtRa+7gBOGuhFJxwOTgMfLLFsILARoaWkZ0fXnvb29dbt+faicNR+NnLUnnWqjo6OjoXOWctZ81CNrnsVCZdpiSBuQDgNuBi6IiNfNZBYRS4GlAG1tbdHe3j6MmImOjg5Gsn4tOWs+Gjnr9Z33AtDefmJD5yzlrPmoR9Y8T3B3AUcUvT4c2FbtypIOBL4PXB4RPx3lbGZmNgR5Fou1wLGSjpY0CVgArKxmxbT/94BvRcR3csxoZmZVyK1YRMQO4GJgNfAocEdEbJS0RNLZAJLeLakLOAf4mqSN6eofAk4GLpT0UPo4Lq+sZmY2uFwnEoyIVcCqkrYrip6vJRmeKl3vFuCWPLOZmVn1/A1uMzPL5GJhZmaZXCzMzCyTi4WZmWVysTAzs0wuFmZmlsnFwszMMrlYmJlZJhcLMzPL5GJhZmaZXCzMzCyTi4WZmWVysTAzs0wuFmZmlsnFwszMMrlYmJlZJhcLMzPL5GJhZmaZXCzMzCyTi4WZmWXKtVhImiOpU9ImSZeVWX6ypAcl7ZA0v2TZBZIeSx8X5JnTzMwGl1uxkNQEXAecCcwAzpU0o6TbFuBC4NaSdQ8BPg2cABwPfFrSwXllNTOzweV5ZHE8sCkiNkdEP7AMmFvcISKejIj1wM6SdWcDd0XECxHxInAXMCfHrGZmNog8i0UrsLXodVfalve6ZmY2yibkuG2VaYvRXFfSQmAhQEtLCx0dHVWHK9Xb2zui9WvJWfPRyFl7evoA6OjoaOicpZw1H/XImmex6AKOKHp9OLBtCOu2l6zbUdopIpYCSwHa2tqivb29tEvVOjo6GMn6teSs+WjkrNd33gtAe/uJDZ2zlLPmox5Z8xyGWgscK+loSZOABcDKKtddDZwh6eD0xPYZaZuZmdVBbsUiInYAF5N8yD8K3BERGyUtkXQ2gKR3S+oCzgG+Jmljuu4LwGdJCs5aYEnaZmZmdTDkYaj0ktgFEfHtrL4RsQpYVdJ2RdHztSRDTOXWvRG4caj5zMxs9FU8spB0oKTFkq6VdIYS/xPYDHyodhHNzKzeBjuyuBl4EbgXuAhYBEwC5kbEQzXIZmZmDWKwYvHWiHgHgKR/Ap4DjoyIV2qSzMzMGsZgJ7i3F55ExADwhAuFmdn4NNiRxe9KepndX5BrLnodEXFg7unMzKwhVCwWEdFUyyBmZta4KhYLSfsBfw68DVgP3Jh+d8LMzMaZwc5ZfBNoAzYAZwFfrEkiMzNrOIOds5hRdDXUDcDPahPJzMwaTbVXQ3n4ycxsHBvsyOK49OonSK6A8tVQZmbj1GDF4uGImFmzJGZm1rAGG4aq9kZFZma2lxvsyOJNkv6q0sKI+Icc8piZWQMarFg0AZMpf4tTMzMbRwYrFk9HxJKaJTEzs4Y12DkLH1GYmRkweLE4rWYpzMysoVUsFr7ntZmZFQx2ZGFmZga4WJiZWRVyLRaS5kjqlLRJ0mVllu8r6fZ0+X2SjkrbJ0r6pqQNkh6VtDjPnGZmNrjcioWkJuA64ExgBnCupBkl3T4KvBgRbwO+BHw+bT8H2Ded9XYW8LFCITEzs9rL88jieGBTRGyOiH5gGTC3pM9ckvtmACwHTpMkkqlGDpA0AWgG+oGXMTOzulBEPlNASZoPzImIi9LX5wMnRMTFRX0eSft0pa8fB04AXgJuJrl8d3/g/0TE0jLvsRBYCNDS0jJr2bJlw87b29vL5MmTh71+LTlrPho56+fu6wNg8QnNDZ2zlLPmYzSznnLKKQ9ERFtWv8G+wT1S5b7UV1qZKvU5HhgApgEHAz+W9KOI2LxHx6SALAVoa2uL9vb2YYft6OhgJOvXkrPmo5GzXt95LwDt7Sc2dM5SzpqPemTNcxiqCzii6PXhwLZKfdIhp4OAF4DzgB9ExPaIeBb4CcktXs3MrA7yLBZrgWMlHS1pErAAWFnSZyVwQfp8PrAmknGxLcCpShwAvAf4RY5ZzcxsELkVi/RWrBcDq4FHgTsiYqOkJZLOTrvdAEyVtAn4K6Bwee11JDPePkJSdL4REevzympmZoPL85wFEbEKWFXSdkXR89dILpMtXa+3XLuZmdWHv8FtZmaZcj2yMBvLVqzr5urVnWzr6WPalGYWzZ7OvJmtdcmxbksP/QM7OemqNbz/yAHaa57CxjsfWZiVsWJdN4vv3EB3Tx8BdPf0sfjODaxY112XHP0DOyHNcdMj/TXPYeYjC7Myrl7dSd/2gT3a+rYPcOny9dz2sy01y1E4oijWvzPJV4+jHBu/fGRhVsa2nr6y7aUf3Hmr9H6V8pnlxUcWZmVMm9JMd5kP5NYpzdz+sRNrluOkq9aUzTFtSnPNMpiBjyzMylo0ezrNE5v2aGue2MSi2dPrnmPSPtQ8h5mPLMzKKJwPuHT5evoHdtJap6uhCu9XfFXW+48c8PkKqzkXC7MK5s1s3XUyu5ZDT+VyFBeHjo6OumWx8cvDUGZmlsnFwszMMrlYmJlZJhcLMzPL5GJhZmaZXCzMzCyTi4WZmWVysTAzs0wuFmZmlsnFwszMMnm6D9ulUe4Md/mKDdx231YGImiSOPeEI7hy3jtqnqP0DnX12h9mjSDXIwtJcyR1Stok6bIyy/eVdHu6/D5JRxUte6ekeyVtlLRB0n55Zh3vGuXOcJev2MAtP93CQAQAAxHc8tMtXL5iQ01zlLtDXT32h1mjyO3IQlITcB1wOtAFrJW0MiJ+XtTto8CLEfE2SQuAzwN/ImkCcAtwfkQ8LGkqsD2vrJbfneF6evq4vvPeqvvf98QLZdtv+ekWHnumd9g5qlGctdwd6vq2D/gOdTZu5XlkcTywKSI2R0Q/sAyYW9JnLvDN9Ply4DRJAs4A1kfEwwAR8XxEDGC5aZQ7wzUK36HObE95nrNoBbYWve4CTqjUJyJ2SHoJmAr8FhCSVgNvBJZFxBdK30DSQmAhQEtLy4imbu7t7R0zUz/nkfWQ/cTzr8Xr2qfuJ/5i+m+Gvd3e3gEmT65+/bVPQLmP6X1gRDmqUZx109Pl98ch+6nuvyfj/Xc1L846uDyLhcq0lf7fV6nPBOC9wLuBXwN3S3ogIu7eo2PEUmApQFtbW7S3tw87bEdHByNZv5byyPqpg5Ix+uKhqOaJTXxq7jtoH8Gwy1CznteTnLN4Xft7jqS9Pd+T3MVZ89ofo2G8/67mxVkHl2ex6AKOKHp9OLCtQp+u9DzFQcALafu/R8RzAJJWAe8C7mYv1AhXIc2b2cr9T72w64O6SeKPZ7XWPEfhqqd6Xw1V7g51vhrKxrM8i8Va4FhJRwPdwALgvJI+K4ELgHuB+cCaiCgMP10qaX+gH/h94Es5Zq2bwlU3hb9gC1fdADX9YFqxrpvvPrD7Sp+BCL77QDdtbzmkLgWjHpfKliq9Q53ZeJZbsUjPQVwMrAaagBsjYqOkJcD9EbESuAG4WdImkiOKBem6L0r6B5KCE8CqiPh+XlnraThXIQ31CqNq+OofMxtMrl/Ki4hVwKqStiuKnr8GnFNh3VtILp/dqzXKVUi++sfMBuNvcNfZtCnNdJf5QG6d0sztHzux7DrJya3yy4brpKvWlM0xbUrzqL6PmY1NnhuqzhbNnk7zxKY92ponNrFo9vRxmcPMGpOPLOqscD7g0uXr6R/YSWsdr4YCX/1jZuW5WDSAeTNbd53MrjT0VKscLg5mVo6HoczMLJOLhZmZZXKxMDOzTC4WZmaWycXCzMwyuViYmVkmFwszM8s07ovFinXdnHTVGi78waucdNWautxjecW6btZt6eG+J16oWwYzs8GM6y/lNcL04IUMhYn86jVFuZnZYMZ1sRjO9OCjzVODm9lYMK6HoRphenBPDW5mY8G4PrIYzvTgo81Tg5vZWDCujywaYVruRshgZpZlXB9ZNML04J4a3MzGgnFdLGD39OA9PT2s/sSpdcvg4mBmjWxcD0OZmVl1XCzMzCxTrsVC0hxJnZI2SbqszPJ9Jd2eLr9P0lEly4+U1Cvp43nmNDOzweVWLCQ1AdcBZwIzgHMlzSjp9lHgxYh4G/Al4PMly78E/FteGc3MrDp5HlkcD2yKiM0R0Q8sA+aW9JkLfDN9vhw4TZIAJM0DNgMbc8xoZmZVyPNqqFZga9HrLuCESn0iYoekl4CpkvqATwCnAxWHoCQtBBYCtLS00NHRMaygPT19DAwMDHv9Wuvt7XXWHIyVrGMlJzhrXuqRNc9ioTJtUWWfvwG+FBG96YFGWRGxFFgK0NbWFu3t7cMKen3nvfT09DDc9Wuto6PDWXMwVrKOlZzgrHmpR9Y8i0UXcETR68OBbRX6dEmaABwEvEByBDJf0heAKcBOSa9FxLU55jUzswryLBZrgWMlHQ10AwuA80r6rAQuAO4F5gNrIiKA9xU6SPoM0OtCYWZWP7kVi/QcxMXAaqAJuDEiNkpaAtwfESuBG4CbJW0iOaJYkFceMzMbvlyn+4iIVcCqkrYrip6/BpyTsY3P5BLOzMyq5m9wm5lZJhcLMzPL5GJhZmaZXCzMzCyTi4WZmWVysTAzs0wuFmZmlsnFwszMMrlYmJlZJhcLMzPL5GJhZmaZXCzMzCyTi4WZmWVysTAzs0zjvlisWNfNui09dL64k5OuWsOKdd31jmRm1nDGdbFYsa6bxXduoH9gJwDdPX0svnODC4aZWYlxXSyuXt1J3/aBPdr6tg9w9erOOiUyM2tM47pYbOvpG1K7mdl4Na6LxbQpzUNqNzMbr8Z1sVg0ezrNE5v2aGue2MSi2dPrlMjMrDHlWiwkzZHUKWmTpMvKLN9X0u3p8vskHZW2ny7pAUkb0v+emke+eTNb+dwfvYPW9EiidUozn/ujdzBvZmseb2dmNmZNyGvDkpqA64DTgS5graSVEfHzom4fBV6MiLdJWgB8HvgT4DnggxGxTdLvAKuBXD7B581sZd7MVjo6Omhvb8/jLczMxrw8jyyOBzZFxOaI6AeWAXNL+swFvpk+Xw6cJkkRsS4itqXtG4H9JO2bY1YzMxuEIiKfDUvzgTkRcVH6+nzghIi4uKjPI2mfrvT142mf50q28+cR8Qdl3mMhsBCgpaVl1rJly4adt7e3l8mTJw97/Vpy1nyMlaxjJSc4a15GM+spp5zyQES0ZXaMiFwewDnAPxW9Ph/4x5I+G4HDi14/Dkwtev32tO2YrPebNWtWjMQ999wzovVryVnzMVayjpWcEc6al9HMCtwfVXym5zkM1QUcUfT6cGBbpT6SJgAHAS+krw8Hvgd8JCIezzGnmZllyLNYrAWOlXS0pEnAAmBlSZ+VwAXp8/nAmogISVOA7wOLI+InOWY0M7Mq5HbOAkDSWcA1QBNwY0T8raQlJIc9KyXtB9wMzCQ5olgQEZslXQ4sBh4r2twZEfHsIO/1X8BTI4h7KMlVWGOBs+ZjrGQdKznBWfMymlnfEhFvzOqUa7EYSyTdH9Wc5GkAzpqPsZJ1rOQEZ81LPbKO629wm5lZdVwszMwsk4vFbkvrHWAInDUfYyXrWMkJzpqXmmf1OQszM8vkIwszM8vkYmFmZpn2mmIx3OnQ02WL0/ZOSbOztpl+0fA+SY+l25xUz6ySjpB0j6RHJW2U9L+L+n9GUrekh9LHWfXMmrY/mU4//5Ck+4vaD5F0V7pf75J0cD2zSppetN8ekvSypEvSZXXZr5Kmpv/WvZKuLVlnVrpfN0n6iiSl7XXZr5WyStpf0vcl/SL9fb2qaNmFkv6raL9eVM+s6bKOdJuFTG8abFv1yCnpDSW/q89JuiZdNqJ9uks1c4I0+oPkS3+PA28FJgEPAzNK+vwl8NX0+QLg9vT5jLT/vsDR6XaaBtsmcAfJFwgBvgr8RZ2zHga8K+3zBuCXRVk/A3y8UfZruuxJ4NAy7/cF4LL0+WXA5+udtWT7vyL5AlM99+sBwHuBPweuLVnnZ8CJgIB/A86s834tmxXYHzglfT4J+HFR1gtLf64G2K8dQFuZ9yu7rXrlLFn/AeDkke7T4sfecmQx7OnQ0/ZlEfGbiHgC2JRur+w203VOTbdBus159cwaEU9HxIMAEfEK8Cijc/+PPPbrYIq3Vff9WrLuacDjETGSWQJGnDUiXo2I/wBeK+4s6TDgwIi4N5JPiG+xe//VZb9WyhoRv46Ie9Ln/cCDJHPHjdSoZ81Q6feprjklHQu8iaQIj5q9pVi0AluLXnfx+g/LXX0iYgfwEjB1kHUrtU8FetJtVHqvWmfdJT1cnQncV9R8saT1km4c4hBEXlkD+KGSuyAuLOrTEhFPp9t6muQXvt5ZCxYAt5W01WPDE0eYAAAD30lEQVS/DrbNrgrbrNd+zaRkHrgPAncXNf9xul+XSzqiwqq1zvqNdAjnU0UFYbjbynWfAueSHIkUX+o63H26y95SLMpV89Jrgiv1Ga32auWRNVlJmgx8F7gkIl5Om68HjgGOA54GvtgAWU+KiHcBZwL/Q9LJQ8hUSZ77dRJwNvCdouX12q8j2eZw5JE1WSmZafo24CsRsTlt/hfgqIh4J/Ajdv91Xc+sH46IdwDvSx/nj2Bb1a43kn/P0j9sRrJPd9lbisVIpkOvtG6l9ueAKek2Kr1XrbMiaSJJofh2RNxZ6BARz0TEQETsBL5O9lBQ7lkjvQtiJBNDfq8o0zPpcEphWKXixJG1ypo6E3gwIp4pNNRxvw62zeKhnOJt1mu/ZlkKPBYR1xQaIuL5iPhN+vLrwKx6Z42I7vS/rwC3svvferg/d277VNLvAhMi4oGi/CPZp7vsLcVi2NOhp+0L0qsPjgaOJTlRWHab6Tr3pNsg3eY/1zNrelh8A/BoRPxD8YYKHxKpPwQeqXPWAyS9Ic12AHBGUabibdV9vxatdy4lQ1B13K9lpcNLr0h6T/r78BF277967deKJF1J8gF4SUl78X49m+T8W92ySpog6dD0+UTgA5T/fa3q584rZ5Gs39Wh7tPdRnqGvFEewFkkVwE9DnwybVsCnJ0+349kGGETyQfBW4vW/WS6XifpVRmVtpm2vzXdxqZ0m/vWMyvJ1REBrAceSh9npctuBjaky1YCh9U561tJrv54mOROicX7dSrJ2PVj6X8PaYDfgf2B54GDSt6rnvv1SZK/MntJ/gItXPnWRvJB9jhwLbtnaKjnfn1dVpK/pIPkQ6vw+3pR2v9z6e/FwyR/lP12nbMeQHJl0fo015fZfVVfxW3V498/Xba5dJ+NdJ8WHp7uw8zMMu0tw1BmZpYjFwszM8vkYmFmZplcLMzMLJOLhZmZZXKxMBsFkgZKZv08SlK7pJckrVMyI/Cn077F7b+Q9Pf1zm+WZUJ2FzOrQl9EHFfckM7T9eOI+ED6BcSHJP1rurjQ3gysk/S9iPhJbSObVc9HFmY1EBGvkny565iS9j6SL6WNxizBZrlxsTAbHc1FQ1DfK10oaSrwHpJv0ha3H0wyvcj/q01Ms+HxMJTZ6HjdMFTqfZLWATuBqyJio6T2tH09MD1t/1UNs5oNmYuFWb5+HBEfqNQu6beA/0jPWTxU63Bm1fIwlFkdRcQvSSZ6+0S9s5gNxsXCrP6+CpycTo9u1pA866yZmWXykYWZmWVysTAzs0wuFmZmlsnFwszMMrlYmJlZJhcLMzPL5GJhZmaZ/j8+xHI7go/iEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# decompose FPR, TPR curve into many rectangulars \n",
    "s = df_fpr_tpr.groupby(['FPR'])['TPR'].max()\n",
    "\n",
    "plt.step(s.index[:10], s.values[:10], 'C0o')\n",
    "plt.xlabel('FPR')\n",
    "plt.ylabel('TPR')\n",
    "plt.title('Zoom-in of ROC curve')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area Under ROC curve: 0.89\n"
     ]
    }
   ],
   "source": [
    "# calculate the area under each rectangular, and accumulate them to get AUC \n",
    "delta_x = (s.index[1:-1] - s.index[:-2]).values\n",
    "delta_y = s.values[:-2]\n",
    "print(\"Area Under ROC curve:\", round(sum(delta_x * delta_y), 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Visualization of Receiver Operating Characteristic Curve "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAH1ZJREFUeJzt3X+0HGWd5/H3J8kEIiQkGMlA+JEoAQ2oQbLEjKtcBlRglTizyoI6Ay4adUXXA+sKOssg6szoqKhHhh8jrj9DBOcoGYziqLniDwiEJaIJRmMiSSAIkQS4GCTAd/+oalJpuvv2c29Xd9/uz+uce053V3XV9+mu++nnqaruUkRgZmbNGdfpAszMxhKHpplZAoemmVkCh6aZWQKHpplZAoemmVkCh2abSXqTpO91uo5uImlI0nM7sN5ZkkLShHavuwyS1kgaGMHzvE0m6OvQlPQ7STvzf9r7JH1R0r5lrjMivhYRrypzHUWS/kLSDyU9IukhSf8uaW671l+jnkFJby0+FhH7RsSGktZ3hKTrJG3L23+npPMkjS9jfSOVh/fho1lGRBwVEYPDrOcZHxSj2SYlHSdpuaQdkh6UdKukt4xkWWNFX4dm7rURsS8wDzgGuLDD9YxIrd6SpIXA94DrgYOA2cDPgZ+W0bPrth6bpOcBK4HNwAsjYj/gDcB8YHKL19Wxtndq3fn29UPgR8DhwLOBdwKnjHB5XfVBVldE9O0f8DvgpML9jwPfLtzfC/gEsAn4PXAFMKkwfRGwGngY+C1wcv74fsDVwFbgHuAjwPh82tnAT/LbVwCfqKrpeuC8/PZBwL8BDwAbgfcU5rsY+Abw1Xz9b63Rvh8D/1Lj8e8AX85vDwBbgA8A2/LX5E3NvAaF574fuA/4CjANuCGveXt+++B8/o8CTwKPAUPA5/LHAzg8v/1F4DLg28AjZKH3vEI9rwLWAQ8B/0L2D/uMtufzfrX4ftaYPitf91l5+7YBHyxMPw64GdiRv5efAyYWpgfwLuA3wMb8sc+QhfTDwO3Aywvzj89f59/mbbsdOAS4KV/Wo/nr8t/y+V9Dtn3tAH4GvKhq230/cCfwJ2AChe05r31VXsfvgU/lj2/K1zWU/y2ksE3m8xwF/AfwYP7cD9R5/X4CXNbg9d1juXXe68uB5Xnb/y7fjsYX5v8r4M789jjggvz1+wNwLbB/23Oj3Svspr+qjexg4BfAZwrTPw0sA/Yn65n8O/CPhY3yIeCV+Zs5E3h+Pu1bwJXAPsABwK3A26s3JOAV+T+Y8vvTgJ1kYTku/6e6CJgIPBfYALw6n/diYBfwunzeSVVtexZZQJ1Qo91vAbbmtweAJ4BPkQXk8fkGfGQTr0HluR/LnzuJrLfxX/P1TwauA75VWPcgVSFX4x/pwfz1nQB8DViaT5tOFgJ/nU/7n/lrUC807wPe0uD9n5Wv+1/z2l9MFkAvyKcfC7w0X9cs4C7gvVV1/0f+2lQ+SN6cvwYTgPPzGvbOp72PbBs7ElC+vmdXvwb5/ZcA9wMLyML2LLLtda/CtruaLHQnFR6rbM83A3+T394XeGlVmycU1nU2u7fJyWQfEOcDe+f3F9R47epuX7WW2+C9fgh4Gdk2vDdZIL6yMP91wAX57fcCt5D9r+5F9j92Tdtzo1OB1Q1/+UY2RPapH8APgKn5NJGFR7GXs5DdPYorgUtrLHNG/o9X7JGeCayosYGK7JP/Ffn9twE/zG8vADZVLftC4P/mty8GbmrQtoPzNj2/xrSTgV357QGy4NunMP1a4P808RoMAI+Th0KdOuYB2wv3Bxk+ND9fmHYq8Kv89t8CNxemiexDp15o7iLv/deZPitf98GFx24Fzqgz/3uBb1bV/ZfDbGPbgRfnt9cBi+rMVx2alwMfrppnHXB8Ydv97zW250po3gR8CJhep831QvNM4I4m/ndm1tu+ai23wXv95arpHwG+kN+enG9/h+X37wJOLMx7YP4eTxiu3lb+eZ8mvC4iJpMFwPPJejMAzyH7NL0938m9A/hu/jhkn/C/rbG8w4A/A7YWnnclWY9zD5G980vJNlSAN5L1rCrLOaiyjHw5HyAL5YrNDdq1HXiKbMOqdiDZUPTpeSPi0cL9u8l6u8O9BgAPRMRjlTuSniXpSkl3S3qY7J93auL+qvsKt/9I1lMir+npNuev35YGy/kDtdvf1Pryg0g35AcJHwb+gd3bR8Ue74Gk8yXdlR902kG2q6bynHrbTC2HAedXvf+HkL0GNddd5RzgCOBXkm6T9Jom19tsjY22rxTVbVgC/LWkvchGFP8vIu7Opx0GfLPwetxF1tudQRs5NHMR8SOyT75P5A9tIxsqHxURU/O//SI7aATZm/28GovaTNbTnF543pSIOKrOqq8BXi/pMLLe5b8VlrOxsIypETE5Ik4tlt2gPY+SDdHeUGPy6WS96oppkvYp3D8UuLeJ16BWDeeTDT8XRMQUsl0QkPUKG9bchK1kPehsgZKK92v4PtmugpG6HPgVMCdvywfY3Y6Kp9sj6eVk+xlPB6ZFxFSy4WflOfW2mVo2Ax+tev+fFRHX1Fp3tYj4TUScSfZh/THgG/l7PNzr31SNEfFHsu2r0ev7KNmHLgCS/rzWoqqWu5bsQ/sUsk7EkqraTql6TfaOiHuGq7eVHJp7+jTwSknzIuIpsn1dl0o6AEDSTEmvzue9GniLpBMljcunPT8itpIdsf6kpCn5tOdJOr7WCiPiDrKDJp8HboyIHfmkW4GHJb1f0iRJ4yUdLek/JbTnAuAsSe+RNFnSNEkfIRtif6hq3g9Jmpj/478GuK6J16CWyWRBu0PS/sDfV03/Pdn+2ZH4NvBCSa/Ljxi/C6j1j1jx98BfSPrnyj+spMMlfVXS1CbWN5lsH+qQpOeTHRkebv4nyN7PCZIuAqYUpn8e+LCkOcq8SNKz82nVr8u/Au+QtCCfdx9J/0VSU0f9Jb1Z0nPy97CyTT2Z1/YU9d+DG4A/l/ReSXvl282COvP+b+BsSe+rtEPSiyUtzaf/HDhK0jxJe5PtUmrGEuA9ZB+41xUevwL4aN7BQNJzJC1qcpkt49AsiIgHgC+T7c+DrNewHrglH559n6wXRUTcSnZA5VKy3sSPyIYPkO17mwisJRvGfIPGw5hrgJMofKpGxJPAa8n2CW4k6/V9nmy412x7fgK8mmyYs5XsE/wY4D9HxG8Ks96X13kv2e6Bd0TEr4Z7Der4NNlBlW1kO+2/WzX9M2Q96+2SPttsW/L2bCPrOX+cbOg9l+wI8Z/qzP9bsg+IWcAaSQ+R9eRXke3HHs7/IuvtPEIWYl8fZv4byc5M+DXZa/0Yew4/P0W2v/h7ZGF8NdlrBVmgfCkfep4eEavI9nF/juy9WU+2j7BZJ5O1eYjsNT8jIh7Le4gfJTvtbIeklxafFBGPkB3cfC3ZdvEb4IRaK4iInwF/mf9tkPQgcBXZ0XAi4tfAJWTbzG/IjrY34xqy3WU/zN/zis+QHZT8nqRHyLaveoFemspRW+tTyr5B8tWIaDTM7UqSxpHt03xTRKzodD3WH9zTtDFF0qslTc0PFFT2Md7S4bKsjzg0baxZSHZ0dxvZEPJ1EbGzsyVZP/Hw3MwsgXuaZmYJHJpmZgm66ldpmjF9+vSYNWtW0nMeffRR9tlnn+Fn7HK90g5wW7pVr7RlJO24/fbbt0XEc4abb8yF5qxZs1i1alXScwYHBxkYGCinoDbqlXaA29KteqUtI2mHpLuHn8vDczOzJA5NM7MEDk0zswQOTTOzBA5NM7MEDk0zswQOTTOzBKWFpqQvSLpf0i/rTJekz0par+xa1C8pqxYzs1Yps6f5RbIfQq3nFGBO/reY7NICZmZdrbRvBEXETZJmNZhlEdmV6ILsV8GnSjowv1yEmY1RS1Zu4vrVbb1szzNMeepPlPXFpk5+jXIme14KYEv+2DNCU9Jist4oM2bMYHBwMGlFQ0NDyc/pRr3SDnBbulVqWwY37+Lme5/Y47F1258C4MhpnTtkMmnSk6W9J50Mzeqr+kGdK+VFxFVk1x5h/vz5kfqd0n7+Pm23cls6q15vcMeOnUydulfTy1m5Mbvy84LZ+z/92IKpsGjeTN644NDRFzpCZb4nnQzNLWTXWK44mOzCXmZ9p91D2pUbHwT2DLuRWDB7/44HZLt1MjSXAefml/tcADzk/Zk2Fg1u3sXlV948qmW0KsSaVS/ssh7awrbUMFaVFpqSKpfhnC5pC9k1qP8MICKuILvM56lklyb9I9nlcM3GjErvcOXGx4EHRxV4/dhjG6vKPHp+5jDTA3hXWes3K3vIW+kdHjltHGcNHOXA6xNj7keIrX+lhmDZQ95K7/CgnRsYcGD2DYemtVUl+Hbs2Mnl69L2A6aGYLuGvIODG0pdvnUXh6a1xe79f7uHtKm838+6gUPTSlUdlnsMaX2U1sYgh6aVZsnKTXzgm78AntlL9JDWxiqHppWmctDmH/7qhR5SW8/w72laqRbM3t+BaT3FoWlmlsChaWaWwKFpZpbAoWlmlsBHz62lil91XLv1YeYeOKXDFZm1lnua1lLXr76HtVsfBmDugVNYNG9mhysyay33NK3l5h44ha+/3d/2sd7k0LSWqAzLPSS3XufhubVEMTA9JLde5p6mtYyH5dYP3NM0M0vgnqaNivdlWr9xT9NGxfsyrd+4p2nJap3A7n2Z1i/c07RkPoHd+pl7mjYi7l1av3Jo2rCqL53rgz7Wzzw8t4Yq1/mpXBgNPCS3/uaepjXk6/yY7cmhaTUVz7/0dX7MdvPw3Gry+ZdmtbmnaXuo/oaPj5Cb7ck9TduDe5hmjbmn2efqnU7kHqZZbe5p9rnit3vApxOZDcc9zT7lfZdmI+PQ7APVQ3Dg6ZPVF8ze3z1LswQOzR5X+UYPZAFZUQlLn39plsah2eP8jR6z1nJo9ih/o8esHA7NHlQ9JPc+S7PWcWj2IA/Jzcrj8zR7zJKVm1i58UEPyc1KUmpoSjpZ0jpJ6yVdUGP6oZJWSLpD0p2STi2znl5XHJZ7SG5WjtKG55LGA5cBrwS2ALdJWhYRawuz/R1wbURcLmkusByYVVZNvapy0Kdy7qWH5WblKXOf5nHA+ojYACBpKbAIKIZmAJXrJuwH3FtiPT2reJTc516alavM0JwJbC7c3wIsqJrnYuB7kt4N7AOcVGI9Pc1fhTRrD0VEOQuW3gC8OiLemt//G+C4iHh3YZ7z8ho+KWkhcDVwdEQ8VbWsxcBigBkzZhy7dOnSpFqGhobYd999R9WeblCrHYObd/HFNY9z5LRxXLhgUocqS9cr7wm4Ld1oJO044YQTbo+I+cPOGBGl/AELgRsL9y8ELqyaZw1wSOH+BuCARss99thjI9WKFSuSn9ONarXj9Ct+Foe9/4b42i13t7+gUeiV9yTCbelGI2kHsCqayLYyj57fBsyRNFvSROAMYFnVPJuAEwEkvQDYG3igxJp6ik8vMmu/0kIzIp4AzgVuBO4iO0q+RtIlkk7LZzsfeJuknwPXAGfniW/D8OlFZp1R6jeCImI52WlExccuKtxeC7yszBp6jU8vMussf41yDMkO+uz5nXIHpll7OTTHgN29y8cB9y7NOsmh2UVq/cI67P6V9SOnjeOsgaMcmGYd5NDsEvV+Yb1yf9G8mRy0cwMDDkyzjnJodolmfs5tcHBDO0sysxocmh1QaxjuX1g3Gxscmm1SDMrilSArfL1xs7HBodkG1fsrfbqQ2djl0GwDX37CrHf4chdt4v2VZr3BoWlmlsChWbLKLxGZWW9waJassj/TR8bNeoNDs0T+vUuz3uOj5yWo/vk29zLNeodDs0Xqnbzu8zHNeotDc5Sqe5U+ed2stzk0R6hWWDoozXqfQ3OErl99z9M/suGwNOsfDs0mVf8y0dqtDzP3wCl8/e0LO1iVmbWbTzlqUqVnWeFfJTLrT+5pJnDP0szc0zQzS+DQNDNL4OH5MCoHgCoHfsysvzk066h3HqaZ9TeHZh0+D9PManFoFhTPxfR5mGZWiw8EFRTPxfR5mGZWi3uaVdy7NLNG3NM0M0vg0DQzS+DQNDNL4NA0M0vg0DQzS+DQNDNL4NA0M0vg0DQzS+DQNDNL4NA0M0vg0DQzS1BqaEo6WdI6SeslXVBnntMlrZW0RtKSMusxMxut0n6wQ9J44DLglcAW4DZJyyJibWGeOcCFwMsiYrukA8qqx8ysFcrsaR4HrI+IDRHxOLAUWFQ1z9uAyyJiO0BE3F9iPWZmo1bmT8PNBDYX7m8BFlTNcwSApJ8C44GLI+K71QuStBhYDDBjxgwGBweTChkaGmrqOTt27ARIXn67NNuOscBt6U690pYy21FmaKrGY1Fj/XOAAeBg4MeSjo6IHXs8KeIq4CqA+fPnx8DAQFIhg4ODNPOcy9fdDMDAQHf+nmaz7RgL3Jbu1CttKbMdZQ7PtwCHFO4fDNxbY57rI2JXRGwE1pGFqJlZVyozNG8D5kiaLWkicAawrGqebwEnAEiaTjZc31BiTWZmo1JaaEbEE8C5wI3AXcC1EbFG0iWSTstnuxH4g6S1wArgfRHxh7JqMjMbrVKvERQRy4HlVY9dVLgdwHn5n5lZ1/M3gszMEjg0zcwSODTNzBI4NM3MEjg0zcwSODTNzBI4NM3MEjg0zcwSODTNzBI4NHNLVm5i5cYHO12GmXU5h2bu+tX3ALBo3swOV2Jm3Sw5NCWNl/SmMorptAWz9+eNCw7tdBlm1sXqhqakKZIulPQ5Sa9S5t1kP912evtKNDPrHo1+5egrwHbgZuCtwPuAicCiiFjdhtrMzLpOo9B8bkS8EEDS54FtwKER8UhbKjMz60KN9mnuqtyIiCeBjQ5MM+t3jXqaL5b0MLsvkDapcD8iYkrp1ZmZdZm6oRkR49tZiJnZWFA3NCXtDbwDOBy4E/hCft0fM7O+1Wif5peA+cAvgFOBT7alIjOzLtZon+bcwtHzq4Fb21OSmVn3avbouYflZmY07mnOy4+WQ3bE3EfPzazvNQrNn0fEMW2rxMxsDGg0PI+2VdFh/lk4M2tWo57mAZLOqzcxIj5VQj0d4Z+FM7NmNQrN8cC+7P5GUE/zz8KZWTMahebWiLikbZWYmY0BjfZp9kUP08wsRaPQPLFtVZiZjRF1QzMifDjZzKyKL6xmZpbAoWlmlsChaWaWwKFpZpbAoWlmlsChaWaWwKFpZpbAoWlmlsChaWaWoNTQlHSypHWS1ku6oMF8r5cUkuaXWY+Z2WiVFpqSxgOXAacAc4EzJc2tMd9k4D3AyrJqMTNrlTJ7mscB6yNiQ0Q8DiwFFtWY78PAx4HHSqzFzKwlygzNmcDmwv0t+WNPk3QMcEhE3FBiHWZmLdPoR4hHq9bvcT593SFJ44BLgbOHXZC0GFgMMGPGDAYHB5MKGRoaavicHTt2AiQvt92Ga8dY4rZ0p15pS6ntiIhS/oCFwI2F+xcCFxbu7wdsA36X/z0G3AvMb7TcY489NlKtWLGi4fTTr/hZnH7Fz5KX227DtWMscVu6U6+0ZSTtAFZFE9lW5vD8NmCOpNmSJgJnAMsKYf1QREyPiFkRMQu4BTgtIlaVWJOZ2aiUFpoR8QRwLnAjcBdwbUSskXSJpNPKWm8qX77XzFKUuU+TiFgOLK967KI68w6UWUs9vnyvmaXwN4Lw5XvNrHkOTTOzBA5NM7MEDk0zswQOTTOzBA5NM7MEfR2aPkfTzFL1dWj6HE0zS9XXoQk+R9PM0vR9aJqZpXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompklcGiamSXo29BcsnITKzc+2OkyzGyM6dvQvH71PQAsmjezw5WY2VhSamhKOlnSOknrJV1QY/p5ktZKulPSDyQdVmY91RbM3p83Lji0nas0szGutNCUNB64DDgFmAucKWlu1Wx3APMj4kXAN4CPl1WPmVkrlNnTPA5YHxEbIuJxYCmwqDhDRKyIiD/md28BDi6xHjOzUZtQ4rJnApsL97cACxrMfw7wnVoTJC0GFgPMmDGDwcHBpEKGhoae8ZwdO3YCJC+rk2q1Y6xyW7pTr7SlzHaUGZqq8VjUnFF6MzAfOL7W9Ii4CrgKYP78+TEwMJBUyODgINXPuXzdzQAMDCxMWlYn1WrHWOW2dKdeaUuZ7ShzeL4FOKRw/2Dg3uqZJJ0EfBA4LSL+VGI9T/PpRmY2UmWG5m3AHEmzJU0EzgCWFWeQdAxwJVlg3l9iLXvw6UZmNlKlhWZEPAGcC9wI3AVcGxFrJF0i6bR8tn8G9gWuk7Ra0rI6i2s5n25kZiNR5j5NImI5sLzqsYsKt08qc/1mZq3Wt98IMjMbCYemmVkCh6aZWQKHpplZgr4LTZ+jaWaj0Xeh6XM0zWw0+io0K71Mn6NpZiPVV6HpXqaZjVZfhSb4m0BmNjp9F5pmZqPh0DQzS+DQNDNL4NA0M0vg0DQzS+DQNDNL4NA0M0vg0DQzS+DQNDNL4NA0M0vg0DQzS+DQNDNL4NA0M0vg0DQzS+DQNDNL4NA0M0vQN6HpC6qZWSv0TWj6Uhdm1gp9E5rgS12Y2ej1RWgObt7lobmZtURfhObN9z4BeGhuZqPXF6EJHpqbWWv0fGguWbmJdduf6nQZZtYjej40fdTczFqp50MT4Mhp4zw0N7OW6IvQNDNrFYemmVkCh6aZWQKHpplZAoemmVkCh6aZWYJSQ1PSyZLWSVov6YIa0/eS9PV8+kpJs8qsx8xstEoLTUnjgcuAU4C5wJmS5lbNdg6wPSIOBy4FPlZWPWZmrVBmT/M4YH1EbIiIx4GlwKKqeRYBX8pvfwM4UZJKrMnMbFTKDM2ZwObC/S35YzXniYgngIeAZ7eyiLkHTeHQKd51a2atMaHEZdfqMcYI5kHSYmAxwIwZMxgcHGy6iOMnw9Ahu5Ke062GhoZ6oh3gtnSrXmlLme0oMzS3AIcU7h8M3Ftnni2SJgD7Ac/4teCIuAq4CmD+/PkxMDCQVMjg4CCpz+lGvdIOcFu6Va+0pcx2lDluvQ2YI2m2pInAGcCyqnmWAWflt18P/DAintHTNDPrFqX1NCPiCUnnAjcC44EvRMQaSZcAqyJiGXA18BVJ68l6mGeUVY+ZWSuUOTwnIpYDy6seu6hw+zHgDWXWYGbWSj6sbGaWwKFpZpbAoWlmlsChaWaWwKFpZpbAoWlmlsChaWaWQGPtCziSHgDuTnzadGBbCeW0W6+0A9yWbtUrbRlJOw6LiOcMN9OYC82RkLQqIuZ3uo7R6pV2gNvSrXqlLWW2w8NzM7MEDk0zswT9EppXdbqAFumVdoDb0q16pS2ltaMv9mmambVKv/Q0zcxaoqdCs1cuGdxEO86TtFbSnZJ+IOmwTtTZjOHaUpjv9ZJCUtceuW2mLZJOz9+bNZKWtLvGZjSxfR0qaYWkO/Jt7NRO1NkMSV+QdL+kX9aZLkmfzdt6p6SXjHqlEdETf2Q/dPxb4LnARODnwNyqef4HcEV++wzg652ue4TtOAF4Vn77nd3Yjmbbks83GbgJuAWY3+m6R/G+zAHuAKbl9w/odN0jbMdVwDvz23OB33W67gbteQXwEuCXdaafCnyH7HpkLwVWjnadvdTT7JVLBg/bjohYERF/zO/eQnb9pW7UzHsC8GHg48Bj7SwuUTNteRtwWURsB4iI+9tcYzOaaUcAU/Lb+/HMa3t1jYi4iRrXFStYBHw5MrcAUyUdOJp19lJodsUlg1ugmXYUnUP2SdqNhm2LpGOAQyLihnYWNgLNvC9HAEdI+qmkWySd3LbqmtdMOy4G3ixpC9mVF97dntJKkfr/NKxSL3fRZi27ZHCHNV2jpDcD84HjS61o5Bq2RdI44FLg7HYVNArNvC8TyIboA2S9/x9LOjoidpRcW4pm2nEm8MWI+KSkhWTX8To6Ip4qv7yWa/n/fC/1NFMuGUyjSwZ3WDPtQNJJwAeB0yLiT22qLdVwbZkMHA0MSvod2T6nZV16MKjZ7ev6iNgVERuBdWQh2k2aacc5wLUAEXEzsDfZd7nHoqb+n1L0Umj2yiWDh21HPqS9kiwwu3G/WUXDtkTEQxExPSJmRcQssv2zp0XEqs6U21Az29e3yA7SIWk62XB9Q1urHF4z7dgEnAgg6QVkoflAW6tsnWXA3+ZH0V8KPBQRW0e1xE4f/WrxkbRTgV+THR38YP7YJWT/iJC9+dcB64Fbged2uuYRtuP7wO+B1fnfsk7XPNK2VM07SJcePW/yfRHwKWAt8AvgjE7XPMJ2zAV+SnZkfTXwqk7X3KAt1wBbgV1kvcpzgHcA7yi8J5flbf1FK7YvfyPIzCxBLw3PzcxK59A0M0vg0DQzS+DQNDNL4NA0M0vg0LQxS9KTklYX/mZJGpD0UP4LPXdJ+vt83uLjv5L0iU7Xb2NTL32N0vrPzoiYV3wg/7m/H0fEayTtA6yWVPlee+XxScAdkr4ZET9tb8k21rmnaT0rIh4FbgeeV/X4TrKTtkf1ww3WnxyaNpZNKgzNv1k9UdKzyb7Pvqbq8Wlk3wm/qT1lWi/x8NzGsmcMz3Mvl3QH8BTwTxGxRtJA/vidwJH54/e1sVbrEQ5N60U/jojX1Htc0hHAT/J9mqvbXZyNbR6eW9+JiF8D/wi8v9O12Njj0LR+dQXwCkmzO12IjS3+lSMzswTuaZqZJXBompklcGiamSVwaJqZJXBompklcGiamSVwaJqZJXBompkl+P/BxCDn/hLyIQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,5))\n",
    "plt.plot(FPR, TPR)\n",
    "plt.xlabel('FPR')\n",
    "plt.ylabel('TPR')\n",
    "plt.title('Receiver Operating Characteristic Curve')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}