AUC, ROC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate synthetic dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_classification(n_samples=2000, n_classes=2, \n",
    "                           n_features=10, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = np.random.RandomState(0)\n",
    "n_samples, n_features = X.shape\n",
    "X = np.c_[X, random_state.randn(n_samples, 200*n_features)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None, var_smoothing=1e-09)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb = GaussianNB()\n",
    "nb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction probabilities "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_probs = [0 for _ in range(len(y_test))]   # for random sample \n",
    "nb_probs= nb.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb_probs = nb_probs[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate AUC-ROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'roc_auc_score' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-e99597366137>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mr_auc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mroc_auc_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr_probs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mnb_auc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mroc_auc_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnb_probs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mnb_auc\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'roc_auc_score' is not defined"
     ]
    }
   ],
   "source": [
    "r_auc = roc_auc_score(y_test, r_probs)\n",
    "nb_auc = roc_auc_score(y_test, nb_probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random (chance) Prediction: AUROC = 0.500\n",
      "Naive Bayes: AUROC = 0.993\n"
     ]
    }
   ],
   "source": [
    "print('Random (chance) Prediction: AUROC = %.3f' % (r_auc))\n",
    "print('Naive Bayes: AUROC = %.3f' % (nb_auc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_fpr, r_tpr, _ = roc_curve(y_test, r_probs)\n",
    "nb_fpr, nb_tpr, _ = roc_curve(y_test, nb_probs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3gVZfbA8e9J6BBqABEICSH0EEoAQRQwNAtFRMFe1h+rgr1hXUV3l7WhrKzYsCKwKgh2AWlqkCIYmvQAAYTQAoSEtPP7Y27uhhCSG5Kbm+Sez/Pchzv9TELmzLwzc15RVYwxxvivAF8HYIwxxrcsERhjjJ+zRGCMMX7OEoExxvg5SwTGGOPnLBEYY4yfs0RgjI+ISLyI9PN1HMZYIjB+wXXQTRGREyLyp4i8LyI1cs3TU0R+FJHjIpIkIl+KSNtc89QUkVdFZJdrXVtdw8Fn2a6KSLJr3j0i8oqIBBYy9j4iklD4vTbGM5YIjD8ZrKo1gI5AJ+Cx7Aki0gP4AZgDnA+EAb8DP4tIc9c8lYAFQDtgEFAT6AkcArrls90o13ZjgOuA/yve3TKmaCwRGL+jqn8C3+MkhGwvAB+q6muqelxVD6vqk8Ay4BnXPDcBIcCVqrpBVbNU9YCqPqeq33iw3T+ApUD73NNEpLLrymKv6/Oqa1x14FvgfNdVxQkROb8o+29MbpYIjN8RkSbApcBW13A1nDP7T/OY/b9Af9f3fsB3qnriHLfbFrgIWJ3H5CeAC3CSUxTOFcaTqprsinWvqtZwffaey/aNORtLBMaffCEix4HdwAHgb67xdXH+Fvblscw+ILv9v95Z5inIbyJyBPgSeAd4L495rgfGu64wEoFngRvPYVvGFJolAuNPhqlqENAHaM3/DvBHgCygUR7LNAIOur4fOss8BemsqnVUNVxVn1TVrDzmOR/YmWN4p2ucMV5nicD4HVVdDLwPvOQaTgZigavzmP0anBvEAPOBga52++K2F2iWYzjENQ7ASgQbr7JEYPzVq0B/Ecm+YTwOuFlE7hGRIBGpIyLPAz1wmmkAPsJpVvpcRFqLSICI1BORx0XksiLGMx14UkTqux5FfRr42DVtP1BPRGoVcRvG5MkSgfFLrnb4D4GnXMM/AQOB4Tj3AXbiPGLaS1W3uOY5hXPD+A9gHnAMWI7TxPRrEUN6HlgJxAFrgd9c47KfNpoObBeRo/bUkCluYh3TGGOMf7MrAmOM8XOWCIwxxs9ZIjDGGD9nicAYY/xcBV8HUFjBwcEaGhrq6zCMMaZMWbVq1UFVrZ/XtDKXCEJDQ1m5cqWvwzDGmDJFRHaebZo1DRljjJ+zRGCMMX7OEoExxvg5SwTGGOPnLBEYY4yf81oiEJGpInJARNadZbqIyCRX599xItLZW7EYY4w5O29eEbyP08H32VwKRLg+o4E3vBiLKU6JsbD+n86/xpiS4cW/O6+9R6CqS0QkNJ9ZhuJ0Fq7AMhGpLSKNVPVcugIs2xJj4cAiaNAH6vfwdTT5S4yF+ReDZgACdaKgopXJN8abMk8lEZAUh6AQWAUuWVCsxwpfvlDWGKeTj2wJrnFnJAIRGY1z1UBISEiJBFckhTmwl7UD67HNrlgBFFL2l+54jSnjklLSST7yJ40Cs0CArDTn+FJOEoHkMS7PzhFU9S3gLYDo6Gjfd6CQ34G+sAf2sn5gbTIUulmrnjHFLSklnX9+s5EZK3ZzWaMd/LvhwwRqOgRUco49xciXiSABaJpjuAn/66O19Mk++FeqByvHnP1AX9QDe2k/sCbGwoI+kJUOARUh7CZfR2RMuZOZpVz1xi9sTzzBX3s35/5+gwg82sVrTci+TARzgbEiMgPoDiSVyP2Bc2mPP+0sPycPDvQFHdjL2oG1fg+IWVR27mkYU4YcSU6jdrWKBAYIDw1oxfm1q9ChSW1nYv0eXvt781oiEJHpQB8gWEQSgL8BFQFUdQrwDXAZsBU4CdzqrVjcEmNhQW/noEsA1Ong2dn6aWf5ueQ+0Bf2wF4WD6xe/A9pjD9SVb5Ys4dnv9zAo4Nac223EAa1P6/Etu/Np4auLWC6AmO8tf08HVjkSgIAWZCWdI7t8QGA5n2gP5cDux1YjfFbe4+m8MTstSzclEinkNpEN6tT4jGUuTLURdKgD85BPAsCq0LPaZ4dgHOf5Xf5N6QdOvuB3g7sxhgPzFmzhydmryMzS3n6irbc3DOUwIC8nqPxLv9KBPV7OM1BaUmeJ4Hs5cpa840xptSrVbUiHZvW5p/DI2lat5rP4vCvRABOU1DFWoU/mNtZvjGmiDIys3j3px2kZ2Yx9pII+rRqQO+W9REp+auAnPwvERhjjA9s2HuMRz+PY+2eJC7v0AhVRUR8ngTAEoExxnjVqYxMXv9xK28s2kbtahX5z/WdubT9eaUiAWSzRGCMMV4Uf/AkUxZvY0jH83nq8rbUqV7J1yGdwRKBMcYUs+RTGczbsJ9hnRrT6rwgFjzQh5B6vrsZXBBLBMYYU4yWbknksVlr2XM0hfaNa9KiQVCpTgJgicAYY4pF0sl0/v7NBv67MoHmwdWZOboHLRoE+Tosj1giMMaYIsrMUq6a8gs7DiZzV59w7omJoErFQF+H5TFLBMYYc44OJ6dRu6pTJO7hga1oXLsq7RuXoTLyLtZ5vTHGFJKq8vmqBPq+tIgZK5z+tQa2O69MJgGwKwJjjCmUhCMneXz2OpZsTqRLszp0C6vr65CKzBKBMcZ4aPbqBJ6cvQ4Fnh3SjhsvaEaAD4rEFTdLBMYY46G61SvTJbQu/7iyPU3qlO5HQgvDEoExxpxFemYWby/dTkamck9MBL1b1ufiiOBSVR6iOFgiMMaYPKzbk8Sjn8exfu8xBkedX6qKxBU3SwTGGJNDanomkxZs4c0l26lTrRJTbujMoPaNfB2WV1kiMMaYHHYeOsnbS7czvFNjnry8LbWqVfR1SF5nicAY4/eST2Xw/fo/Gd65Ca3OC+LHB/v4tMewkmaJwBjj1xZvTuTxWWvZm5RChya1aNEgyK+SAFgiMMb4qSPJaTz39QZm/baH8PrV+fSvZadIXHGzRGCM8TvZReJ2HjrJ2L4tGHtJizJVJK64WSIwxviNQydOUadaJQIDhHGDWtO4TlXanV826wMVJys6Z4wp91SV/67cTd+XFjF9xS4ABrQ7z5KAi10RGGPKtd2HT/L47LUs3XKQbqF16dG8nq9DKnUsERhjyq1ZvyXw5BfrEOC5Ye25vltIuSgSV9wsERhjyq3gGpXpFlaXv18ZSePaVX0dTqllicAYU26kZ2bx5uJtZGbBvf0iuLhlfS5uWd/XYZV6lgiMMeXCuj1JPPxZHBv3HWNox/8ViTMFs0RgjCnTUtMzeXX+Ft5eup261Svx5o1dGNjuPF+HVaZ49fFRERkkIptEZKuIjMtjeoiILBSR1SISJyKXeTMeY0z5s+vwSd79aTsjOjdh/v29LQmcA69dEYhIIDAZ6A8kACtEZK6qbsgx25PAf1X1DRFpC3wDhHorJmNM+XA8NZ3v1v3J1dFNadkwiIUP9SlXPYaVNG82DXUDtqrqdgARmQEMBXImAgVqur7XAvZ6MR5jTDmw8I8DPDF7LX8eS6VTSG1aNAiyJFBE3kwEjYHdOYYTgO655nkG+EFE7gaqA/3yWpGIjAZGA4SEhBR7oMaY0u9wchrPfbWB2av3ENGgBp/d2dNvi8QVN28mgrxu12uu4WuB91X1ZRHpAXwkIu1VNeu0hVTfAt4CiI6Ozr0OY0w5l5mljHjjF3YdPsk9MRGM6RtO5Qr+WySuuHkzESQATXMMN+HMpp+/AIMAVDVWRKoAwcABL8ZljCkjEo+fol51p0jc45e1oXGdqrRpVLPgBU2hePOpoRVAhIiEiUglYBQwN9c8u4AYABFpA1QBEr0YkzGmDFBVZq7YxSUvL+KT5U6RuH5tG1oS8BKvXRGoaoaIjAW+BwKBqaq6XkTGAytVdS7wIPC2iNyP02x0i6pa048xfmzXoZOMmxXHL9sO0T2sLr1aBPs6pHLPqy+Uqeo3OI+E5hz3dI7vG4ALvRmDMabs+GxVAk99sY7AAOHvV7bn2q5WJK4k2JvFxphSo2HNyvQMr8fzV7anUS0rEldSLBEYY3wmLSOLNxZtI0uV+/u35KKI+lwUYUXiSpolAmOMT/y++yiPfBbHpv3HGd6psRWJ8yFLBMaYEpWSlskr8zbx7k87aBBUhXduiqZf24a+DsuvWSIwxpSo3UdO8sEvOxnVLYRxl7amZpWKvg7J71kiMMZ43TFXkbhrXEXiFj3ch/Otx7BSwxKBMcarfvxjP4/PWseB46l0DqlDiwY1LAmUMpYIjDFecejEKcZ/tYE5a/bSqmEQU27sQosGNXwdlsmDJQJjTLHLzFKunhLL7iMnub9fS+7sE06lCl7tB8sUgUeJwFUrKERVt3o5HmNMGXbgeCrB1SsTGCA8cXkbmtSpRqvzrFR0aVdgihaRy4G1wDzXcEcRme3twIwxZUdWljLt151c8tJiprmKxMW0aWhJoIzw5IpgPE6HMgsBVHWNiLTwalTGmDIj/mAy42bFsWz7YXqG16O3vRlc5niSCNJV9WiuN/6sQqgxhv+u3M1TX6yjUmAAE4ZHMrJrU3s7uAzyJBFsFJFrgAARCQPuBZZ5NyxjTFnQuHZVLm5Zn+eGtue8WlV8HY45R54kgrHA00AWMAunf4HHvBmUMaZ0OpWRyX8WbkNVeWBAKy5sEcyF1l9AmedJIhioqo8Cj2aPEJHhOEnBGOMnVu86wqOfx7F5/wmu6tzEisSVI54kgic586D/RB7jjDHl0Mm0DF7+YTNTf97BeTWrMPWWaC5pbUXiypOzJgIRGYjTsXxjEXklx6SaOM1Exhg/sOdICh8t28n13UN4dFBrgqxIXLmT3xXBAWAdkAqszzH+ODDOm0EZY3wrKSWdb9fuY1S3ECIaBrH44T7WY1g5dtZEoKqrgdUiMk1VU0swJmOMD/2w/k+e/GIdh5LTiA6tS4sGNSwJlHOe3CNoLCJ/B9oC7ufDVLWl16IyxpS4gydO8czc9XwVt4/W5wXxzs3RViTOT3iSCN4HngdeAi4FbsXuERhTrmRmKSPe+IW9R1N5aEBL/to7nIqBViTOX3iSCKqp6vci8pKqbgOeFJGl3g7MGON9+4+lUr+GUyTub4Pb0aROVSIaWn0gf+NJyj8lzsPC20TkDhEZDDTwclzGGC/KylI+WraTmJcXM+3XnQD0bd3AkoCf8uSK4H6gBnAP8HegFnCbN4MyxnjP9sQTjJu1luU7DtOrRTB9Wtl5nb8rMBGo6q+ur8eBGwFEpIk3gzLGeMfMFbt4es56KlcI4IURHbi6SxN7O9jknwhEpCvQGPhJVQ+KSDucUhOXAJYMjCljmtSpRp9WTpG4BjWtSJxx5Pdm8T+Bq4DfcW4Qz8apPPov4I6SCc8YUxSnMjL59wKnY8GHBlqROJO3/K4IhgJRqpoiInWBva7hTSUTmjGmKFbtPMwjn8WxLTGZa6KtSJw5u/wSQaqqpgCo6mER+cOSgDGlX/KpDF78fhMfxMZzfq2qfHBbN3q3tF7DzNnllwiai0h2hVEBQnMMo6rDC1q5iAwCXgMCgXdUdUIe81wDPIPT69nvqnqd5+EbY3LbezSFT5bv4qYLmvHwoNbUqOzJw4HGn+X3P+SqXMOvF2bFIhIITAb6AwnAChGZq6obcswTgdPJzYWqekRE7Dk2Y85B0sl0vl67j+u6O0Xilj7Sl4Z2M9h4KL+icwuKuO5uwFZV3Q4gIjNw7jtsyDHP/wGTVfWIa5sHirhNY/zOd+v+5Kk56zicnEb35nUJr1/DkoApFG8WE2kM7M4xnOAal1NLoKWI/Cwiy1xNSWcQkdEislJEViYmJnopXGPKlgPHU7lr2iru+HgV9WtUZs6YCwmvb0XiTOF5s/Ewr8cTNI/tRwB9cN5LWCoi7VX16GkLqb4FvAUQHR2dex3G+J3MLOWaKbHsTUrl4YGtGH1xcysSZ86Zx4lARCqr6qlCrDsBaJpjuAnOI6i551mmqunADhHZhJMYVhRiO8b4jX1JKTQMquIUiRvSjqZ1qlmpaFNkBZ5CiEg3EVkLbHENR4nIvz1Y9wogQkTCRKQSMAqYm2ueL4C+rvUG4zQVbS9E/Mb4haws5f2fdxDz8mI+zi4S16qBJQFTLDy5IpgEXIFz0EZVfxeRvgUtpKoZIjIW+B7n8dGpqrpeRMYDK1V1rmvaABHZAGQCD6vqoXPcF2PKpa0HTjDu8zhW7jzCxS3rc0lre7jOFC9PEkGAqu7M9UZipicrV9VvgG9yjXs6x3cFHnB9jDG5zFi+i6fnrqdqxUBevjqK4Z0b29vBpth5kgh2i0g3QF3vBtwNbPZuWMYYgJB61ejXpgHPDmlP/aDKvg7HlFOeJII7cZqHQoD9wHzXOGNMMUtNz2TSgi0APDKoNT3Dg+kZbkXijHd5kggyVHWU1yMxxs+tjD/MI5/HsT0xmVFdm1qROFNiPEkEK1yPdc4EZqnqcS/HZIxfOXEqgxe/+4MPl+2kce2qfHhbNy62InGmBHnSQ1m4iPTEefzzWRFZA8xQ1Rlej84YP/BnUgozVuzm5h6hPDywFdWtSJwpYR69iqiqv6jqPUBn4BgwzatRGVPOHUlO46NlzvsALRo4ReKeGdLOkoDxiQL/14lIDZxicaOANsAcoKeX4zKmXFJVvl33J0/PWcfRk+n0DK9HeP0a1m2k8SlPTj/WAV8CL6jqUi/HY0y5deBYKk/NWcf36/cT2bgWH97W3YrEmVLBk0TQXFWzvB6JMeVYZpZy9Zux/JmUymOXtuYvvcKoYEXiTCmRX+f1L6vqg8DnInJGxU9Peigzxt/tPZrCeTWdInHjh7anaZ2qNLerAFPK5HdFMNP1b6F6JjPGOFcAH8bG88J3m3jsstbc1CPU+g02pVZ+PZQtd31to6qnJQNXMbmi9mBmTLm09cBxHvksjt92HaVPq/rEtGno65CMyZcnjZS35THuL8UdiDHlwSe/7uKy135ix8FkJo6M4r1butK4dlVfh2VMvvK7RzAS55HRMBGZlWNSEHA076WM8W+hwdUY0K4hzwxpR3ANKxJnyob87hEsBw7h9Cw2Ocf448BqbwZlTFmRmp7JxPmbEYRxl1qROFM25XePYAewA6faqDEml1+3H2LcrLXsOJjM9d1DrEicKbPyaxparKq9ReQIp3c6Lzh9ytT1enTGlELHU9P513d/8PGyXYTUrcYnt3enZwu7CjBlV35NQ9ndUdr/cGNy2H/sFJ+tSuD2XmE8MKAl1SpZfSBTtuXXNJT9NnFTYK+qpolIL6AD8DFO8Tlj/MLh5DS+jtvLjT1CadGgBksfucR6DDPlhiePj36B001lOPAhTuG5T7walTGlhKry5e976f/KYsZ/tYHtiScALAmYcsWTa9osVU0XkeHAq6o6SUTsqSFT7u0/lsoTs9cxf+N+OjSpxbQR3a08hCmXPOqqUkSuBm4EhrnGVfReSMb4XmaWco2rSNwTl7Xh1gtDrUicKbc8SQS3AXfhlKHeLiJhwHTvhmWMbyQcOUmjWlUJDBCeG9qekLrVCA2u7uuwjPGqAk9xVHUdcA+wUkRaA7tV9e9ej8yYEpSZpbyzdDv9XlnMx66ewy5uWd+SgPELnvRQdhHwEbAH5x2C80TkRlX92dvBGVMSNv15nEc+j+P33UeJad2AAe2sSJzxL540DU0ELlPVDQAi0gYnMUR7MzBjSsLHy3by7JfrCapSkddGdWRI1Pn2drDxO54kgkrZSQBAVTeKSCUvxmSM12WXg2jRoAaXRTbi6SvaUs+KxBk/5Uki+E1E3sS5CgC4His6Z8qolLRMXpm3iYAA4bFL23BB83pc0Lyer8Myxqc8eR7uDmAb8AjwKLAd+Ks3gzLGG2K3HWLQa0t4e+kOTp7KRPWMHliN8Uv5XhGISCQQDsxW1RdKJiRjitex1HT++c0fTF++i2b1qvHJ/3W3UtHG5JBf9dHHcXoi+w3oKiLjVXVqiUVmTDE5cOwUX6zew+iLm3N/v5ZUrRTo65CMKVXyaxq6HuigqlcDXYE7C7tyERkkIptEZKuIjMtnvhEioiJiTyKZYnHoxCne/3kHAC0a1OCnR/vy+GVtLAkYk4f8moZOqWoygKomikih3q8XkUCcns36AwnAChGZm/MJJNd8QTgvrP1aqMiNyYOqMvf3vTwzdz0nTmVwccv6NK9fw54IMiYf+SWC5jn6KhYgPGffxao6vIB1dwO2qup2ABGZAQwFNuSa7zngBeChwgRuTG57j6bw5Bfr+PGPA3RsWpsXRnSwInHGeCC/RHBVruHXC7nuxsDuHMMJQPecM4hIJ6Cpqn4lImdNBCIyGhgNEBISUsgwjD/IyMxi1FvLSDx+iqeuaMstPUMJDLAXw4zxRH4d0ywo4rrz+it0P6/namqaCNxS0IpU9S3gLYDo6Gh75s+47T58kvNrV6VCYAD/uDKSkLrVCKlXzddhGVOmeLOubgJO72bZmgB7cwwHAe2BRSISD1wAzLUbxsYTGZlZvLVkG/1eWcxHsfEA9IoItiRgzDnwZmerK4AIV9nqPcAo4LrsiaqaRI7+kEVkEfCQqq70YkymHNi47xiPfh5HXEIS/ds25NLIRr4OyZgyzeNEICKVVfWUp/OraoaIjAW+BwKBqaq6XkTGAytVdW7hwzX+7qPYeJ79cgO1qlbk9es6cXlkIysSZ0wReVKGuhvwLlALCBGRKOB2Vb27oGVV9Rvgm1zjnj7LvH08Cdj4p+wicS0bBjE46nyeuqItdatb7UNjioMnVwSTgCtwOrFHVX8Xkb5ejcoYl5NpGbz0/WYqBAqPX9aG7s3r0d2KxBlTrDy5WRygqjtzjcv0RjDG5PTz1oMMfHUJU3/eQVpGlhWJM8ZLPLki2O1qHlLX28J3A5u9G5bxZ0kp6fzj643MXLmbsODq/PevPegWVtfXYRlTbnmSCO7EaR4KAfYD8zmHukPGeOrgiVN8GbeXO3qHc1+/CKpUtPpAxnhTgYlAVQ/gPPppjNckHj/Fl7/v5bZeYYTXr8FPj15iN4ONKSGePDX0NjneCM6mqqO9EpHxK6rKF2v28OyXGzh5KpO+rRsQFlzdkoAxJciTpqH5Ob5XAa7k9BpCxpyTPUdTeGL2WhZtSqRziFMkLiy4uq/DMsbveNI0NDPnsIh8BMzzWkTGLzhF4mI5dCKNZwa35cYeViTOGF85lxITYUCz4g7E+Iddh07SuI5TJG7C8A6E1K1G07pWH8gYXyrwPQIROSIih12fozhXA497PzRTnmRkZvHGom30m7iYD2PjAbiwRbAlAWNKgYI6rxcgCqdoHECW2ls9ppDW703i0c/jWLfnGAPbNeRyKxJnTKmSbyJQVRWR2arapaQCMuXLB7/E89xXG6hdrRJvXN/ZKoUaUwp5co9guYh0VtXfvB6NKTeyi8S1Pi+IoR0b89QVbahdzR4JNaY0OmsiEJEKqpoB9AL+T0S2Ack4PY+pqnYuoRhNGZJ8KoMXv99ExUDhicvbWpE4Y8qA/K4IlgOdgWElFIsp45ZsTuSxWWvZm5TCzT1C3VcFxpjSLb9EIACquq2EYjFlVNLJdJ77egOfrUqgeX2nSFzXUCsSZ0xZkV8iqC8iD5xtoqq+4oV4TBl0MPkU367dx119wrknxorEGVPW5JcIAoEauK4MjMnpwPFU5q7Zy+0XNXcXiatj9YGMKZPySwT7VHV8iUViygRV5fPf9vDcVxtISc8kpk1DwoKrWxIwpgwr8B6BMdl2Hz7J47PXsnTLQaKb1WHCVVYkzpjyIL9EEFNiUZhSLyMzi2vfXsaR5DSeG9qO67s3I8CKxBlTLpw1Eajq4ZIMxJRO8QeTaVq3GhUCA3hhhFMkrkkdqw9kTHniSef1xg+lZ2YxeeFWBkxc4i4S1zM82JKAMeXQuZShNuXcuj1JPPJZHBv2HePyyEZc0eF8X4dkjPEiSwTmNO/9vIPnv95I3eqVmHJDFwa1P8/XIRljvMz/EkF6EqQlQWIs1O/h62hKjexyEO3Or8XwTo158vK21KpW0ddhGWNKgH8lgsRYOBIHZMGPMXDJAr9PBidOZfDCd39QKTCAJ69oS7ewunQLs/IQxvgT/7pZfGARkOV8zzzlGvZfizYdYODEJXy0bCeKc1VgjPE//nNFkBgLB5fnGJEFlfyzPPKR5DSe+3oDs37bQ4sGNfjsjp50aVbH12EZY3zEPxJBYiws6A1Z6aePP7LaN/H42JGTafywfj/3XNKCMZe0oHIFKxJnjD/zatOQiAwSkU0islVExuUx/QER2SAicSKyQESaeSWQA4vOTAJ+5sCxVN5asg1VpXn9Gvz86CU8MKCVJQFjjPcSgYgEApOBS4G2wLUi0jbXbKuBaFXtAHwGvOCVYM5oAhIIqARhN3llc6WJqvLfFbuJeWUxL/+wmfhDJwHsiSBjjJs3rwi6AVtVdbuqpgEzgKE5Z1DVhap60jW4DGjilUjSDuUYEDivP8QsKvdPDO0+fJIb313OI5/H0aZRTb699yIrEmeMOYM37xE0BnbnGE4Auucz/1+Ab/OaICKjgdEAISEhhY+kQR+cnJcFgVUg8plynwSyi8QdPZnO88Pac123ECsSZ4zJkzcTQV5HnTyfTxSRG4BooHde01X1LeAtgOjo6MI/41i/B9Tp4LxI1nNauU4COw4mE+IqEvfiiCia1avG+bWr+josY0wp5s2moQSgaY7hJsDe3DOJSD/gCWCIqp7yWjQVa0H1kHKbBNIzs/j3gi0MnLiED36JB6BHeD1LAsaYAnnzimAFECEiYcAeYBRwXc4ZRKQT8CYwSFUPeDGWcl1aIi7hKI98Fscffx5ncNT5DOloReKMMZ7zWiJQ1QwRGQt8j9P/8VRVXS8i44GVqjoXeBGnX+RPRQRgl6oOKfZgEmPhyO+AwoK+ELOw3CSDqT/t4PmvN1A/qDJv3xRN/7YNfR2SMaaM8eoLZar6DfBNrnFP5/jez5vbd9vxIe7bE1mnnOEyngiyi8R1aFKLkV2bMu7SNtSqao+EGmMKzz/eLBEDiBAAAB9NSURBVC5HjqemM+HbP6hcIZCnB7clOrQu0aFWJM4Yc+78o+hc2E24H2Iqwy+SLfzjAAMmLmH68l1UCBQrEmeMKRb+cUVQvwfUiSqzj48eTk5j/Jfr+WLNXlo2rMF/ru9JpxArEmeMKR7+kQjKuKSUdBZsPMC9MRGM6duCShX840LOGFMy/OOIkt0hTfIOp0OaxFhfR1SgP5NSmbLYKRIXFlydn8Zdwv39W1oSMMYUO/84quTskCYrrVR3SKOqTF++i/6vLObV+ZvZmV0kzp4IMsZ4iX80DeWsNRRQyTVc+uw8lMy4z9cSu/0QFzSvy4ThHQgtB0Xi0tPTSUhIIDU11dehGFPuValShSZNmlCxoucnj/6RCMpAraGMzCyue/tXklLS+ceVkYzq2rTcFIlLSEggKCiI0NBQXC8OGmO8QFU5dOgQCQkJhIWFebycfyQCcGoNVaxV6pLAtsQTNHMViXv5GqdIXKNa5as+UGpqqiUBY0qAiFCvXj0SExMLtZx/3CMohdIysnh1/mYGvbqED2N3AnBB83rlLglksyRgTMk4l781/7kiKEXW7D7Ko5/FsWn/cYZ2PJ9hnRr7OiRjjB+zK4IS9u5POxj+n59JSknn3ZujeW1UJ+pWr+TrsMq9wMBAOnbsSPv27Rk8eDBHjx4tlvXGx8fTvn37YlmXNz3zzDO89NJLADz99NPMnz//rPOuWbOGb775X4mwuXPnMmHChGKJIyUlhd69e5OZmekeN3HiRKpUqUJSUpJ73Pvvv8/YsWNPW7ZPnz6sXLkSgNDQUCIjI+nQoQO9e/dm586d7vkSEhIYOnQoERERhIeHc++995KWluaevnz5ci6++GJatWpF69atuf322zl58iRFsWPHDrp3705ERAQjR448bXvZ4uPjqVq1Kh07dqRjx47ccccd7mmrVq0iMjKSFi1acM8997irBhw+fJj+/fsTERFB//79OXLkCABfffUVf/vb34oUc06WCEpI9i+2Y9NajOoWwg8PXExMG6sUWlKqVq3KmjVrWLduHXXr1mXy5Mm+DqnIMjIyzmm58ePH06/f2es95k4EQ4YMYdy4cee0rdymTp3K8OHDCQwMdI+bPn06Xbt2Zfbs2YVa18KFC4mLi6NPnz48//zzgPN3Nnz4cIYNG8aWLVvYvHkzJ06c4IknngBg//79XH311fzrX/9i06ZNbNy4kUGDBnH8+PEi7dejjz7K/fffz5YtW6hTpw7vvvtunvOFh4ezZs0a1qxZw5QpU9zj77zzTt566y22bNnCli1b+O677wCYMGECMTExbNmyhZiYGHdCvvzyy5k7d26RE1g2SwRediw1ncdmrWX8VxsA6NKsLv+4MpKaVfz3vYCRb8ae8fkoNh6AlLTMPKd/utLp9fRwctoZ0wqrR48e7NmzB4ATJ04QExND586diYyMZM6cOYBz9tamTRv+7//+j3bt2jFgwABSUlIA5+wtKiqKHj16nJZQUlNTufXWW4mMjKRTp04sXLgQcM5uhw0bxuDBgwkLC+P111/nlVdeoVOnTlxwwQUcPnz4jBhvueUW7rjjDi666CJatmzJV1995V7X1VdfzeDBgxkwYAAAL774Il27dqVDhw6nnSX+/e9/p1WrVvTr149Nmzadtu7PPvsMgBUrVtCzZ0+ioqLo1q0bSUlJPP3008ycOZOOHTsyc+bM087Od+7cSUxMDB06dCAmJoZdu3a513nPPffQs2dPmjdv7l5/btOmTWPo0P91Xb5t2zZOnDjB888/z/Tp0z3+HeaU8/f5448/UqVKFW699VbAuRKcOHEiU6dO5eTJk0yePJmbb76ZHj2ch0ZEhBEjRtCw4bmflKkqP/74IyNGjADg5ptv5osvvvB4+X379nHs2DF69OiBiHDTTTe5l58zZw4333zzGesVEfr06eP+f1FUlgi8aP6G/fR/ZTEzV+yiUoUAKxJXCmRmZrJgwQKGDHG6vahSpQqzZ8/mt99+Y+HChTz44IPu39OWLVsYM2YM69evp3bt2nz++ecA3HrrrUyaNInY2NOTUHZSWLt2LdOnT+fmm292vzuxbt06PvnkE5YvX84TTzxBtWrVWL16NT169ODDDz/MM9b4+HgWL17M119/zR133OFeV2xsLB988AE//vgjP/zwA1u2bGH58uWsWbOGVatWsWTJElatWsWMGTNYvXo1s2bNYsWKFWesPy0tjZEjR/Laa6/x+++/M3/+fKpXr8748eMZOXIka9asYeTIkactM3bsWG666Sbi4uK4/vrrueeee9zT9u3bx08//cRXX32V5xVEWloa27dvJzQ01D1u+vTpXHvttVx00UVs2rSJAwcK3z/Vd999x7BhwwBYv349Xbp0OW16zZo1CQkJYevWraxbt+6M6XnZtGmTuwkn9yd3s+KhQ4eoXbs2FSo4t1ybNGniTky57dixg06dOtG7d2+WLl0KwJ49e2jSpIl7npzL79+/n0aNGgHQqFGj034+0dHR7nUUld0s9oJDJ07x7JcbmPv7XlqfF8RbN0YT1bS2r8MqNWb+9eyP8FatFJjv9LrVK+U7/WxSUlLo2LEj8fHxdOnShf79+wPO2dzjjz/OkiVLCAgIYM+ePezfvx+AsLAwOnbsCECXLl2Ij48nKSmJo0eP0ru30732jTfeyLfffgvATz/9xN133w1A69atadasGZs3bwagb9++BAUFERQURK1atRg8eDAAkZGRxMXF5RnzNddcQ0BAABERETRv3pw//vgDgP79+1O3rlN6/IcffuCHH36gU6dOgHOFs2XLFo4fP86VV15JtWrVANyJL6dNmzbRqFEjunbtCjgHzILExsYya9Ys974/8sgj7mnDhg0jICCAtm3bun+GOR08eJDatU//O5gxYwazZ88mICCA4cOH8+mnnzJmzJizPvmSc3zfvn3Zv38/DRo0OK1pKK9lzzb+bFq1asWaNWs8mjevE7y8ttWoUSN27dpFvXr1WLVqFcOGDWP9+vUeL59bgwYN2Lv3jN5/z4ldEXjB8dQMFm46wP39WjJ3bC9LAqVA9j2CnTt3kpaW5j57nzZtGomJiaxatYo1a9bQsGFD95l35cqV3csHBgaSkZGR7wElvyu+nOsKCAhwDwcEBJy1rT/3drKHq1f/39vmqspjjz3mbnfeunUrf/nLX/JcPq94i/pYb87lc+5jXj+LqlWrnvZ2eVxcHFu2bKF///6EhoYyY8YMd/NQvXr13DdGsx0+fJjg4GD38MKFC9m5cyft2rXj6aed/q7atWvnvqGc7dixY+zevZvw8HDatWvHqlWrCtyvwlwRBAcHc/ToUffvMSEhgfPPP7O72MqVK1OvXj3AObEIDw9n8+bNNGnShISEBPd8OZdv2LAh+/btA5wrrgYNGrjnS01NpWrV4nnc3BJBMdl7NIXJC7eiqoQGV+fncZdwb78IKxJXytSqVYtJkybx0ksvkZ6eTlJSEg0aNKBixYruA0t+ateuTa1atfjpp58AJ5Fku/jii93DmzdvZteuXbRq1eqcY/3000/Jyspi27ZtbN++Pc91DRw4kKlTp3LixAnAaWY4cOAAF198MbNnzyYlJYXjx4/z5ZdfnrFs69at2bt3r7vZ6Pjx42RkZBAUFHTWm6c9e/ZkxowZ7n3v1auXx/tTp04dMjMz3clg+vTpPPPMM8THxxMfH8/evXvZs2cPO3fupGvXrvz888/8+eefAKxcuZJTp07RtGnT09ZZtWpVXn31VT788EMOHz5MTEwMJ0+edDe3ZWZm8uCDD3LLLbdQrVo1xo4dywcffMCvv/7qXsfHH3/s3k627CuCvD65r2pEhL59+7rvi3zwwQen3QfJlpiY6H5aavv27WzZsoXmzZvTqFEjgoKCWLZsGarKhx9+6F5+yJAhfPDBB3mud/PmzcX2xJodpYooK0v5eNlOBkxcwus/bnUXifPnm8GlXadOnYiKimLGjBlcf/31rFy5kujoaKZNm0br1q0LXP69995jzJgx9OjR47QzsrvuuovMzEwiIyMZOXIk77///mlnyYXVqlUrevfuzaWXXsqUKVOoUqXKGfMMGDCA6667jh49ehAZGcmIESM4fvw4nTt3ZuTIkXTs2JGrrrqKiy666IxlK1WqxMyZM7n77ruJioqif//+pKam0rdvXzZs2OC+WZzTpEmTeO+99+jQoQMfffQRr732WqH2acCAAe4kOmPGDK688srTpl955ZXMmDGDhg0b8tprr3HZZZfRsWNH7rvvPqZPn05AwJmHrEaNGnHttdcyefJkRITZs2fz6aefEhERQcuWLalSpQr/+Mc/AOcMe8aMGTz00EO0atWKNm3asHTpUo+axfLzr3/9i1deeYUWLVpw6NAh91XZ3Llz3VcrS5YsoUOHDkRFRTFixAimTJnibuJ74403uP3222nRogXh4eFceumlAIwbN4558+YRERHBvHnzTrv3snDhQi6//PIixZ1NytoNzOjoaM196eeR+X2cf/stKrZYdhxMZtzncfy64zAXtqjHP6/sQEi9asW2/vJi48aNtGnTxtdhlCm33HILV1xxhftJlPJi9erVvPLKK3z00Ue+DqVM279/P9dddx0LFizIc3pef3MiskpVo/Oa324Wn6OMzCxueOdXjqWm88JVHbg6uomVUTCmAJ06daJv375kZmae9i6BKZxdu3bx8ssvF9v6LBEU0tYDxwmtV50KgQFMHNmRZvWq0bDmmZfsxhTF+++/7+sQvOa2227zdQhlXvaTXsXF7hF46FRGJq/M28ygV5fygatIXLewupYEjDFlnl0ReOC3XUd49LM4thw4wfBOjRluReKMMeWIJYICvL1kO//4diONalbhvVu70rdVg4IXMsaYMsQSwVlkZSkBAULnZrW5vnsIjw5qTZA9EmqMKYfsHkEuSSnpPPLZ7zz75XrAKRL3/LBISwJlnIjw4IMPuodfeuklnnnmmXyXKa7yy++//z7169enY8eOtGvXjhEjRhRb1cjC8OcS0Glpae6CgFFRUSxatMg9bebMmXTo0IF27dqdVjJjypQpREZG0rFjR3r16sWGDU7hyLVr13LLLbcUKebSxhJBDt+v/5P+ryzm89/2UL1yBSsS50uJsbD+n86/xaBy5crMmjWLgwcPerxMcZZfzi7itn79eveLXCXNn0tAv/3224BzEJ83bx4PPvggWVlZHDp0iIcffpgFCxawfv169u/f7342/7rrrmPt2rWsWbOGRx55hAceeABw6kMlJCS4K6+WB5YIgIMnTjFm2m/89aNVBNeozJwxF/LIoNb2XoA3rLrPebkvv8+3nWBeL/j9ceffbzvlP/+q+wrcbIUKFRg9ejQTJ048Y9qXX35J9+7d6dSpE/369XMXTMs+M05KSiI0NJSsrCwATp48SdOmTUlPT2fbtm0MGjSILl26cNFFF7kLw51NRkYGycnJ1KlT56zbzsrKIiIiwt3vbFZWFi1atODgwYMkJiZy1VVX0bVrV3cZBoDFixe7a+F06tQpz4OrP5eA3rBhAzExMYBTrK127dqsXLmS7du307JlS+rXrw9Av3793FVmc75tnJycfNrxYPDgwe5SG+WBJQLgRGoGS7ck8vDAVswZeyHtG9fydUj+LS0JyHINZLmGi27MmDFMmzbttGYQgF69erFs2TJWr17NqFGjeOGFF06bXqtWLaKioli8eDHgHLwHDhxIxYoVGT16NP/+979ZtWoVL730EnfddVee286u79+4cWMOHz7srj6a17YDAgK44YYb3HWL5s+fT1RUFMHBwdx7773cf//9rFixgs8//5zbb78dcJq6Jk+ezJo1a1i6dOkZxcj8vQR0VFQUc+bMISMjgx07drBq1Sp2795NixYt+OOPP4iPjycjI4MvvviC3bt3u5ebPHky4eHhPPLII0yaNMk9vjhLQJcGfnuzeM/RFGb/lsCYvi0IDa7OL4/FUKOy3/44Sk6XVwueJzEWfoyBrDQIqAQ9p0H9wpeezq1mzZrcdNNNTJo06bQDZUJCAiNHjmTfvn2kpaURFhZ2xrIjR45k5syZ9O3blxkzZnDXXXdx4sQJfvnlF66++mr3fKdOncpz2yNHjuT1119HVRkzZgwvvvgi48aNO+u2b7vtNoYOHcp9993H1KlT3WfZ8+fPd7dVg1NZ8/jx41x44YU88MADXH/99QwfPvy0+vZgJaBvu+02Nm7cSHR0NM2aNaNnz55UqFCBOnXq8MYbbzBy5EgCAgLo2bMn27dvdy83ZswYxowZwyeffMLzzz/vLgBXnCWgSwOvXhGIyCAR2SQiW0XkjMZWEaksIjNd038VkVBvxgPO00AfxcYz4JXFTF64zV0kzpJAKVK/B1yyADo85/xbDEkg23333ce7775LcnKye9zdd9/N2LFjWbt2LW+++eZppZKzDRkyhG+//ZbDhw+zatUqLrnkErKysqhdu/ZplSk3btyY7/ZFhMGDB7NkyZJ8t920aVMaNmzIjz/+yK+//uouQpaVlUVsbKx7e3v27CEoKIhx48bxzjvvkJKSwgUXXHBGE5W/l4CuUKECEydOZM2aNcyZM4ejR48SEREBOM08v/76K7GxsbRq1co9PqdRo0ad1uRUnCWgSwOvJQIRCQQmA5cCbYFrRaRtrtn+AhxR1RbAROBf3oqH9CTSj8Xz1Ltv89Sc9XRuVocf7r+Y0ODqBS9rSl79HtDusWJNAgB169blmmuuOe2GYlJSEo0bOy8JZp/x5VajRg26devGvffeyxVXXEFgYCA1a9YkLCyMTz/9FHDOTn///fcCY/jpp58IDw8vcNu33347N9xwA9dcc437Bu+AAQN4/fXX3fNknzlv27aNyMhIHn30UaKjo89IBP5eAvrkyZPu5D9v3jwqVKhA27bO4Si7SezIkSP85z//cTe3bdmyxb38119/fVqCKM4S0KWCqnrlA/QAvs8x/BjwWK55vgd6uL5XAA7iqoh6tk+XLl200A78olnTAjTrY/TkR5V1/pLZmpWVVfj1mHOyYcMGX4eg1atXd3//888/tWrVqvq3v/1NVVW/+OILDQsL0169eulDDz2kvXv3VlXV9957T8eMGeNe7tNPP1VAFy1a5B63fft2HThwoHbo0EHbtGmjzz777Bnbfu+99zQ4OFijoqI0MjJSL730Ut2/f3++21ZVTUtL06CgIN24caN7XGJiol5zzTUaGRmpbdq00b/+9a+qqjp27Fht166ddujQQUeNGqWpqalnxHHbbbfpvHnzVFU1NDT0tPWqqt5///06YcIEd1ydOnXSqKgovfDCC3XVqlXu+Zo1a6aJiYnu4bFjx+r48eNVVXXXrl16xRVXaIsWLbR58+Y6duzY02L55ZdftFevXtqyZUtt3bq1jh49WpOTk8+ItTC2bdumXbt21fDwcB0xYoR7e3PmzNGnnnpKVVV37Njh3mZMTIzGx8e7lx81apS2adNG27Rpo9OnT3ePv+eee7Rt27YaFRWlffr00XXr1rmnjRkzRufOnVukuL0pr785YKWe7Xh9tglF/QAjgHdyDN8IvJ5rnnVAkxzD24DgPNY1GlgJrAwJCSn8T2XdP1Snieo0NGtaoDNsSkxpSARl0YoVK7RXr17Ftr7ffvtNb7jhhmJbn79KTU3V7t27a3p6uq9DOavCJgJv3iPI6+5Q7js7nsyDqr6lqtGqGp39mFehNOgDgVVAApHASs6wMaXYhAkTuOqqq/jnP/9ZbOvMWQLanLtdu3YxYcIE95NK5YE39yQByNmo2ATIfZs9e54EEakA1AIOF3sk2TcfDyxykkAxtzsbU9zGjRtXbC+z5WQloIsuIiIizxvKZZk3E8EKIEJEwoA9wCjgulzzzAVuBmJxmpJ+dF3CFL/6PSwB+JAWQ0fpxpiCncsh1GtNQ6qaAYzFuSG8Efivqq4XkfEiMsQ127tAPRHZCjwAFP8pkPG5KlWqcOjQISvZYYyXqSqHDh3Ks3/r/PhPn8XGZ9LT00lISMjz+XxjTPGqUqUKTZo0oWLF0wtlWp/FxqcqVqyY59u6xpjSwWoNGWOMn7NEYIwxfs4SgTHG+Lkyd7NYRBKBnQXOmLdgnDIW/sT22T/YPvuHouxzM1XN843cMpcIikJEVp7trnl5ZfvsH2yf/YO39tmahowxxs9ZIjDGGD/nb4ngLV8H4AO2z/7B9tk/eGWf/eoegTHGmDP52xWBMcaYXCwRGGOMnyuXiUBEBonIJhHZKiJnVDQVkcoiMtM1/VcRCS35KIuXB/v8gIhsEJE4EVkgIs18EWdxKmifc8w3QkRURMr8o4ae7LOIXOP6Xa8XkU9KOsbi5sH/7RARWSgiq13/vy/zRZzFRUSmisgBEVl3lukiIpNcP484Eelc5I2ereuysvoBAnG6vGwOVAJ+B9rmmucuYIrr+yhgpq/jLoF97gtUc32/0x/22TVfELAEWAZE+zruEvg9RwCrgTqu4Qa+jrsE9vkt4E7X97ZAvK/jLuI+Xwx0BtadZfplwLc4PTxeAPxa1G2WxyuCbsBWVd2uqmnADGBornmGAh+4vn8GxEjZ7jWlwH1W1YWqetI1uAynx7iyzJPfM8BzwAtAeaiB7ck+/x8wWVWPAKjqgRKOsbh5ss8K1HR9r8WZPSGWKaq6hPx7ahwKfKiOZUBtEWlUlG2Wx0TQGNidYzjBNS7PedTpQCcJqFci0XmHJ/uc019wzijKsgL3WUQ6AU1V9auSDMyLPPk9twRaisjPIrJMRAaVWHTe4ck+PwPcICIJwDfA3SUTms8U9u+9QOWxP4K8zuxzPyPryTxlicf7IyI3ANFAb69G5H357rOIBAATgVtKKqAS4MnvuQJO81AfnKu+pSLSXlWPejk2b/Fkn68F3lfVl0WkB/CRa5+zvB+eTxT78as8XhEkAE1zDDfhzEtF9zwiUgHncjK/S7HSzpN9RkT6AU8AQ1T1VAnF5i0F7XMQ0B5YJCLxOG2pc8v4DWNP/2/PUdV0Vd0BbMJJDGWVJ/v8F+C/AKoaC1TBKc5WXnn0914Y5TERrAAiRCRMRCrh3Ayem2ueucDNru8jgB/VdRemjCpwn13NJG/iJIGy3m4MBeyzqiaparCqhqpqKM59kSGqWpb7OfXk//YXOA8GICLBOE1F20s0yuLlyT7vAmIARKQNTiJILNEoS9Zc4CbX00MXAEmquq8oKyx3TUOqmiEiY4HvcZ44mKqq60VkPLBSVecC7+JcPm7FuRIY5buIi87DfX4RqAF86rovvktVh/gs6CLycJ/LFQ/3+XtggIhsADKBh1X1kO+iLhoP9/lB4G0RuR+nieSWsnxiJyLTcZr2gl33Pf4GVARQ1Sk490EuA7YCJ4Fbi7zNMvzzMsYYUwzKY9OQMcaYQrBEYIwxfs4SgTHG+DlLBMYY4+csERhjjJ+zRGBKHRHJFJE1OT6h+cwberYqjYXc5iJXhcvfXeUZWp3DOu4QkZtc328RkfNzTHtHRNoWc5wrRKSjB8vcJyLVirptU35ZIjClUYqqdszxiS+h7V6vqlE4BQlfLOzCqjpFVT90Dd4CnJ9j2u2quqFYovxfnP/BszjvAywRmLOyRGDKBNeZ/1IR+c316ZnHPO1EZLnrKiJORCJc42/IMf5NEQksYHNLgBauZWNcde7XuurEV3aNnyD/69/hJde4Z0TkIREZgVPPaZprm1VdZ/LRInKniLyQI+ZbROTf5xhnLDmKjYnIGyKyUpx+CJ51jbsHJyEtFJGFrnEDRCTW9XP8VERqFLAdU85ZIjClUdUczUKzXeMOAP1VtTMwEpiUx3J3AK+pakecA3GCq+TASOBC1/hM4PoCtj8YWCsiVYD3gZGqGonzJv6dIlIXuBJop6odgOdzLqyqnwErcc7cO6pqSo7JnwHDcwyPBGaeY5yDcEpKZHtCVaOBDkBvEemgqpNw6tD0VdW+rrITTwL9XD/LlcADBWzHlHPlrsSEKRdSXAfDnCoCr7vaxDNxaujkFgs8ISJNgFmqukVEYoAuwApXaY2qOEklL9NEJAWIxyll3ArYoaqbXdM/AMYAr+P0b/COiHwNeFzmWlUTRWS7q0bMFtc2fnattzBxVscpuZCzd6prRGQ0zt91I5xOWuJyLXuBa/zPru1Uwvm5GT9micCUFfcD+4EonCvZMzqaUdVPRORX4HLgexG5Hadk7weq+pgH27g+Z1E6EcmzjwpX/ZtuOIXORgFjgUsKsS8zgWuAP4DZqqriHJU9jhOnp64JwGRguIiEAQ8BXVX1iIi8j1N8LTcB5qnqtYWI15Rz1jRkyopawD5Xjfkbcc6GTyMizYHtruaQuThNJAuAESLSwDVPXfG8v+Y/gFARaeEavhFY7GpTr6Wq3+DciM3ryZ3jOKWw8zILGIZTR3+ma1yh4lTVdJwmngtczUo1gWQgSUQaApeeJZZlwIXZ+yQi1UQkr6sr40csEZiy4j/AzSKyDKdZKDmPeUYC60RkDdAapzu/DTgHzB9EJA6Yh9NsUiBVTcWp7PipiKwFsoApOAfVr1zrW4xztZLb+8CU7JvFudZ7BNgANFPV5a5xhY7Tde/hZeAhVf0dp6/i9cBUnOambG8B34rIQlVNxHmiabprO8twflbGj1n1UWOM8XN2RWCMMX7OEoExxvg5SwTGGOPnLBEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5/4fq+7FeIAf4LwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(r_fpr, r_tpr, linestyle='--', label='Random prediction (AUROC = %0.3f)' % r_auc)\n",
    "plt.plot(nb_fpr, nb_tpr, marker='.', label='Naive Bayes (AUROC = %0.3f)' % nb_auc, color='orange')\n",
    "\n",
    "plt.title('ROC Plot')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}