Logistic Regression Converted.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('Leads.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9240"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Prospect ID</th>\n",
       "      <th>Lead Number</th>\n",
       "      <th>Lead Origin</th>\n",
       "      <th>Lead Source</th>\n",
       "      <th>Do Not Email</th>\n",
       "      <th>Do Not Call</th>\n",
       "      <th>Converted</th>\n",
       "      <th>TotalVisits</th>\n",
       "      <th>Total Time Spent on Website</th>\n",
       "      <th>Page Views Per Visit</th>\n",
       "      <th>...</th>\n",
       "      <th>Get updates on DM Content</th>\n",
       "      <th>Lead Profile</th>\n",
       "      <th>City</th>\n",
       "      <th>Asymmetrique Activity Index</th>\n",
       "      <th>Asymmetrique Profile Index</th>\n",
       "      <th>Asymmetrique Activity Score</th>\n",
       "      <th>Asymmetrique Profile Score</th>\n",
       "      <th>I agree to pay the amount through cheque</th>\n",
       "      <th>A free copy of Mastering The Interview</th>\n",
       "      <th>Last Notable Activity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7927b2df-8bba-4d29-b9a2-b6e0beafe620</td>\n",
       "      <td>660737</td>\n",
       "      <td>API</td>\n",
       "      <td>Olark Chat</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Select</td>\n",
       "      <td>Select</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Modified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2a272436-5132-4136-86fa-dcc88c88f482</td>\n",
       "      <td>660728</td>\n",
       "      <td>API</td>\n",
       "      <td>Organic Search</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>674</td>\n",
       "      <td>2.5</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Select</td>\n",
       "      <td>Select</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Email Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8cc8c611-a219-4f35-ad23-fdfd2656bd8a</td>\n",
       "      <td>660727</td>\n",
       "      <td>Landing Page Submission</td>\n",
       "      <td>Direct Traffic</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1532</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Potential Lead</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>01.High</td>\n",
       "      <td>14.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Email Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0cc2df48-7cf4-4e39-9de9-19797f9b38cc</td>\n",
       "      <td>660719</td>\n",
       "      <td>Landing Page Submission</td>\n",
       "      <td>Direct Traffic</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>305</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Select</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>01.High</td>\n",
       "      <td>13.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Modified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3256f628-e534-4826-9d63-4a8b88782852</td>\n",
       "      <td>660681</td>\n",
       "      <td>Landing Page Submission</td>\n",
       "      <td>Google</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1428</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Select</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>02.Medium</td>\n",
       "      <td>01.High</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Modified</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Prospect ID  Lead Number              Lead Origin  \\\n",
       "0  7927b2df-8bba-4d29-b9a2-b6e0beafe620       660737                      API   \n",
       "1  2a272436-5132-4136-86fa-dcc88c88f482       660728                      API   \n",
       "2  8cc8c611-a219-4f35-ad23-fdfd2656bd8a       660727  Landing Page Submission   \n",
       "3  0cc2df48-7cf4-4e39-9de9-19797f9b38cc       660719  Landing Page Submission   \n",
       "4  3256f628-e534-4826-9d63-4a8b88782852       660681  Landing Page Submission   \n",
       "\n",
       "      Lead Source Do Not Email Do Not Call  Converted  TotalVisits  \\\n",
       "0      Olark Chat           No          No          0          0.0   \n",
       "1  Organic Search           No          No          0          5.0   \n",
       "2  Direct Traffic           No          No          1          2.0   \n",
       "3  Direct Traffic           No          No          0          1.0   \n",
       "4          Google           No          No          1          2.0   \n",
       "\n",
       "   Total Time Spent on Website  Page Views Per Visit  ...  \\\n",
       "0                            0                   0.0  ...   \n",
       "1                          674                   2.5  ...   \n",
       "2                         1532                   2.0  ...   \n",
       "3                          305                   1.0  ...   \n",
       "4                         1428                   1.0  ...   \n",
       "\n",
       "  Get updates on DM Content    Lead Profile    City  \\\n",
       "0                        No          Select  Select   \n",
       "1                        No          Select  Select   \n",
       "2                        No  Potential Lead  Mumbai   \n",
       "3                        No          Select  Mumbai   \n",
       "4                        No          Select  Mumbai   \n",
       "\n",
       "  Asymmetrique Activity Index Asymmetrique Profile Index  \\\n",
       "0                   02.Medium                  02.Medium   \n",
       "1                   02.Medium                  02.Medium   \n",
       "2                   02.Medium                    01.High   \n",
       "3                   02.Medium                    01.High   \n",
       "4                   02.Medium                    01.High   \n",
       "\n",
       "  Asymmetrique Activity Score Asymmetrique Profile Score  \\\n",
       "0                        15.0                       15.0   \n",
       "1                        15.0                       15.0   \n",
       "2                        14.0                       20.0   \n",
       "3                        13.0                       17.0   \n",
       "4                        15.0                       18.0   \n",
       "\n",
       "  I agree to pay the amount through cheque  \\\n",
       "0                                       No   \n",
       "1                                       No   \n",
       "2                                       No   \n",
       "3                                       No   \n",
       "4                                       No   \n",
       "\n",
       "  A free copy of Mastering The Interview Last Notable Activity  \n",
       "0                                     No              Modified  \n",
       "1                                     No          Email Opened  \n",
       "2                                    Yes          Email Opened  \n",
       "3                                     No              Modified  \n",
       "4                                     No              Modified  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Prospect ID                                       object\n",
       "Lead Number                                        int64\n",
       "Lead Origin                                       object\n",
       "Lead Source                                       object\n",
       "Do Not Email                                      object\n",
       "Do Not Call                                       object\n",
       "Converted                                          int64\n",
       "TotalVisits                                      float64\n",
       "Total Time Spent on Website                        int64\n",
       "Page Views Per Visit                             float64\n",
       "Last Activity                                     object\n",
       "Country                                           object\n",
       "Specialization                                    object\n",
       "How did you hear about X Education                object\n",
       "What is your current occupation                   object\n",
       "What matters most to you in choosing a course     object\n",
       "Search                                            object\n",
       "Magazine                                          object\n",
       "Newspaper Article                                 object\n",
       "X Education Forums                                object\n",
       "Newspaper                                         object\n",
       "Digital Advertisement                             object\n",
       "Through Recommendations                           object\n",
       "Receive More Updates About Our Courses            object\n",
       "Tags                                              object\n",
       "Lead Quality                                      object\n",
       "Update me on Supply Chain Content                 object\n",
       "Get updates on DM Content                         object\n",
       "Lead Profile                                      object\n",
       "City                                              object\n",
       "Asymmetrique Activity Index                       object\n",
       "Asymmetrique Profile Index                        object\n",
       "Asymmetrique Activity Score                      float64\n",
       "Asymmetrique Profile Score                       float64\n",
       "I agree to pay the amount through cheque          object\n",
       "A free copy of Mastering The Interview            object\n",
       "Last Notable Activity                             object\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Prospect ID                                         0\n",
       "Lead Number                                         0\n",
       "Lead Origin                                         0\n",
       "Lead Source                                        36\n",
       "Do Not Email                                        0\n",
       "Do Not Call                                         0\n",
       "Converted                                           0\n",
       "TotalVisits                                       137\n",
       "Total Time Spent on Website                         0\n",
       "Page Views Per Visit                              137\n",
       "Last Activity                                     103\n",
       "Country                                          2461\n",
       "Specialization                                   1438\n",
       "How did you hear about X Education               2207\n",
       "What is your current occupation                  2690\n",
       "What matters most to you in choosing a course    2709\n",
       "Search                                              0\n",
       "Magazine                                            0\n",
       "Newspaper Article                                   0\n",
       "X Education Forums                                  0\n",
       "Newspaper                                           0\n",
       "Digital Advertisement                               0\n",
       "Through Recommendations                             0\n",
       "Receive More Updates About Our Courses              0\n",
       "Tags                                             3353\n",
       "Lead Quality                                     4767\n",
       "Update me on Supply Chain Content                   0\n",
       "Get updates on DM Content                           0\n",
       "Lead Profile                                     2709\n",
       "City                                             1420\n",
       "Asymmetrique Activity Index                      4218\n",
       "Asymmetrique Profile Index                       4218\n",
       "Asymmetrique Activity Score                      4218\n",
       "Asymmetrique Profile Score                       4218\n",
       "I agree to pay the amount through cheque            0\n",
       "A free copy of Mastering The Interview              0\n",
       "Last Notable Activity                               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = df.columns.str.lower().str.replace(' ', '_')\n",
    "\n",
    "string_columns = list(df.dtypes[df.dtypes == 'object'].index)\n",
    "\n",
    "for col in string_columns:\n",
    "    df[col] = df[col].str.lower().str.replace(' ', '_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = [\n",
    "            'lead_origin','lead_source','do_not_email','do_not_call',\n",
    "            'last_activity','country','specialization', 'how_did_you_hear_about_x_education',\n",
    "            'what_is_your_current_occupation','what_matters_most_to_you_in_choosing_a_course','search',\n",
    "            'magazine','newspaper_article','digital_advertisement', 'through_recommendations',\n",
    "            'receive_more_updates_about_our_courses', 'tags', 'lead_quality','update_me_on_supply_chain_content',\n",
    "            'lead_profile','city', 'asymmetrique_activity_index','asymmetrique_profile_index',\n",
    "            'i_agree_to_pay_the_amount_through_cheque', 'a_free_copy_of_mastering_the_interview',\n",
    "            'last_notable_activity', 'newspaper','x_education_forums','get_updates_on_dm_content'\n",
    "                ]\n",
    "\n",
    "\n",
    "numerical = [\n",
    "    'lead_number', 'totalvisits','total_time_spent_on_website',\n",
    "    'page_views_per_visit','asymmetrique_activity_score','asymmetrique_profile_score'\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cat in categorical:\n",
    "    df[cat]= df[cat].fillna(df[cat].mode()[0])\n",
    "for num in numerical :\n",
    "    df[num]= df[num].fillna(df[num].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prospect_id                                      0\n",
       "lead_number                                      0\n",
       "lead_origin                                      0\n",
       "lead_source                                      0\n",
       "do_not_email                                     0\n",
       "do_not_call                                      0\n",
       "converted                                        0\n",
       "totalvisits                                      0\n",
       "total_time_spent_on_website                      0\n",
       "page_views_per_visit                             0\n",
       "last_activity                                    0\n",
       "country                                          0\n",
       "specialization                                   0\n",
       "how_did_you_hear_about_x_education               0\n",
       "what_is_your_current_occupation                  0\n",
       "what_matters_most_to_you_in_choosing_a_course    0\n",
       "search                                           0\n",
       "magazine                                         0\n",
       "newspaper_article                                0\n",
       "x_education_forums                               0\n",
       "newspaper                                        0\n",
       "digital_advertisement                            0\n",
       "through_recommendations                          0\n",
       "receive_more_updates_about_our_courses           0\n",
       "tags                                             0\n",
       "lead_quality                                     0\n",
       "update_me_on_supply_chain_content                0\n",
       "get_updates_on_dm_content                        0\n",
       "lead_profile                                     0\n",
       "city                                             0\n",
       "asymmetrique_activity_index                      0\n",
       "asymmetrique_profile_index                       0\n",
       "asymmetrique_activity_score                      0\n",
       "asymmetrique_profile_score                       0\n",
       "i_agree_to_pay_the_amount_through_cheque         0\n",
       "a_free_copy_of_mastering_the_interview           0\n",
       "last_notable_activity                            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>prospect_id</th>\n",
       "      <td>7927b2df-8bba-4d29-b9a2-b6e0beafe620</td>\n",
       "      <td>2a272436-5132-4136-86fa-dcc88c88f482</td>\n",
       "      <td>8cc8c611-a219-4f35-ad23-fdfd2656bd8a</td>\n",
       "      <td>0cc2df48-7cf4-4e39-9de9-19797f9b38cc</td>\n",
       "      <td>3256f628-e534-4826-9d63-4a8b88782852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_number</th>\n",
       "      <td>660737</td>\n",
       "      <td>660728</td>\n",
       "      <td>660727</td>\n",
       "      <td>660719</td>\n",
       "      <td>660681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_origin</th>\n",
       "      <td>api</td>\n",
       "      <td>api</td>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>landing_page_submission</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_source</th>\n",
       "      <td>olark_chat</td>\n",
       "      <td>organic_search</td>\n",
       "      <td>direct_traffic</td>\n",
       "      <td>direct_traffic</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>do_not_email</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>do_not_call</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>converted</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>totalvisits</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_time_spent_on_website</th>\n",
       "      <td>0</td>\n",
       "      <td>674</td>\n",
       "      <td>1532</td>\n",
       "      <td>305</td>\n",
       "      <td>1428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>page_views_per_visit</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_activity</th>\n",
       "      <td>page_visited_on_website</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>unreachable</td>\n",
       "      <td>converted_to_lead</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specialization</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>business_administration</td>\n",
       "      <td>media_and_advertising</td>\n",
       "      <td>select</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>how_did_you_hear_about_x_education</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>word_of_mouth</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>what_is_your_current_occupation</th>\n",
       "      <td>unemployed</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>student</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>unemployed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>what_matters_most_to_you_in_choosing_a_course</th>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>search</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magazine</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newspaper_article</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_education_forums</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newspaper</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>digital_advertisement</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>through_recommendations</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>receive_more_updates_about_our_courses</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tags</th>\n",
       "      <td>interested_in_other_courses</td>\n",
       "      <td>ringing</td>\n",
       "      <td>will_revert_after_reading_the_email</td>\n",
       "      <td>ringing</td>\n",
       "      <td>will_revert_after_reading_the_email</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_quality</th>\n",
       "      <td>low_in_relevance</td>\n",
       "      <td>might_be</td>\n",
       "      <td>might_be</td>\n",
       "      <td>not_sure</td>\n",
       "      <td>might_be</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>update_me_on_supply_chain_content</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>get_updates_on_dm_content</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_profile</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>potential_lead</td>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>mumbai</td>\n",
       "      <td>mumbai</td>\n",
       "      <td>mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_activity_index</th>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_profile_index</th>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>01.high</td>\n",
       "      <td>01.high</td>\n",
       "      <td>01.high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_activity_score</th>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_profile_score</th>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i_agree_to_pay_the_amount_through_cheque</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a_free_copy_of_mastering_the_interview</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_notable_activity</th>\n",
       "      <td>modified</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>modified</td>\n",
       "      <td>modified</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                  0  \\\n",
       "prospect_id                                    7927b2df-8bba-4d29-b9a2-b6e0beafe620   \n",
       "lead_number                                                                  660737   \n",
       "lead_origin                                                                     api   \n",
       "lead_source                                                              olark_chat   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         0   \n",
       "totalvisits                                                                     0.0   \n",
       "total_time_spent_on_website                                                       0   \n",
       "page_views_per_visit                                                            0.0   \n",
       "last_activity                                               page_visited_on_website   \n",
       "country                                                                       india   \n",
       "specialization                                                               select   \n",
       "how_did_you_hear_about_x_education                                           select   \n",
       "what_is_your_current_occupation                                          unemployed   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                                    interested_in_other_courses   \n",
       "lead_quality                                                       low_in_relevance   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                                 select   \n",
       "city                                                                         select   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                02.medium   \n",
       "asymmetrique_activity_score                                                    15.0   \n",
       "asymmetrique_profile_score                                                     15.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                           no   \n",
       "last_notable_activity                                                      modified   \n",
       "\n",
       "                                                                                  1  \\\n",
       "prospect_id                                    2a272436-5132-4136-86fa-dcc88c88f482   \n",
       "lead_number                                                                  660728   \n",
       "lead_origin                                                                     api   \n",
       "lead_source                                                          organic_search   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         0   \n",
       "totalvisits                                                                     5.0   \n",
       "total_time_spent_on_website                                                     674   \n",
       "page_views_per_visit                                                            2.5   \n",
       "last_activity                                                          email_opened   \n",
       "country                                                                       india   \n",
       "specialization                                                               select   \n",
       "how_did_you_hear_about_x_education                                           select   \n",
       "what_is_your_current_occupation                                          unemployed   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                                                        ringing   \n",
       "lead_quality                                                               might_be   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                                 select   \n",
       "city                                                                         select   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                02.medium   \n",
       "asymmetrique_activity_score                                                    15.0   \n",
       "asymmetrique_profile_score                                                     15.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                           no   \n",
       "last_notable_activity                                                  email_opened   \n",
       "\n",
       "                                                                                  2  \\\n",
       "prospect_id                                    8cc8c611-a219-4f35-ad23-fdfd2656bd8a   \n",
       "lead_number                                                                  660727   \n",
       "lead_origin                                                 landing_page_submission   \n",
       "lead_source                                                          direct_traffic   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         1   \n",
       "totalvisits                                                                     2.0   \n",
       "total_time_spent_on_website                                                    1532   \n",
       "page_views_per_visit                                                            2.0   \n",
       "last_activity                                                          email_opened   \n",
       "country                                                                       india   \n",
       "specialization                                              business_administration   \n",
       "how_did_you_hear_about_x_education                                           select   \n",
       "what_is_your_current_occupation                                             student   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                            will_revert_after_reading_the_email   \n",
       "lead_quality                                                               might_be   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                         potential_lead   \n",
       "city                                                                         mumbai   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                  01.high   \n",
       "asymmetrique_activity_score                                                    14.0   \n",
       "asymmetrique_profile_score                                                     20.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                          yes   \n",
       "last_notable_activity                                                  email_opened   \n",
       "\n",
       "                                                                                  3  \\\n",
       "prospect_id                                    0cc2df48-7cf4-4e39-9de9-19797f9b38cc   \n",
       "lead_number                                                                  660719   \n",
       "lead_origin                                                 landing_page_submission   \n",
       "lead_source                                                          direct_traffic   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         0   \n",
       "totalvisits                                                                     1.0   \n",
       "total_time_spent_on_website                                                     305   \n",
       "page_views_per_visit                                                            1.0   \n",
       "last_activity                                                           unreachable   \n",
       "country                                                                       india   \n",
       "specialization                                                media_and_advertising   \n",
       "how_did_you_hear_about_x_education                                    word_of_mouth   \n",
       "what_is_your_current_occupation                                          unemployed   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                                                        ringing   \n",
       "lead_quality                                                               not_sure   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                                 select   \n",
       "city                                                                         mumbai   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                  01.high   \n",
       "asymmetrique_activity_score                                                    13.0   \n",
       "asymmetrique_profile_score                                                     17.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                           no   \n",
       "last_notable_activity                                                      modified   \n",
       "\n",
       "                                                                                  4  \n",
       "prospect_id                                    3256f628-e534-4826-9d63-4a8b88782852  \n",
       "lead_number                                                                  660681  \n",
       "lead_origin                                                 landing_page_submission  \n",
       "lead_source                                                                  google  \n",
       "do_not_email                                                                     no  \n",
       "do_not_call                                                                      no  \n",
       "converted                                                                         1  \n",
       "totalvisits                                                                     2.0  \n",
       "total_time_spent_on_website                                                    1428  \n",
       "page_views_per_visit                                                            1.0  \n",
       "last_activity                                                     converted_to_lead  \n",
       "country                                                                       india  \n",
       "specialization                                                               select  \n",
       "how_did_you_hear_about_x_education                                            other  \n",
       "what_is_your_current_occupation                                          unemployed  \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects  \n",
       "search                                                                           no  \n",
       "magazine                                                                         no  \n",
       "newspaper_article                                                                no  \n",
       "x_education_forums                                                               no  \n",
       "newspaper                                                                        no  \n",
       "digital_advertisement                                                            no  \n",
       "through_recommendations                                                          no  \n",
       "receive_more_updates_about_our_courses                                           no  \n",
       "tags                                            will_revert_after_reading_the_email  \n",
       "lead_quality                                                               might_be  \n",
       "update_me_on_supply_chain_content                                                no  \n",
       "get_updates_on_dm_content                                                        no  \n",
       "lead_profile                                                                 select  \n",
       "city                                                                         mumbai  \n",
       "asymmetrique_activity_index                                               02.medium  \n",
       "asymmetrique_profile_index                                                  01.high  \n",
       "asymmetrique_activity_score                                                    15.0  \n",
       "asymmetrique_profile_score                                                     18.0  \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no  \n",
       "a_free_copy_of_mastering_the_interview                                           no  \n",
       "last_notable_activity                                                      modified  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = df.columns.str.lower().str.replace(' ', '_')\n",
    "\n",
    "string_columns = list(df.dtypes[df.dtypes == 'object'].index)\n",
    "\n",
    "for col in string_columns:\n",
    "    df[col] = df[col].str.lower().str.replace(' ', '_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>prospect_id</th>\n",
       "      <td>7927b2df-8bba-4d29-b9a2-b6e0beafe620</td>\n",
       "      <td>2a272436-5132-4136-86fa-dcc88c88f482</td>\n",
       "      <td>8cc8c611-a219-4f35-ad23-fdfd2656bd8a</td>\n",
       "      <td>0cc2df48-7cf4-4e39-9de9-19797f9b38cc</td>\n",
       "      <td>3256f628-e534-4826-9d63-4a8b88782852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_number</th>\n",
       "      <td>660737</td>\n",
       "      <td>660728</td>\n",
       "      <td>660727</td>\n",
       "      <td>660719</td>\n",
       "      <td>660681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_origin</th>\n",
       "      <td>api</td>\n",
       "      <td>api</td>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>landing_page_submission</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_source</th>\n",
       "      <td>olark_chat</td>\n",
       "      <td>organic_search</td>\n",
       "      <td>direct_traffic</td>\n",
       "      <td>direct_traffic</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>do_not_email</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>do_not_call</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>converted</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>totalvisits</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_time_spent_on_website</th>\n",
       "      <td>0</td>\n",
       "      <td>674</td>\n",
       "      <td>1532</td>\n",
       "      <td>305</td>\n",
       "      <td>1428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>page_views_per_visit</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_activity</th>\n",
       "      <td>page_visited_on_website</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>unreachable</td>\n",
       "      <td>converted_to_lead</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "      <td>india</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specialization</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>business_administration</td>\n",
       "      <td>media_and_advertising</td>\n",
       "      <td>select</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>how_did_you_hear_about_x_education</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>word_of_mouth</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>what_is_your_current_occupation</th>\n",
       "      <td>unemployed</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>student</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>unemployed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>what_matters_most_to_you_in_choosing_a_course</th>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "      <td>better_career_prospects</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>search</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magazine</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newspaper_article</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_education_forums</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newspaper</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>digital_advertisement</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>through_recommendations</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>receive_more_updates_about_our_courses</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tags</th>\n",
       "      <td>interested_in_other_courses</td>\n",
       "      <td>ringing</td>\n",
       "      <td>will_revert_after_reading_the_email</td>\n",
       "      <td>ringing</td>\n",
       "      <td>will_revert_after_reading_the_email</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_quality</th>\n",
       "      <td>low_in_relevance</td>\n",
       "      <td>might_be</td>\n",
       "      <td>might_be</td>\n",
       "      <td>not_sure</td>\n",
       "      <td>might_be</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>update_me_on_supply_chain_content</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>get_updates_on_dm_content</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_profile</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>potential_lead</td>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>select</td>\n",
       "      <td>select</td>\n",
       "      <td>mumbai</td>\n",
       "      <td>mumbai</td>\n",
       "      <td>mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_activity_index</th>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_profile_index</th>\n",
       "      <td>02.medium</td>\n",
       "      <td>02.medium</td>\n",
       "      <td>01.high</td>\n",
       "      <td>01.high</td>\n",
       "      <td>01.high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_activity_score</th>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_profile_score</th>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i_agree_to_pay_the_amount_through_cheque</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a_free_copy_of_mastering_the_interview</th>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_notable_activity</th>\n",
       "      <td>modified</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>email_opened</td>\n",
       "      <td>modified</td>\n",
       "      <td>modified</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                  0  \\\n",
       "prospect_id                                    7927b2df-8bba-4d29-b9a2-b6e0beafe620   \n",
       "lead_number                                                                  660737   \n",
       "lead_origin                                                                     api   \n",
       "lead_source                                                              olark_chat   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         0   \n",
       "totalvisits                                                                     0.0   \n",
       "total_time_spent_on_website                                                       0   \n",
       "page_views_per_visit                                                            0.0   \n",
       "last_activity                                               page_visited_on_website   \n",
       "country                                                                       india   \n",
       "specialization                                                               select   \n",
       "how_did_you_hear_about_x_education                                           select   \n",
       "what_is_your_current_occupation                                          unemployed   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                                    interested_in_other_courses   \n",
       "lead_quality                                                       low_in_relevance   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                                 select   \n",
       "city                                                                         select   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                02.medium   \n",
       "asymmetrique_activity_score                                                    15.0   \n",
       "asymmetrique_profile_score                                                     15.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                           no   \n",
       "last_notable_activity                                                      modified   \n",
       "\n",
       "                                                                                  1  \\\n",
       "prospect_id                                    2a272436-5132-4136-86fa-dcc88c88f482   \n",
       "lead_number                                                                  660728   \n",
       "lead_origin                                                                     api   \n",
       "lead_source                                                          organic_search   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         0   \n",
       "totalvisits                                                                     5.0   \n",
       "total_time_spent_on_website                                                     674   \n",
       "page_views_per_visit                                                            2.5   \n",
       "last_activity                                                          email_opened   \n",
       "country                                                                       india   \n",
       "specialization                                                               select   \n",
       "how_did_you_hear_about_x_education                                           select   \n",
       "what_is_your_current_occupation                                          unemployed   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                                                        ringing   \n",
       "lead_quality                                                               might_be   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                                 select   \n",
       "city                                                                         select   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                02.medium   \n",
       "asymmetrique_activity_score                                                    15.0   \n",
       "asymmetrique_profile_score                                                     15.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                           no   \n",
       "last_notable_activity                                                  email_opened   \n",
       "\n",
       "                                                                                  2  \\\n",
       "prospect_id                                    8cc8c611-a219-4f35-ad23-fdfd2656bd8a   \n",
       "lead_number                                                                  660727   \n",
       "lead_origin                                                 landing_page_submission   \n",
       "lead_source                                                          direct_traffic   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         1   \n",
       "totalvisits                                                                     2.0   \n",
       "total_time_spent_on_website                                                    1532   \n",
       "page_views_per_visit                                                            2.0   \n",
       "last_activity                                                          email_opened   \n",
       "country                                                                       india   \n",
       "specialization                                              business_administration   \n",
       "how_did_you_hear_about_x_education                                           select   \n",
       "what_is_your_current_occupation                                             student   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                            will_revert_after_reading_the_email   \n",
       "lead_quality                                                               might_be   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                         potential_lead   \n",
       "city                                                                         mumbai   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                  01.high   \n",
       "asymmetrique_activity_score                                                    14.0   \n",
       "asymmetrique_profile_score                                                     20.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                          yes   \n",
       "last_notable_activity                                                  email_opened   \n",
       "\n",
       "                                                                                  3  \\\n",
       "prospect_id                                    0cc2df48-7cf4-4e39-9de9-19797f9b38cc   \n",
       "lead_number                                                                  660719   \n",
       "lead_origin                                                 landing_page_submission   \n",
       "lead_source                                                          direct_traffic   \n",
       "do_not_email                                                                     no   \n",
       "do_not_call                                                                      no   \n",
       "converted                                                                         0   \n",
       "totalvisits                                                                     1.0   \n",
       "total_time_spent_on_website                                                     305   \n",
       "page_views_per_visit                                                            1.0   \n",
       "last_activity                                                           unreachable   \n",
       "country                                                                       india   \n",
       "specialization                                                media_and_advertising   \n",
       "how_did_you_hear_about_x_education                                    word_of_mouth   \n",
       "what_is_your_current_occupation                                          unemployed   \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects   \n",
       "search                                                                           no   \n",
       "magazine                                                                         no   \n",
       "newspaper_article                                                                no   \n",
       "x_education_forums                                                               no   \n",
       "newspaper                                                                        no   \n",
       "digital_advertisement                                                            no   \n",
       "through_recommendations                                                          no   \n",
       "receive_more_updates_about_our_courses                                           no   \n",
       "tags                                                                        ringing   \n",
       "lead_quality                                                               not_sure   \n",
       "update_me_on_supply_chain_content                                                no   \n",
       "get_updates_on_dm_content                                                        no   \n",
       "lead_profile                                                                 select   \n",
       "city                                                                         mumbai   \n",
       "asymmetrique_activity_index                                               02.medium   \n",
       "asymmetrique_profile_index                                                  01.high   \n",
       "asymmetrique_activity_score                                                    13.0   \n",
       "asymmetrique_profile_score                                                     17.0   \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no   \n",
       "a_free_copy_of_mastering_the_interview                                           no   \n",
       "last_notable_activity                                                      modified   \n",
       "\n",
       "                                                                                  4  \n",
       "prospect_id                                    3256f628-e534-4826-9d63-4a8b88782852  \n",
       "lead_number                                                                  660681  \n",
       "lead_origin                                                 landing_page_submission  \n",
       "lead_source                                                                  google  \n",
       "do_not_email                                                                     no  \n",
       "do_not_call                                                                      no  \n",
       "converted                                                                         1  \n",
       "totalvisits                                                                     2.0  \n",
       "total_time_spent_on_website                                                    1428  \n",
       "page_views_per_visit                                                            1.0  \n",
       "last_activity                                                     converted_to_lead  \n",
       "country                                                                       india  \n",
       "specialization                                                               select  \n",
       "how_did_you_hear_about_x_education                                            other  \n",
       "what_is_your_current_occupation                                          unemployed  \n",
       "what_matters_most_to_you_in_choosing_a_course               better_career_prospects  \n",
       "search                                                                           no  \n",
       "magazine                                                                         no  \n",
       "newspaper_article                                                                no  \n",
       "x_education_forums                                                               no  \n",
       "newspaper                                                                        no  \n",
       "digital_advertisement                                                            no  \n",
       "through_recommendations                                                          no  \n",
       "receive_more_updates_about_our_courses                                           no  \n",
       "tags                                            will_revert_after_reading_the_email  \n",
       "lead_quality                                                               might_be  \n",
       "update_me_on_supply_chain_content                                                no  \n",
       "get_updates_on_dm_content                                                        no  \n",
       "lead_profile                                                                 select  \n",
       "city                                                                         mumbai  \n",
       "asymmetrique_activity_index                                               02.medium  \n",
       "asymmetrique_profile_index                                                  01.high  \n",
       "asymmetrique_activity_score                                                    15.0  \n",
       "asymmetrique_profile_score                                                     18.0  \n",
       "i_agree_to_pay_the_amount_through_cheque                                         no  \n",
       "a_free_copy_of_mastering_the_interview                                           no  \n",
       "last_notable_activity                                                      modified  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_full, df_test = train_test_split(df, test_size=0.2, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_val = train_test_split(df_train_full, test_size=0.25, random_state=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5544, 1848, 1848)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_train), len(df_val), len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train.reset_index(drop=True)\n",
    "df_val = df_val.reset_index(drop=True)\n",
    "df_test = df_test.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = df_train.converted.values\n",
    "y_val = df_val.converted.values\n",
    "y_test = df_test.converted.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df_train['converted']\n",
    "del df_val['converted']\n",
    "del df_test['converted']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploratory data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full_train = df_train_full.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4560\n",
       "1    2832\n",
       "Name: converted, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full_train.converted.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.616883\n",
       "1    0.383117\n",
       "Name: converted, dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full_train.converted.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prospect_id                                      0\n",
       "lead_number                                      0\n",
       "lead_origin                                      0\n",
       "lead_source                                      0\n",
       "do_not_email                                     0\n",
       "do_not_call                                      0\n",
       "converted                                        0\n",
       "totalvisits                                      0\n",
       "total_time_spent_on_website                      0\n",
       "page_views_per_visit                             0\n",
       "last_activity                                    0\n",
       "country                                          0\n",
       "specialization                                   0\n",
       "how_did_you_hear_about_x_education               0\n",
       "what_is_your_current_occupation                  0\n",
       "what_matters_most_to_you_in_choosing_a_course    0\n",
       "search                                           0\n",
       "magazine                                         0\n",
       "newspaper_article                                0\n",
       "x_education_forums                               0\n",
       "newspaper                                        0\n",
       "digital_advertisement                            0\n",
       "through_recommendations                          0\n",
       "receive_more_updates_about_our_courses           0\n",
       "tags                                             0\n",
       "lead_quality                                     0\n",
       "update_me_on_supply_chain_content                0\n",
       "get_updates_on_dm_content                        0\n",
       "lead_profile                                     0\n",
       "city                                             0\n",
       "asymmetrique_activity_index                      0\n",
       "asymmetrique_profile_index                       0\n",
       "asymmetrique_activity_score                      0\n",
       "asymmetrique_profile_score                       0\n",
       "i_agree_to_pay_the_amount_through_cheque         0\n",
       "a_free_copy_of_mastering_the_interview           0\n",
       "last_notable_activity                            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.616883\n",
       "1    0.383117\n",
       "Name: converted, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_full.converted.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.383"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "global_mean = df_train_full.converted.mean()\n",
    "round(global_mean, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = [\n",
    "            'lead_origin','lead_source','do_not_email','do_not_call',\n",
    "            'last_activity','country','specialization', 'how_did_you_hear_about_x_education',\n",
    "            'what_is_your_current_occupation','what_matters_most_to_you_in_choosing_a_course','search',\n",
    "            'magazine','newspaper_article','digital_advertisement', 'through_recommendations',\n",
    "            'receive_more_updates_about_our_courses', 'tags', 'lead_quality','update_me_on_supply_chain_content',\n",
    "            'lead_profile','city', 'asymmetrique_activity_index','asymmetrique_profile_index',\n",
    "            'i_agree_to_pay_the_amount_through_cheque', 'a_free_copy_of_mastering_the_interview',\n",
    "            'last_notable_activity', 'newspaper','x_education_forums','get_updates_on_dm_content'\n",
    "                ]\n",
    "\n",
    "\n",
    "numerical = [\n",
    "    'lead_number', 'totalvisits','total_time_spent_on_website',\n",
    "    'page_views_per_visit','asymmetrique_activity_score','asymmetrique_profile_score'\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lead_origin',\n",
       " 'lead_source',\n",
       " 'do_not_email',\n",
       " 'do_not_call',\n",
       " 'last_activity',\n",
       " 'country',\n",
       " 'specialization',\n",
       " 'how_did_you_hear_about_x_education',\n",
       " 'what_is_your_current_occupation',\n",
       " 'what_matters_most_to_you_in_choosing_a_course',\n",
       " 'search',\n",
       " 'magazine',\n",
       " 'newspaper_article',\n",
       " 'digital_advertisement',\n",
       " 'through_recommendations',\n",
       " 'receive_more_updates_about_our_courses',\n",
       " 'tags',\n",
       " 'lead_quality',\n",
       " 'update_me_on_supply_chain_content',\n",
       " 'lead_profile',\n",
       " 'city',\n",
       " 'asymmetrique_activity_index',\n",
       " 'asymmetrique_profile_index',\n",
       " 'i_agree_to_pay_the_amount_through_cheque',\n",
       " 'a_free_copy_of_mastering_the_interview',\n",
       " 'last_notable_activity',\n",
       " 'newspaper',\n",
       " 'x_education_forums',\n",
       " 'get_updates_on_dm_content']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lead_origin                                       5\n",
       "lead_source                                      19\n",
       "do_not_email                                      2\n",
       "do_not_call                                       2\n",
       "last_activity                                    16\n",
       "country                                          36\n",
       "specialization                                   19\n",
       "how_did_you_hear_about_x_education               10\n",
       "what_is_your_current_occupation                   6\n",
       "what_matters_most_to_you_in_choosing_a_course     2\n",
       "search                                            2\n",
       "magazine                                          1\n",
       "newspaper_article                                 2\n",
       "digital_advertisement                             2\n",
       "through_recommendations                           2\n",
       "receive_more_updates_about_our_courses            1\n",
       "tags                                             26\n",
       "lead_quality                                      5\n",
       "update_me_on_supply_chain_content                 1\n",
       "lead_profile                                      6\n",
       "city                                              7\n",
       "asymmetrique_activity_index                       3\n",
       "asymmetrique_profile_index                        3\n",
       "i_agree_to_pay_the_amount_through_cheque          1\n",
       "a_free_copy_of_mastering_the_interview            2\n",
       "last_notable_activity                            16\n",
       "newspaper                                         2\n",
       "x_education_forums                                1\n",
       "get_updates_on_dm_content                         1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_full[categorical].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.38311688311688313"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "global_mean = df_train_full.converted.mean()\n",
    "global_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_origin</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>api</th>\n",
       "      <td>0.308414</td>\n",
       "      <td>-0.074702</td>\n",
       "      <td>0.805014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>landing_page_submission</th>\n",
       "      <td>0.360677</td>\n",
       "      <td>-0.022440</td>\n",
       "      <td>0.941427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_add_form</th>\n",
       "      <td>0.919861</td>\n",
       "      <td>0.536744</td>\n",
       "      <td>2.400992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_import</th>\n",
       "      <td>0.225000</td>\n",
       "      <td>-0.158117</td>\n",
       "      <td>0.587288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quick_add_form</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             mean      diff      risk\n",
       "lead_origin                                          \n",
       "api                      0.308414 -0.074702  0.805014\n",
       "landing_page_submission  0.360677 -0.022440  0.941427\n",
       "lead_add_form            0.919861  0.536744  2.400992\n",
       "lead_import              0.225000 -0.158117  0.587288\n",
       "quick_add_form           1.000000  0.616883  2.610169"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bing</th>\n",
       "      <td>0.166667</td>\n",
       "      <td>-0.216450</td>\n",
       "      <td>0.435028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>blog</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>click2call</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.366883</td>\n",
       "      <td>1.957627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>direct_traffic</th>\n",
       "      <td>0.317624</td>\n",
       "      <td>-0.065493</td>\n",
       "      <td>0.829052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>facebook</th>\n",
       "      <td>0.225000</td>\n",
       "      <td>-0.158117</td>\n",
       "      <td>0.587288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google</th>\n",
       "      <td>0.404567</td>\n",
       "      <td>0.021450</td>\n",
       "      <td>1.055988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live_chat</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nc_edm</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>olark_chat</th>\n",
       "      <td>0.251768</td>\n",
       "      <td>-0.131349</td>\n",
       "      <td>0.657157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organic_search</th>\n",
       "      <td>0.374320</td>\n",
       "      <td>-0.008797</td>\n",
       "      <td>0.977038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pay_per_click_ads</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>press_release</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reference</th>\n",
       "      <td>0.910588</td>\n",
       "      <td>0.527471</td>\n",
       "      <td>2.376790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>referral_sites</th>\n",
       "      <td>0.268041</td>\n",
       "      <td>-0.115076</td>\n",
       "      <td>0.699633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social_media</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>welearn</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>welearnblog_home</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>welingak_website</th>\n",
       "      <td>0.982906</td>\n",
       "      <td>0.599789</td>\n",
       "      <td>2.565551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>youtubechannel</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       mean      diff      risk\n",
       "lead_source                                    \n",
       "bing               0.166667 -0.216450  0.435028\n",
       "blog               0.000000 -0.383117  0.000000\n",
       "click2call         0.750000  0.366883  1.957627\n",
       "direct_traffic     0.317624 -0.065493  0.829052\n",
       "facebook           0.225000 -0.158117  0.587288\n",
       "google             0.404567  0.021450  1.055988\n",
       "live_chat          1.000000  0.616883  2.610169\n",
       "nc_edm             1.000000  0.616883  2.610169\n",
       "olark_chat         0.251768 -0.131349  0.657157\n",
       "organic_search     0.374320 -0.008797  0.977038\n",
       "pay_per_click_ads  0.000000 -0.383117  0.000000\n",
       "press_release      0.000000 -0.383117  0.000000\n",
       "reference          0.910588  0.527471  2.376790\n",
       "referral_sites     0.268041 -0.115076  0.699633\n",
       "social_media       0.500000  0.116883  1.305085\n",
       "welearn            1.000000  0.616883  2.610169\n",
       "welearnblog_home   0.000000 -0.383117  0.000000\n",
       "welingak_website   0.982906  0.599789  2.565551\n",
       "youtubechannel     0.000000 -0.383117  0.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>do_not_email</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.401265</td>\n",
       "      <td>0.018148</td>\n",
       "      <td>1.047368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>0.174281</td>\n",
       "      <td>-0.208836</td>\n",
       "      <td>0.454903</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  mean      diff      risk\n",
       "do_not_email                              \n",
       "no            0.401265  0.018148  1.047368\n",
       "yes           0.174281 -0.208836  0.454903"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>do_not_call</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383033</td>\n",
       "      <td>-0.000083</td>\n",
       "      <td>0.999782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 mean      diff      risk\n",
       "do_not_call                              \n",
       "no           0.383033 -0.000083  0.999782\n",
       "yes          1.000000  0.616883  2.610169"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_activity</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>approached_upfront</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>converted_to_lead</th>\n",
       "      <td>0.126437</td>\n",
       "      <td>-0.256680</td>\n",
       "      <td>0.330021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_bounced</th>\n",
       "      <td>0.092308</td>\n",
       "      <td>-0.290809</td>\n",
       "      <td>0.240939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_link_clicked</th>\n",
       "      <td>0.239437</td>\n",
       "      <td>-0.143680</td>\n",
       "      <td>0.624970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_marked_spam</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_opened</th>\n",
       "      <td>0.375311</td>\n",
       "      <td>-0.007806</td>\n",
       "      <td>0.979626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_received</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>form_submitted_on_website</th>\n",
       "      <td>0.255102</td>\n",
       "      <td>-0.128015</td>\n",
       "      <td>0.665860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>had_a_phone_conversation</th>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.344156</td>\n",
       "      <td>1.898305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>olark_chat_conversation</th>\n",
       "      <td>0.086185</td>\n",
       "      <td>-0.296932</td>\n",
       "      <td>0.224958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>page_visited_on_website</th>\n",
       "      <td>0.231518</td>\n",
       "      <td>-0.151599</td>\n",
       "      <td>0.604300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resubscribed_to_emails</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sms_sent</th>\n",
       "      <td>0.627611</td>\n",
       "      <td>0.244494</td>\n",
       "      <td>1.638172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unreachable</th>\n",
       "      <td>0.313433</td>\n",
       "      <td>-0.069684</td>\n",
       "      <td>0.818113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unsubscribed</th>\n",
       "      <td>0.265306</td>\n",
       "      <td>-0.117811</td>\n",
       "      <td>0.692494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>view_in_browser_link_clicked</th>\n",
       "      <td>0.166667</td>\n",
       "      <td>-0.216450</td>\n",
       "      <td>0.435028</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  mean      diff      risk\n",
       "last_activity                                             \n",
       "approached_upfront            1.000000  0.616883  2.610169\n",
       "converted_to_lead             0.126437 -0.256680  0.330021\n",
       "email_bounced                 0.092308 -0.290809  0.240939\n",
       "email_link_clicked            0.239437 -0.143680  0.624970\n",
       "email_marked_spam             1.000000  0.616883  2.610169\n",
       "email_opened                  0.375311 -0.007806  0.979626\n",
       "email_received                1.000000  0.616883  2.610169\n",
       "form_submitted_on_website     0.255102 -0.128015  0.665860\n",
       "had_a_phone_conversation      0.727273  0.344156  1.898305\n",
       "olark_chat_conversation       0.086185 -0.296932  0.224958\n",
       "page_visited_on_website       0.231518 -0.151599  0.604300\n",
       "resubscribed_to_emails        1.000000  0.616883  2.610169\n",
       "sms_sent                      0.627611  0.244494  1.638172\n",
       "unreachable                   0.313433 -0.069684  0.818113\n",
       "unsubscribed                  0.265306 -0.117811  0.692494\n",
       "view_in_browser_link_clicked  0.166667 -0.216450  0.435028"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asia/pacific_region</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>australia</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>-0.160895</td>\n",
       "      <td>0.580038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bahrain</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.283550</td>\n",
       "      <td>1.740113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bangladesh</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>belgium</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>canada</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>china</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>denmark</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>france</th>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.216883</td>\n",
       "      <td>1.566102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>germany</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>-0.133117</td>\n",
       "      <td>0.652542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ghana</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hong_kong</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>india</th>\n",
       "      <td>0.385923</td>\n",
       "      <td>0.002806</td>\n",
       "      <td>1.007325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>indonesia</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>italy</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kenya</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kuwait</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>liberia</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>netherlands</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nigeria</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oman</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.366883</td>\n",
       "      <td>1.957627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>philippines</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>qatar</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>russia</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>saudi_arabia</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>-0.160895</td>\n",
       "      <td>0.580038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>singapore</th>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.033550</td>\n",
       "      <td>1.087571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>south_africa</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sri_lanka</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sweden</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>switzerland</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uganda</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>united_arab_emirates</th>\n",
       "      <td>0.341463</td>\n",
       "      <td>-0.041653</td>\n",
       "      <td>0.891277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>united_kingdom</th>\n",
       "      <td>0.384615</td>\n",
       "      <td>0.001499</td>\n",
       "      <td>1.003911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>united_states</th>\n",
       "      <td>0.263158</td>\n",
       "      <td>-0.119959</td>\n",
       "      <td>0.686887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unknown</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>-0.049784</td>\n",
       "      <td>0.870056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vietnam</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          mean      diff      risk\n",
       "country                                           \n",
       "asia/pacific_region   0.500000  0.116883  1.305085\n",
       "australia             0.222222 -0.160895  0.580038\n",
       "bahrain               0.666667  0.283550  1.740113\n",
       "bangladesh            0.500000  0.116883  1.305085\n",
       "belgium               0.000000 -0.383117  0.000000\n",
       "canada                0.000000 -0.383117  0.000000\n",
       "china                 0.000000 -0.383117  0.000000\n",
       "denmark               1.000000  0.616883  2.610169\n",
       "france                0.600000  0.216883  1.566102\n",
       "germany               0.250000 -0.133117  0.652542\n",
       "ghana                 0.000000 -0.383117  0.000000\n",
       "hong_kong             0.500000  0.116883  1.305085\n",
       "india                 0.385923  0.002806  1.007325\n",
       "indonesia             0.000000 -0.383117  0.000000\n",
       "italy                 0.000000 -0.383117  0.000000\n",
       "kenya                 0.000000 -0.383117  0.000000\n",
       "kuwait                0.000000 -0.383117  0.000000\n",
       "liberia               0.000000 -0.383117  0.000000\n",
       "netherlands           0.000000 -0.383117  0.000000\n",
       "nigeria               0.000000 -0.383117  0.000000\n",
       "oman                  0.750000  0.366883  1.957627\n",
       "philippines           0.000000 -0.383117  0.000000\n",
       "qatar                 0.000000 -0.383117  0.000000\n",
       "russia                0.000000 -0.383117  0.000000\n",
       "saudi_arabia          0.222222 -0.160895  0.580038\n",
       "singapore             0.416667  0.033550  1.087571\n",
       "south_africa          0.500000  0.116883  1.305085\n",
       "sri_lanka             0.000000 -0.383117  0.000000\n",
       "sweden                0.000000 -0.383117  0.000000\n",
       "switzerland           0.000000 -0.383117  0.000000\n",
       "uganda                0.000000 -0.383117  0.000000\n",
       "united_arab_emirates  0.341463 -0.041653  0.891277\n",
       "united_kingdom        0.384615  0.001499  1.003911\n",
       "united_states         0.263158 -0.119959  0.686887\n",
       "unknown               0.333333 -0.049784  0.870056\n",
       "vietnam               0.000000 -0.383117  0.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specialization</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>banking,_investment_and_insurance</th>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.075216</td>\n",
       "      <td>1.196328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>business_administration</th>\n",
       "      <td>0.457478</td>\n",
       "      <td>0.074361</td>\n",
       "      <td>1.194095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e-business</th>\n",
       "      <td>0.408163</td>\n",
       "      <td>0.025046</td>\n",
       "      <td>1.065375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e-commerce</th>\n",
       "      <td>0.356322</td>\n",
       "      <td>-0.026795</td>\n",
       "      <td>0.930060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>finance_management</th>\n",
       "      <td>0.453865</td>\n",
       "      <td>0.070748</td>\n",
       "      <td>1.184665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>healthcare_management</th>\n",
       "      <td>0.504065</td>\n",
       "      <td>0.120948</td>\n",
       "      <td>1.315695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hospitality_management</th>\n",
       "      <td>0.363636</td>\n",
       "      <td>-0.019481</td>\n",
       "      <td>0.949153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>human_resource_management</th>\n",
       "      <td>0.455474</td>\n",
       "      <td>0.072358</td>\n",
       "      <td>1.188866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>international_business</th>\n",
       "      <td>0.361702</td>\n",
       "      <td>-0.021415</td>\n",
       "      <td>0.944104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>it_projects_management</th>\n",
       "      <td>0.375427</td>\n",
       "      <td>-0.007690</td>\n",
       "      <td>0.979927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>marketing_management</th>\n",
       "      <td>0.479699</td>\n",
       "      <td>0.096582</td>\n",
       "      <td>1.252096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>media_and_advertising</th>\n",
       "      <td>0.411043</td>\n",
       "      <td>0.027926</td>\n",
       "      <td>1.072892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>operations_management</th>\n",
       "      <td>0.470588</td>\n",
       "      <td>0.087471</td>\n",
       "      <td>1.228315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>retail_management</th>\n",
       "      <td>0.357143</td>\n",
       "      <td>-0.025974</td>\n",
       "      <td>0.932203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rural_and_agribusiness</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.016883</td>\n",
       "      <td>1.044068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>select</th>\n",
       "      <td>0.285024</td>\n",
       "      <td>-0.098093</td>\n",
       "      <td>0.743961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>services_excellence</th>\n",
       "      <td>0.310345</td>\n",
       "      <td>-0.072772</td>\n",
       "      <td>0.810053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>supply_chain_management</th>\n",
       "      <td>0.423488</td>\n",
       "      <td>0.040371</td>\n",
       "      <td>1.105374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>travel_and_tourism</th>\n",
       "      <td>0.350000</td>\n",
       "      <td>-0.033117</td>\n",
       "      <td>0.913559</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       mean      diff      risk\n",
       "specialization                                                 \n",
       "banking,_investment_and_insurance  0.458333  0.075216  1.196328\n",
       "business_administration            0.457478  0.074361  1.194095\n",
       "e-business                         0.408163  0.025046  1.065375\n",
       "e-commerce                         0.356322 -0.026795  0.930060\n",
       "finance_management                 0.453865  0.070748  1.184665\n",
       "healthcare_management              0.504065  0.120948  1.315695\n",
       "hospitality_management             0.363636 -0.019481  0.949153\n",
       "human_resource_management          0.455474  0.072358  1.188866\n",
       "international_business             0.361702 -0.021415  0.944104\n",
       "it_projects_management             0.375427 -0.007690  0.979927\n",
       "marketing_management               0.479699  0.096582  1.252096\n",
       "media_and_advertising              0.411043  0.027926  1.072892\n",
       "operations_management              0.470588  0.087471  1.228315\n",
       "retail_management                  0.357143 -0.025974  0.932203\n",
       "rural_and_agribusiness             0.400000  0.016883  1.044068\n",
       "select                             0.285024 -0.098093  0.743961\n",
       "services_excellence                0.310345 -0.072772  0.810053\n",
       "supply_chain_management            0.423488  0.040371  1.105374\n",
       "travel_and_tourism                 0.350000 -0.033117  0.913559"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>how_did_you_hear_about_x_education</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>advertisements</th>\n",
       "      <td>0.450980</td>\n",
       "      <td>0.067864</td>\n",
       "      <td>1.177135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email</th>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.037936</td>\n",
       "      <td>1.099019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multiple_sources</th>\n",
       "      <td>0.349593</td>\n",
       "      <td>-0.033523</td>\n",
       "      <td>0.912498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>online_search</th>\n",
       "      <td>0.425868</td>\n",
       "      <td>0.042751</td>\n",
       "      <td>1.111586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.412903</td>\n",
       "      <td>0.029786</td>\n",
       "      <td>1.077747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>select</th>\n",
       "      <td>0.371148</td>\n",
       "      <td>-0.011969</td>\n",
       "      <td>0.968760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sms</th>\n",
       "      <td>0.200000</td>\n",
       "      <td>-0.183117</td>\n",
       "      <td>0.522034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social_media</th>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>1.118644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>student_of_someschool</th>\n",
       "      <td>0.475410</td>\n",
       "      <td>0.092293</td>\n",
       "      <td>1.240900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_of_mouth</th>\n",
       "      <td>0.441281</td>\n",
       "      <td>0.058164</td>\n",
       "      <td>1.151819</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        mean      diff      risk\n",
       "how_did_you_hear_about_x_education                              \n",
       "advertisements                      0.450980  0.067864  1.177135\n",
       "email                               0.421053  0.037936  1.099019\n",
       "multiple_sources                    0.349593 -0.033523  0.912498\n",
       "online_search                       0.425868  0.042751  1.111586\n",
       "other                               0.412903  0.029786  1.077747\n",
       "select                              0.371148 -0.011969  0.968760\n",
       "sms                                 0.200000 -0.183117  0.522034\n",
       "social_media                        0.428571  0.045455  1.118644\n",
       "student_of_someschool               0.475410  0.092293  1.240900\n",
       "word_of_mouth                       0.441281  0.058164  1.151819"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>what_is_your_current_occupation</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>businessman</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>housewife</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.232268</td>\n",
       "      <td>1.606258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>student</th>\n",
       "      <td>0.359281</td>\n",
       "      <td>-0.023835</td>\n",
       "      <td>0.937785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unemployed</th>\n",
       "      <td>0.337451</td>\n",
       "      <td>-0.045666</td>\n",
       "      <td>0.880804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>working_professional</th>\n",
       "      <td>0.916071</td>\n",
       "      <td>0.532955</td>\n",
       "      <td>2.391102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     mean      diff      risk\n",
       "what_is_your_current_occupation                              \n",
       "businessman                      0.500000  0.116883  1.305085\n",
       "housewife                        1.000000  0.616883  2.610169\n",
       "other                            0.615385  0.232268  1.606258\n",
       "student                          0.359281 -0.023835  0.937785\n",
       "unemployed                       0.337451 -0.045666  0.880804\n",
       "working_professional             0.916071  0.532955  2.391102"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>what_matters_most_to_you_in_choosing_a_course</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>better_career_prospects</th>\n",
       "      <td>0.383085</td>\n",
       "      <td>-0.000032</td>\n",
       "      <td>0.999917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flexibility_&amp;_convenience</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   mean      diff      risk\n",
       "what_matters_most_to_you_in_choosing_a_course                              \n",
       "better_career_prospects                        0.383085 -0.000032  0.999917\n",
       "flexibility_&_convenience                      0.500000  0.116883  1.305085"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>search</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383146</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>1.000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>0.363636</td>\n",
       "      <td>-0.019481</td>\n",
       "      <td>0.949153</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            mean      diff      risk\n",
       "search                              \n",
       "no      0.383146  0.000029  1.000076\n",
       "yes     0.363636 -0.019481  0.949153"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magazine</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              mean  diff  risk\n",
       "magazine                      \n",
       "no        0.383117   0.0   1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newspaper_article</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383033</td>\n",
       "      <td>-0.000083</td>\n",
       "      <td>0.999782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       mean      diff      risk\n",
       "newspaper_article                              \n",
       "no                 0.383033 -0.000083  0.999782\n",
       "yes                1.000000  0.616883  2.610169"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>digital_advertisement</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383221</td>\n",
       "      <td>0.000104</td>\n",
       "      <td>1.000271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           mean      diff      risk\n",
       "digital_advertisement                              \n",
       "no                     0.383221  0.000104  1.000271\n",
       "yes                    0.000000 -0.383117  0.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>through_recommendations</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.382918</td>\n",
       "      <td>-0.000199</td>\n",
       "      <td>0.999482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.366883</td>\n",
       "      <td>1.957627</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             mean      diff      risk\n",
       "through_recommendations                              \n",
       "no                       0.382918 -0.000199  0.999482\n",
       "yes                      0.750000  0.366883  1.957627"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>receive_more_updates_about_our_courses</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            mean  diff  risk\n",
       "receive_more_updates_about_our_courses                      \n",
       "no                                      0.383117   0.0   1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tags</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>already_a_student</th>\n",
       "      <td>0.007752</td>\n",
       "      <td>-0.375365</td>\n",
       "      <td>0.020234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>busy</th>\n",
       "      <td>0.567742</td>\n",
       "      <td>0.184625</td>\n",
       "      <td>1.481903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>closed_by_horizzon</th>\n",
       "      <td>0.996454</td>\n",
       "      <td>0.613337</td>\n",
       "      <td>2.600914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diploma_holder_(not_eligible)</th>\n",
       "      <td>0.020833</td>\n",
       "      <td>-0.362284</td>\n",
       "      <td>0.054379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>graduation_in_progress</th>\n",
       "      <td>0.063830</td>\n",
       "      <td>-0.319287</td>\n",
       "      <td>0.166607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>in_confusion_whether_part_time_or_dlp</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>-0.133117</td>\n",
       "      <td>0.652542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>in_touch_with_eins</th>\n",
       "      <td>0.272727</td>\n",
       "      <td>-0.110390</td>\n",
       "      <td>0.711864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interested__in_full_time_mba</th>\n",
       "      <td>0.010989</td>\n",
       "      <td>-0.372128</td>\n",
       "      <td>0.028683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interested_in_next_batch</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interested_in_other_courses</th>\n",
       "      <td>0.023810</td>\n",
       "      <td>-0.359307</td>\n",
       "      <td>0.062147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>invalid_number</th>\n",
       "      <td>0.014493</td>\n",
       "      <td>-0.368624</td>\n",
       "      <td>0.037829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lateral_student</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lost_to_eins</th>\n",
       "      <td>0.979310</td>\n",
       "      <td>0.596193</td>\n",
       "      <td>2.556166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lost_to_others</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not_doing_further_education</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number_not_provided</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>opp_hangup</th>\n",
       "      <td>0.068966</td>\n",
       "      <td>-0.314151</td>\n",
       "      <td>0.180012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recognition_issue_(dec_approval)</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ringing</th>\n",
       "      <td>0.028332</td>\n",
       "      <td>-0.354785</td>\n",
       "      <td>0.073950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shall_take_in_the_next_coming_month</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.116883</td>\n",
       "      <td>1.305085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>still_thinking</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>switched_off</th>\n",
       "      <td>0.020408</td>\n",
       "      <td>-0.362709</td>\n",
       "      <td>0.053269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>university_not_recognized</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>want_to_take_admission_but_has_financial_problems</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>-0.133117</td>\n",
       "      <td>0.652542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>will_revert_after_reading_the_email</th>\n",
       "      <td>0.523223</td>\n",
       "      <td>0.140107</td>\n",
       "      <td>1.365702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wrong_number_given</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       mean      diff  \\\n",
       "tags                                                                    \n",
       "already_a_student                                  0.007752 -0.375365   \n",
       "busy                                               0.567742  0.184625   \n",
       "closed_by_horizzon                                 0.996454  0.613337   \n",
       "diploma_holder_(not_eligible)                      0.020833 -0.362284   \n",
       "graduation_in_progress                             0.063830 -0.319287   \n",
       "in_confusion_whether_part_time_or_dlp              0.250000 -0.133117   \n",
       "in_touch_with_eins                                 0.272727 -0.110390   \n",
       "interested__in_full_time_mba                       0.010989 -0.372128   \n",
       "interested_in_next_batch                           1.000000  0.616883   \n",
       "interested_in_other_courses                        0.023810 -0.359307   \n",
       "invalid_number                                     0.014493 -0.368624   \n",
       "lateral_student                                    1.000000  0.616883   \n",
       "lost_to_eins                                       0.979310  0.596193   \n",
       "lost_to_others                                     0.000000 -0.383117   \n",
       "not_doing_further_education                        0.000000 -0.383117   \n",
       "number_not_provided                                0.000000 -0.383117   \n",
       "opp_hangup                                         0.068966 -0.314151   \n",
       "recognition_issue_(dec_approval)                   0.000000 -0.383117   \n",
       "ringing                                            0.028332 -0.354785   \n",
       "shall_take_in_the_next_coming_month                0.500000  0.116883   \n",
       "still_thinking                                     0.000000 -0.383117   \n",
       "switched_off                                       0.020408 -0.362709   \n",
       "university_not_recognized                          0.000000 -0.383117   \n",
       "want_to_take_admission_but_has_financial_problems  0.250000 -0.133117   \n",
       "will_revert_after_reading_the_email                0.523223  0.140107   \n",
       "wrong_number_given                                 0.000000 -0.383117   \n",
       "\n",
       "                                                       risk  \n",
       "tags                                                         \n",
       "already_a_student                                  0.020234  \n",
       "busy                                               1.481903  \n",
       "closed_by_horizzon                                 2.600914  \n",
       "diploma_holder_(not_eligible)                      0.054379  \n",
       "graduation_in_progress                             0.166607  \n",
       "in_confusion_whether_part_time_or_dlp              0.652542  \n",
       "in_touch_with_eins                                 0.711864  \n",
       "interested__in_full_time_mba                       0.028683  \n",
       "interested_in_next_batch                           2.610169  \n",
       "interested_in_other_courses                        0.062147  \n",
       "invalid_number                                     0.037829  \n",
       "lateral_student                                    2.610169  \n",
       "lost_to_eins                                       2.556166  \n",
       "lost_to_others                                     0.000000  \n",
       "not_doing_further_education                        0.000000  \n",
       "number_not_provided                                0.000000  \n",
       "opp_hangup                                         0.180012  \n",
       "recognition_issue_(dec_approval)                   0.000000  \n",
       "ringing                                            0.073950  \n",
       "shall_take_in_the_next_coming_month                1.305085  \n",
       "still_thinking                                     0.000000  \n",
       "switched_off                                       0.053269  \n",
       "university_not_recognized                          0.000000  \n",
       "want_to_take_admission_but_has_financial_problems  0.652542  \n",
       "will_revert_after_reading_the_email                1.365702  \n",
       "wrong_number_given                                 0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_quality</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>high_in_relevance</th>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.565601</td>\n",
       "      <td>2.476315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>low_in_relevance</th>\n",
       "      <td>0.803063</td>\n",
       "      <td>0.419947</td>\n",
       "      <td>2.096132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>might_be</th>\n",
       "      <td>0.348024</td>\n",
       "      <td>-0.035093</td>\n",
       "      <td>0.908401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not_sure</th>\n",
       "      <td>0.242597</td>\n",
       "      <td>-0.140520</td>\n",
       "      <td>0.633219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worst</th>\n",
       "      <td>0.020408</td>\n",
       "      <td>-0.362709</td>\n",
       "      <td>0.053269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       mean      diff      risk\n",
       "lead_quality                                   \n",
       "high_in_relevance  0.948718  0.565601  2.476315\n",
       "low_in_relevance   0.803063  0.419947  2.096132\n",
       "might_be           0.348024 -0.035093  0.908401\n",
       "not_sure           0.242597 -0.140520  0.633219\n",
       "worst              0.020408 -0.362709  0.053269"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>update_me_on_supply_chain_content</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       mean  diff  risk\n",
       "update_me_on_supply_chain_content                      \n",
       "no                                 0.383117   0.0   1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_profile</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dual_specialization_student</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lateral_student</th>\n",
       "      <td>0.952381</td>\n",
       "      <td>0.569264</td>\n",
       "      <td>2.485876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_leads</th>\n",
       "      <td>0.351621</td>\n",
       "      <td>-0.031496</td>\n",
       "      <td>0.917790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>potential_lead</th>\n",
       "      <td>0.781591</td>\n",
       "      <td>0.398474</td>\n",
       "      <td>2.040086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>select</th>\n",
       "      <td>0.301152</td>\n",
       "      <td>-0.081965</td>\n",
       "      <td>0.786058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>student_of_someschool</th>\n",
       "      <td>0.024752</td>\n",
       "      <td>-0.358364</td>\n",
       "      <td>0.064608</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 mean      diff      risk\n",
       "lead_profile                                             \n",
       "dual_specialization_student  1.000000  0.616883  2.610169\n",
       "lateral_student              0.952381  0.569264  2.485876\n",
       "other_leads                  0.351621 -0.031496  0.917790\n",
       "potential_lead               0.781591  0.398474  2.040086\n",
       "select                       0.301152 -0.081965  0.786058\n",
       "student_of_someschool        0.024752 -0.358364  0.064608"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mumbai</th>\n",
       "      <td>0.312331</td>\n",
       "      <td>-0.070786</td>\n",
       "      <td>0.815237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_cities</th>\n",
       "      <td>0.408015</td>\n",
       "      <td>0.024898</td>\n",
       "      <td>1.064987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_cities_of_maharashtra</th>\n",
       "      <td>0.437326</td>\n",
       "      <td>0.054209</td>\n",
       "      <td>1.141495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other_metro_cities</th>\n",
       "      <td>0.401961</td>\n",
       "      <td>0.018844</td>\n",
       "      <td>1.049186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>select</th>\n",
       "      <td>0.485479</td>\n",
       "      <td>0.102363</td>\n",
       "      <td>1.267184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thane_&amp;_outskirts</th>\n",
       "      <td>0.447412</td>\n",
       "      <td>0.064295</td>\n",
       "      <td>1.167822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tier_ii_cities</th>\n",
       "      <td>0.339286</td>\n",
       "      <td>-0.043831</td>\n",
       "      <td>0.885593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 mean      diff      risk\n",
       "city                                                     \n",
       "mumbai                       0.312331 -0.070786  0.815237\n",
       "other_cities                 0.408015  0.024898  1.064987\n",
       "other_cities_of_maharashtra  0.437326  0.054209  1.141495\n",
       "other_metro_cities           0.401961  0.018844  1.049186\n",
       "select                       0.485479  0.102363  1.267184\n",
       "thane_&_outskirts            0.447412  0.064295  1.167822\n",
       "tier_ii_cities               0.339286 -0.043831  0.885593"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_activity_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>01.high</th>\n",
       "      <td>0.313522</td>\n",
       "      <td>-0.069595</td>\n",
       "      <td>0.818344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02.medium</th>\n",
       "      <td>0.404083</td>\n",
       "      <td>0.020966</td>\n",
       "      <td>1.054725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03.low</th>\n",
       "      <td>0.092715</td>\n",
       "      <td>-0.290402</td>\n",
       "      <td>0.242002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 mean      diff      risk\n",
       "asymmetrique_activity_index                              \n",
       "01.high                      0.313522 -0.069595  0.818344\n",
       "02.medium                    0.404083  0.020966  1.054725\n",
       "03.low                       0.092715 -0.290402  0.242002"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_profile_index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>01.high</th>\n",
       "      <td>0.482525</td>\n",
       "      <td>0.099408</td>\n",
       "      <td>1.259473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02.medium</th>\n",
       "      <td>0.351448</td>\n",
       "      <td>-0.031669</td>\n",
       "      <td>0.917339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03.low</th>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.033550</td>\n",
       "      <td>1.087571</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                mean      diff      risk\n",
       "asymmetrique_profile_index                              \n",
       "01.high                     0.482525  0.099408  1.259473\n",
       "02.medium                   0.351448 -0.031669  0.917339\n",
       "03.low                      0.416667  0.033550  1.087571"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i_agree_to_pay_the_amount_through_cheque</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              mean  diff  risk\n",
       "i_agree_to_pay_the_amount_through_cheque                      \n",
       "no                                        0.383117   0.0   1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a_free_copy_of_mastering_the_interview</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.39578</td>\n",
       "      <td>0.012663</td>\n",
       "      <td>1.033053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>0.35545</td>\n",
       "      <td>-0.027667</td>\n",
       "      <td>0.927785</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           mean      diff      risk\n",
       "a_free_copy_of_mastering_the_interview                             \n",
       "no                                      0.39578  0.012663  1.033053\n",
       "yes                                     0.35545 -0.027667  0.927785"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_notable_activity</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>approached_upfront</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_bounced</th>\n",
       "      <td>0.160000</td>\n",
       "      <td>-0.223117</td>\n",
       "      <td>0.417627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_link_clicked</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>-0.160895</td>\n",
       "      <td>0.580038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_marked_spam</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_opened</th>\n",
       "      <td>0.364362</td>\n",
       "      <td>-0.018755</td>\n",
       "      <td>0.951046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email_received</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>form_submitted_on_website</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>had_a_phone_conversation</th>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.516883</td>\n",
       "      <td>2.349153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>modified</th>\n",
       "      <td>0.230430</td>\n",
       "      <td>-0.152687</td>\n",
       "      <td>0.601461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>olark_chat_conversation</th>\n",
       "      <td>0.156863</td>\n",
       "      <td>-0.226254</td>\n",
       "      <td>0.409438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>page_visited_on_website</th>\n",
       "      <td>0.294355</td>\n",
       "      <td>-0.088762</td>\n",
       "      <td>0.768316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resubscribed_to_emails</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.616883</td>\n",
       "      <td>2.610169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sms_sent</th>\n",
       "      <td>0.690286</td>\n",
       "      <td>0.307169</td>\n",
       "      <td>1.801763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unreachable</th>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.253247</td>\n",
       "      <td>1.661017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unsubscribed</th>\n",
       "      <td>0.300000</td>\n",
       "      <td>-0.083117</td>\n",
       "      <td>0.783051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>view_in_browser_link_clicked</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  mean      diff      risk\n",
       "last_notable_activity                                     \n",
       "approached_upfront            1.000000  0.616883  2.610169\n",
       "email_bounced                 0.160000 -0.223117  0.417627\n",
       "email_link_clicked            0.222222 -0.160895  0.580038\n",
       "email_marked_spam             1.000000  0.616883  2.610169\n",
       "email_opened                  0.364362 -0.018755  0.951046\n",
       "email_received                1.000000  0.616883  2.610169\n",
       "form_submitted_on_website     0.000000 -0.383117  0.000000\n",
       "had_a_phone_conversation      0.900000  0.516883  2.349153\n",
       "modified                      0.230430 -0.152687  0.601461\n",
       "olark_chat_conversation       0.156863 -0.226254  0.409438\n",
       "page_visited_on_website       0.294355 -0.088762  0.768316\n",
       "resubscribed_to_emails        1.000000  0.616883  2.610169\n",
       "sms_sent                      0.690286  0.307169  1.801763\n",
       "unreachable                   0.636364  0.253247  1.661017\n",
       "unsubscribed                  0.300000 -0.083117  0.783051\n",
       "view_in_browser_link_clicked  0.000000 -0.383117  0.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>newspaper</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383169</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>1.000135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yes</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.383117</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               mean      diff      risk\n",
       "newspaper                              \n",
       "no         0.383169  0.000052  1.000135\n",
       "yes        0.000000 -0.383117  0.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_education_forums</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        mean  diff  risk\n",
       "x_education_forums                      \n",
       "no                  0.383117   0.0   1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>diff</th>\n",
       "      <th>risk</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>get_updates_on_dm_content</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>no</th>\n",
       "      <td>0.383117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               mean  diff  risk\n",
       "get_updates_on_dm_content                      \n",
       "no                         0.383117   0.0   1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for col in categorical:\n",
    "    df_group = df_train_full.groupby(by=col).converted.agg(['mean'])\n",
    "    df_group['diff'] = df_group['mean'] - global_mean\n",
    "    df_group['risk'] = df_group['mean'] / global_mean\n",
    "    display(df_group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mutual_info_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tags</th>\n",
       "      <td>0.209400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_quality</th>\n",
       "      <td>0.106262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_activity</th>\n",
       "      <td>0.084629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_profile</th>\n",
       "      <td>0.082977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_notable_activity</th>\n",
       "      <td>0.072517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             MI\n",
       "tags                   0.209400\n",
       "lead_quality           0.106262\n",
       "last_activity          0.084629\n",
       "lead_profile           0.082977\n",
       "last_notable_activity  0.072517"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>update_me_on_supply_chain_content</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magazine</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>i_agree_to_pay_the_amount_through_cheque</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_education_forums</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>get_updates_on_dm_content</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           MI\n",
       "update_me_on_supply_chain_content         0.0\n",
       "magazine                                  0.0\n",
       "i_agree_to_pay_the_amount_through_cheque  0.0\n",
       "x_education_forums                        0.0\n",
       "get_updates_on_dm_content                 0.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def calculate_mi(series):\n",
    "    return mutual_info_score(series, df_train_full.converted)\n",
    "\n",
    "df_mi = df_train_full[categorical].apply(calculate_mi)\n",
    "df_mi = df_mi.sort_values(ascending=False).to_frame(name='MI')\n",
    "\n",
    "\n",
    "display(df_mi.head())\n",
    "display(df_mi.tail())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importance: Correlation Coefficient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>correlation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>lead_number</th>\n",
       "      <td>0.021904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>totalvisits</th>\n",
       "      <td>0.033282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_time_spent_on_website</th>\n",
       "      <td>0.363470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>page_views_per_visit</th>\n",
       "      <td>0.001532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_activity_score</th>\n",
       "      <td>0.122210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asymmetrique_profile_score</th>\n",
       "      <td>0.165779</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             correlation\n",
       "lead_number                     0.021904\n",
       "totalvisits                     0.033282\n",
       "total_time_spent_on_website     0.363470\n",
       "page_views_per_visit            0.001532\n",
       "asymmetrique_activity_score     0.122210\n",
       "asymmetrique_profile_score      0.165779"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_full[numerical].corrwith(df_train_full.converted).to_frame('correlation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lead_number</th>\n",
       "      <th>totalvisits</th>\n",
       "      <th>total_time_spent_on_website</th>\n",
       "      <th>page_views_per_visit</th>\n",
       "      <th>asymmetrique_activity_score</th>\n",
       "      <th>asymmetrique_profile_score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>converted</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>616790.432018</td>\n",
       "      <td>3.328928</td>\n",
       "      <td>331.447149</td>\n",
       "      <td>2.369454</td>\n",
       "      <td>14.207726</td>\n",
       "      <td>16.181521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>617845.297316</td>\n",
       "      <td>3.676262</td>\n",
       "      <td>740.735876</td>\n",
       "      <td>2.376293</td>\n",
       "      <td>14.467645</td>\n",
       "      <td>16.633914</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             lead_number  totalvisits  total_time_spent_on_website  \\\n",
       "converted                                                            \n",
       "0          616790.432018     3.328928                   331.447149   \n",
       "1          617845.297316     3.676262                   740.735876   \n",
       "\n",
       "           page_views_per_visit  asymmetrique_activity_score  \\\n",
       "converted                                                      \n",
       "0                      2.369454                    14.207726   \n",
       "1                      2.376293                    14.467645   \n",
       "\n",
       "           asymmetrique_profile_score  \n",
       "converted                              \n",
       "0                           16.181521  \n",
       "1                           16.633914  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_full.groupby(by='converted')[numerical].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-hot encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction import DictVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train_dict = df_train[categorical + numerical].to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lead_origin</th>\n",
       "      <th>lead_source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>organic_search</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>direct_traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>direct_traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>api</td>\n",
       "      <td>olark_chat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>direct_traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>direct_traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>api</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>direct_traffic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>api</td>\n",
       "      <td>olark_chat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>landing_page_submission</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               lead_origin     lead_source\n",
       "0  landing_page_submission  organic_search\n",
       "1  landing_page_submission  direct_traffic\n",
       "2  landing_page_submission  direct_traffic\n",
       "3                      api      olark_chat\n",
       "4  landing_page_submission  direct_traffic\n",
       "5  landing_page_submission  direct_traffic\n",
       "6                      api          google\n",
       "7  landing_page_submission  direct_traffic\n",
       "8                      api      olark_chat\n",
       "9  landing_page_submission          google"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[['lead_origin','lead_source']].iloc[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lead_origin': 'landing_page_submission',\n",
       " 'lead_source': 'organic_search',\n",
       " 'do_not_email': 'no',\n",
       " 'do_not_call': 'no',\n",
       " 'last_activity': 'sms_sent',\n",
       " 'country': 'netherlands',\n",
       " 'specialization': 'human_resource_management',\n",
       " 'how_did_you_hear_about_x_education': 'select',\n",
       " 'what_is_your_current_occupation': 'unemployed',\n",
       " 'what_matters_most_to_you_in_choosing_a_course': 'better_career_prospects',\n",
       " 'search': 'no',\n",
       " 'magazine': 'no',\n",
       " 'newspaper_article': 'no',\n",
       " 'digital_advertisement': 'no',\n",
       " 'through_recommendations': 'no',\n",
       " 'receive_more_updates_about_our_courses': 'no',\n",
       " 'tags': 'ringing',\n",
       " 'lead_quality': 'not_sure',\n",
       " 'update_me_on_supply_chain_content': 'no',\n",
       " 'lead_profile': 'select',\n",
       " 'city': 'mumbai',\n",
       " 'asymmetrique_activity_index': '02.medium',\n",
       " 'asymmetrique_profile_index': '02.medium',\n",
       " 'i_agree_to_pay_the_amount_through_cheque': 'no',\n",
       " 'a_free_copy_of_mastering_the_interview': 'yes',\n",
       " 'last_notable_activity': 'sms_sent',\n",
       " 'newspaper': 'no',\n",
       " 'x_education_forums': 'no',\n",
       " 'get_updates_on_dm_content': 'no',\n",
       " 'lead_number': 596953,\n",
       " 'totalvisits': 3.0,\n",
       " 'total_time_spent_on_website': 1328,\n",
       " 'page_views_per_visit': 1.5,\n",
       " 'asymmetrique_activity_score': 14.306252489048187,\n",
       " 'asymmetrique_profile_score': 16.344882516925527}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dict[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>DictVectorizer(sparse=False)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DictVectorizer</label><div class=\"sk-toggleable__content\"><pre>DictVectorizer(sparse=False)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "DictVectorizer(sparse=False)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dv = DictVectorizer(sparse=False)\n",
    "dv.fit(train_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = dv.transform(train_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5544, 202)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1848,)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['a_free_copy_of_mastering_the_interview=no',\n",
       " 'a_free_copy_of_mastering_the_interview=yes',\n",
       " 'asymmetrique_activity_index=01.high',\n",
       " 'asymmetrique_activity_index=02.medium',\n",
       " 'asymmetrique_activity_index=03.low',\n",
       " 'asymmetrique_activity_score',\n",
       " 'asymmetrique_profile_index=01.high',\n",
       " 'asymmetrique_profile_index=02.medium',\n",
       " 'asymmetrique_profile_index=03.low',\n",
       " 'asymmetrique_profile_score',\n",
       " 'city=mumbai',\n",
       " 'city=other_cities',\n",
       " 'city=other_cities_of_maharashtra',\n",
       " 'city=other_metro_cities',\n",
       " 'city=select',\n",
       " 'city=thane_&_outskirts',\n",
       " 'city=tier_ii_cities',\n",
       " 'country=asia/pacific_region',\n",
       " 'country=australia',\n",
       " 'country=bahrain',\n",
       " 'country=bangladesh',\n",
       " 'country=belgium',\n",
       " 'country=canada',\n",
       " 'country=china',\n",
       " 'country=denmark',\n",
       " 'country=france',\n",
       " 'country=germany',\n",
       " 'country=ghana',\n",
       " 'country=hong_kong',\n",
       " 'country=india',\n",
       " 'country=italy',\n",
       " 'country=kenya',\n",
       " 'country=kuwait',\n",
       " 'country=liberia',\n",
       " 'country=netherlands',\n",
       " 'country=nigeria',\n",
       " 'country=oman',\n",
       " 'country=philippines',\n",
       " 'country=qatar',\n",
       " 'country=russia',\n",
       " 'country=saudi_arabia',\n",
       " 'country=singapore',\n",
       " 'country=south_africa',\n",
       " 'country=sri_lanka',\n",
       " 'country=sweden',\n",
       " 'country=switzerland',\n",
       " 'country=uganda',\n",
       " 'country=united_arab_emirates',\n",
       " 'country=united_kingdom',\n",
       " 'country=united_states',\n",
       " 'country=unknown',\n",
       " 'country=vietnam',\n",
       " 'digital_advertisement=no',\n",
       " 'digital_advertisement=yes',\n",
       " 'do_not_call=no',\n",
       " 'do_not_call=yes',\n",
       " 'do_not_email=no',\n",
       " 'do_not_email=yes',\n",
       " 'get_updates_on_dm_content=no',\n",
       " 'how_did_you_hear_about_x_education=advertisements',\n",
       " 'how_did_you_hear_about_x_education=email',\n",
       " 'how_did_you_hear_about_x_education=multiple_sources',\n",
       " 'how_did_you_hear_about_x_education=online_search',\n",
       " 'how_did_you_hear_about_x_education=other',\n",
       " 'how_did_you_hear_about_x_education=select',\n",
       " 'how_did_you_hear_about_x_education=sms',\n",
       " 'how_did_you_hear_about_x_education=social_media',\n",
       " 'how_did_you_hear_about_x_education=student_of_someschool',\n",
       " 'how_did_you_hear_about_x_education=word_of_mouth',\n",
       " 'i_agree_to_pay_the_amount_through_cheque=no',\n",
       " 'last_activity=approached_upfront',\n",
       " 'last_activity=converted_to_lead',\n",
       " 'last_activity=email_bounced',\n",
       " 'last_activity=email_link_clicked',\n",
       " 'last_activity=email_marked_spam',\n",
       " 'last_activity=email_opened',\n",
       " 'last_activity=email_received',\n",
       " 'last_activity=form_submitted_on_website',\n",
       " 'last_activity=had_a_phone_conversation',\n",
       " 'last_activity=olark_chat_conversation',\n",
       " 'last_activity=page_visited_on_website',\n",
       " 'last_activity=resubscribed_to_emails',\n",
       " 'last_activity=sms_sent',\n",
       " 'last_activity=unreachable',\n",
       " 'last_activity=unsubscribed',\n",
       " 'last_activity=view_in_browser_link_clicked',\n",
       " 'last_notable_activity=approached_upfront',\n",
       " 'last_notable_activity=email_bounced',\n",
       " 'last_notable_activity=email_link_clicked',\n",
       " 'last_notable_activity=email_marked_spam',\n",
       " 'last_notable_activity=email_opened',\n",
       " 'last_notable_activity=email_received',\n",
       " 'last_notable_activity=form_submitted_on_website',\n",
       " 'last_notable_activity=had_a_phone_conversation',\n",
       " 'last_notable_activity=modified',\n",
       " 'last_notable_activity=olark_chat_conversation',\n",
       " 'last_notable_activity=page_visited_on_website',\n",
       " 'last_notable_activity=resubscribed_to_emails',\n",
       " 'last_notable_activity=sms_sent',\n",
       " 'last_notable_activity=unreachable',\n",
       " 'last_notable_activity=unsubscribed',\n",
       " 'last_notable_activity=view_in_browser_link_clicked',\n",
       " 'lead_number',\n",
       " 'lead_origin=api',\n",
       " 'lead_origin=landing_page_submission',\n",
       " 'lead_origin=lead_add_form',\n",
       " 'lead_origin=lead_import',\n",
       " 'lead_origin=quick_add_form',\n",
       " 'lead_profile=dual_specialization_student',\n",
       " 'lead_profile=lateral_student',\n",
       " 'lead_profile=other_leads',\n",
       " 'lead_profile=potential_lead',\n",
       " 'lead_profile=select',\n",
       " 'lead_profile=student_of_someschool',\n",
       " 'lead_quality=high_in_relevance',\n",
       " 'lead_quality=low_in_relevance',\n",
       " 'lead_quality=might_be',\n",
       " 'lead_quality=not_sure',\n",
       " 'lead_quality=worst',\n",
       " 'lead_source=bing',\n",
       " 'lead_source=blog',\n",
       " 'lead_source=click2call',\n",
       " 'lead_source=direct_traffic',\n",
       " 'lead_source=facebook',\n",
       " 'lead_source=google',\n",
       " 'lead_source=live_chat',\n",
       " 'lead_source=nc_edm',\n",
       " 'lead_source=olark_chat',\n",
       " 'lead_source=organic_search',\n",
       " 'lead_source=pay_per_click_ads',\n",
       " 'lead_source=press_release',\n",
       " 'lead_source=reference',\n",
       " 'lead_source=referral_sites',\n",
       " 'lead_source=social_media',\n",
       " 'lead_source=welearn',\n",
       " 'lead_source=welearnblog_home',\n",
       " 'lead_source=welingak_website',\n",
       " 'magazine=no',\n",
       " 'newspaper=no',\n",
       " 'newspaper=yes',\n",
       " 'newspaper_article=no',\n",
       " 'page_views_per_visit',\n",
       " 'receive_more_updates_about_our_courses=no',\n",
       " 'search=no',\n",
       " 'search=yes',\n",
       " 'specialization=banking,_investment_and_insurance',\n",
       " 'specialization=business_administration',\n",
       " 'specialization=e-business',\n",
       " 'specialization=e-commerce',\n",
       " 'specialization=finance_management',\n",
       " 'specialization=healthcare_management',\n",
       " 'specialization=hospitality_management',\n",
       " 'specialization=human_resource_management',\n",
       " 'specialization=international_business',\n",
       " 'specialization=it_projects_management',\n",
       " 'specialization=marketing_management',\n",
       " 'specialization=media_and_advertising',\n",
       " 'specialization=operations_management',\n",
       " 'specialization=retail_management',\n",
       " 'specialization=rural_and_agribusiness',\n",
       " 'specialization=select',\n",
       " 'specialization=services_excellence',\n",
       " 'specialization=supply_chain_management',\n",
       " 'specialization=travel_and_tourism',\n",
       " 'tags=already_a_student',\n",
       " 'tags=busy',\n",
       " 'tags=closed_by_horizzon',\n",
       " 'tags=diploma_holder_(not_eligible)',\n",
       " 'tags=graduation_in_progress',\n",
       " 'tags=in_confusion_whether_part_time_or_dlp',\n",
       " 'tags=in_touch_with_eins',\n",
       " 'tags=interested__in_full_time_mba',\n",
       " 'tags=interested_in_next_batch',\n",
       " 'tags=interested_in_other_courses',\n",
       " 'tags=invalid_number',\n",
       " 'tags=lost_to_eins',\n",
       " 'tags=lost_to_others',\n",
       " 'tags=not_doing_further_education',\n",
       " 'tags=number_not_provided',\n",
       " 'tags=opp_hangup',\n",
       " 'tags=ringing',\n",
       " 'tags=shall_take_in_the_next_coming_month',\n",
       " 'tags=still_thinking',\n",
       " 'tags=switched_off',\n",
       " 'tags=university_not_recognized',\n",
       " 'tags=want_to_take_admission_but_has_financial_problems',\n",
       " 'tags=will_revert_after_reading_the_email',\n",
       " 'tags=wrong_number_given',\n",
       " 'through_recommendations=no',\n",
       " 'through_recommendations=yes',\n",
       " 'total_time_spent_on_website',\n",
       " 'totalvisits',\n",
       " 'update_me_on_supply_chain_content=no',\n",
       " 'what_is_your_current_occupation=businessman',\n",
       " 'what_is_your_current_occupation=housewife',\n",
       " 'what_is_your_current_occupation=other',\n",
       " 'what_is_your_current_occupation=student',\n",
       " 'what_is_your_current_occupation=unemployed',\n",
       " 'what_is_your_current_occupation=working_professional',\n",
       " 'what_matters_most_to_you_in_choosing_a_course=better_career_prospects',\n",
       " 'what_matters_most_to_you_in_choosing_a_course=flexibility_&_convenience',\n",
       " 'x_education_forums=no']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dv.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression(random_state=1, solver=&#x27;liblinear&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(random_state=1, solver=&#x27;liblinear&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression(random_state=1, solver='liblinear')"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LogisticRegression(solver='liblinear', random_state=1)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dict = df_val[categorical + numerical].to_dict(orient='records')\n",
    "X_val = dv.transform(val_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.74944579, 0.25055421],\n",
       "       [0.30230949, 0.69769051],\n",
       "       [0.5210388 , 0.4789612 ],\n",
       "       ...,\n",
       "       [0.6362481 , 0.3637519 ],\n",
       "       [0.7608449 , 0.2391551 ],\n",
       "       [0.31100129, 0.68899871]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_proba(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict_proba(X_val)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.25055421, 0.69769051, 0.4789612 , ..., 0.3637519 , 0.2391551 ,\n",
       "       0.68899871])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "converted = y_pred > 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7267316017316018"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y_val == converted).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.475379844166226e-08"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.intercept_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'a_free_copy_of_mastering_the_interview=no': 0.0,\n",
       " 'a_free_copy_of_mastering_the_interview=yes': -0.0,\n",
       " 'asymmetrique_activity_index=01.high': -0.0,\n",
       " 'asymmetrique_activity_index=02.medium': 0.0,\n",
       " 'asymmetrique_activity_index=03.low': -0.0,\n",
       " 'asymmetrique_activity_score': 0.0,\n",
       " 'asymmetrique_profile_index=01.high': 0.0,\n",
       " 'asymmetrique_profile_index=02.medium': -0.0,\n",
       " 'asymmetrique_profile_index=03.low': 0.0,\n",
       " 'asymmetrique_profile_score': 0.0,\n",
       " 'city=mumbai': -0.0,\n",
       " 'city=other_cities': 0.0,\n",
       " 'city=other_cities_of_maharashtra': 0.0,\n",
       " 'city=other_metro_cities': 0.0,\n",
       " 'city=select': 0.0,\n",
       " 'city=thane_&_outskirts': 0.0,\n",
       " 'city=tier_ii_cities': -0.0,\n",
       " 'country=asia/pacific_region': 0.0,\n",
       " 'country=australia': -0.0,\n",
       " 'country=bahrain': 0.0,\n",
       " 'country=bangladesh': 0.0,\n",
       " 'country=belgium': -0.0,\n",
       " 'country=canada': -0.0,\n",
       " 'country=china': -0.0,\n",
       " 'country=denmark': 0.0,\n",
       " 'country=france': -0.0,\n",
       " 'country=germany': -0.0,\n",
       " 'country=ghana': -0.0,\n",
       " 'country=hong_kong': 0.0,\n",
       " 'country=india': 0.0,\n",
       " 'country=italy': -0.0,\n",
       " 'country=kenya': -0.0,\n",
       " 'country=kuwait': -0.0,\n",
       " 'country=liberia': -0.0,\n",
       " 'country=netherlands': -0.0,\n",
       " 'country=nigeria': -0.0,\n",
       " 'country=oman': 0.0,\n",
       " 'country=philippines': -0.0,\n",
       " 'country=qatar': -0.0,\n",
       " 'country=russia': -0.0,\n",
       " 'country=saudi_arabia': -0.0,\n",
       " 'country=singapore': -0.0,\n",
       " 'country=south_africa': 0.0,\n",
       " 'country=sri_lanka': -0.0,\n",
       " 'country=sweden': -0.0,\n",
       " 'country=switzerland': -0.0,\n",
       " 'country=uganda': -0.0,\n",
       " 'country=united_arab_emirates': -0.0,\n",
       " 'country=united_kingdom': 0.0,\n",
       " 'country=united_states': -0.0,\n",
       " 'country=unknown': -0.0,\n",
       " 'country=vietnam': -0.0,\n",
       " 'digital_advertisement=no': -0.0,\n",
       " 'digital_advertisement=yes': -0.0,\n",
       " 'do_not_call=no': -0.0,\n",
       " 'do_not_call=yes': 0.0,\n",
       " 'do_not_email=no': 0.0,\n",
       " 'do_not_email=yes': -0.0,\n",
       " 'get_updates_on_dm_content=no': -0.0,\n",
       " 'how_did_you_hear_about_x_education=advertisements': 0.0,\n",
       " 'how_did_you_hear_about_x_education=email': -0.0,\n",
       " 'how_did_you_hear_about_x_education=multiple_sources': -0.0,\n",
       " 'how_did_you_hear_about_x_education=online_search': 0.0,\n",
       " 'how_did_you_hear_about_x_education=other': 0.0,\n",
       " 'how_did_you_hear_about_x_education=select': -0.0,\n",
       " 'how_did_you_hear_about_x_education=sms': -0.0,\n",
       " 'how_did_you_hear_about_x_education=social_media': -0.0,\n",
       " 'how_did_you_hear_about_x_education=student_of_someschool': 0.0,\n",
       " 'how_did_you_hear_about_x_education=word_of_mouth': 0.0,\n",
       " 'i_agree_to_pay_the_amount_through_cheque=no': -0.0,\n",
       " 'last_activity=approached_upfront': 0.0,\n",
       " 'last_activity=converted_to_lead': -0.0,\n",
       " 'last_activity=email_bounced': -0.0,\n",
       " 'last_activity=email_link_clicked': -0.0,\n",
       " 'last_activity=email_marked_spam': 0.0,\n",
       " 'last_activity=email_opened': -0.0,\n",
       " 'last_activity=email_received': 0.0,\n",
       " 'last_activity=form_submitted_on_website': -0.0,\n",
       " 'last_activity=had_a_phone_conversation': 0.0,\n",
       " 'last_activity=olark_chat_conversation': -0.0,\n",
       " 'last_activity=page_visited_on_website': -0.0,\n",
       " 'last_activity=resubscribed_to_emails': 0.0,\n",
       " 'last_activity=sms_sent': 0.0,\n",
       " 'last_activity=unreachable': -0.0,\n",
       " 'last_activity=unsubscribed': -0.0,\n",
       " 'last_activity=view_in_browser_link_clicked': -0.0,\n",
       " 'last_notable_activity=approached_upfront': 0.0,\n",
       " 'last_notable_activity=email_bounced': -0.0,\n",
       " 'last_notable_activity=email_link_clicked': -0.0,\n",
       " 'last_notable_activity=email_marked_spam': 0.0,\n",
       " 'last_notable_activity=email_opened': -0.0,\n",
       " 'last_notable_activity=email_received': 0.0,\n",
       " 'last_notable_activity=form_submitted_on_website': -0.0,\n",
       " 'last_notable_activity=had_a_phone_conversation': 0.0,\n",
       " 'last_notable_activity=modified': -0.0,\n",
       " 'last_notable_activity=olark_chat_conversation': -0.0,\n",
       " 'last_notable_activity=page_visited_on_website': -0.0,\n",
       " 'last_notable_activity=resubscribed_to_emails': 0.0,\n",
       " 'last_notable_activity=sms_sent': 0.0,\n",
       " 'last_notable_activity=unreachable': 0.0,\n",
       " 'last_notable_activity=unsubscribed': -0.0,\n",
       " 'last_notable_activity=view_in_browser_link_clicked': -0.0,\n",
       " 'lead_number': -0.0,\n",
       " 'lead_origin=api': -0.0,\n",
       " 'lead_origin=landing_page_submission': -0.0,\n",
       " 'lead_origin=lead_add_form': 0.0,\n",
       " 'lead_origin=lead_import': -0.0,\n",
       " 'lead_origin=quick_add_form': 0.0,\n",
       " 'lead_profile=dual_specialization_student': 0.0,\n",
       " 'lead_profile=lateral_student': 0.0,\n",
       " 'lead_profile=other_leads': -0.0,\n",
       " 'lead_profile=potential_lead': 0.0,\n",
       " 'lead_profile=select': -0.0,\n",
       " 'lead_profile=student_of_someschool': -0.0,\n",
       " 'lead_quality=high_in_relevance': 0.0,\n",
       " 'lead_quality=low_in_relevance': 0.0,\n",
       " 'lead_quality=might_be': -0.0,\n",
       " 'lead_quality=not_sure': -0.0,\n",
       " 'lead_quality=worst': -0.0,\n",
       " 'lead_source=bing': -0.0,\n",
       " 'lead_source=blog': -0.0,\n",
       " 'lead_source=click2call': 0.0,\n",
       " 'lead_source=direct_traffic': -0.0,\n",
       " 'lead_source=facebook': -0.0,\n",
       " 'lead_source=google': -0.0,\n",
       " 'lead_source=live_chat': 0.0,\n",
       " 'lead_source=nc_edm': 0.0,\n",
       " 'lead_source=olark_chat': -0.0,\n",
       " 'lead_source=organic_search': -0.0,\n",
       " 'lead_source=pay_per_click_ads': -0.0,\n",
       " 'lead_source=press_release': -0.0,\n",
       " 'lead_source=reference': 0.0,\n",
       " 'lead_source=referral_sites': -0.0,\n",
       " 'lead_source=social_media': 0.0,\n",
       " 'lead_source=welearn': 0.0,\n",
       " 'lead_source=welearnblog_home': -0.0,\n",
       " 'lead_source=welingak_website': 0.0,\n",
       " 'magazine=no': -0.0,\n",
       " 'newspaper=no': -0.0,\n",
       " 'newspaper=yes': -0.0,\n",
       " 'newspaper_article=no': -0.0,\n",
       " 'page_views_per_visit': -0.0,\n",
       " 'receive_more_updates_about_our_courses=no': -0.0,\n",
       " 'search=no': -0.0,\n",
       " 'search=yes': 0.0,\n",
       " 'specialization=banking,_investment_and_insurance': 0.0,\n",
       " 'specialization=business_administration': 0.0,\n",
       " 'specialization=e-business': 0.0,\n",
       " 'specialization=e-commerce': -0.0,\n",
       " 'specialization=finance_management': 0.0,\n",
       " 'specialization=healthcare_management': 0.0,\n",
       " 'specialization=hospitality_management': -0.0,\n",
       " 'specialization=human_resource_management': 0.0,\n",
       " 'specialization=international_business': -0.0,\n",
       " 'specialization=it_projects_management': -0.0,\n",
       " 'specialization=marketing_management': 0.0,\n",
       " 'specialization=media_and_advertising': 0.0,\n",
       " 'specialization=operations_management': 0.0,\n",
       " 'specialization=retail_management': -0.0,\n",
       " 'specialization=rural_and_agribusiness': 0.0,\n",
       " 'specialization=select': -0.0,\n",
       " 'specialization=services_excellence': -0.0,\n",
       " 'specialization=supply_chain_management': 0.0,\n",
       " 'specialization=travel_and_tourism': -0.0,\n",
       " 'tags=already_a_student': -0.0,\n",
       " 'tags=busy': 0.0,\n",
       " 'tags=closed_by_horizzon': 0.0,\n",
       " 'tags=diploma_holder_(not_eligible)': -0.0,\n",
       " 'tags=graduation_in_progress': -0.0,\n",
       " 'tags=in_confusion_whether_part_time_or_dlp': -0.0,\n",
       " 'tags=in_touch_with_eins': -0.0,\n",
       " 'tags=interested__in_full_time_mba': -0.0,\n",
       " 'tags=interested_in_next_batch': 0.0,\n",
       " 'tags=interested_in_other_courses': -0.0,\n",
       " 'tags=invalid_number': -0.0,\n",
       " 'tags=lost_to_eins': 0.0,\n",
       " 'tags=lost_to_others': -0.0,\n",
       " 'tags=not_doing_further_education': -0.0,\n",
       " 'tags=number_not_provided': -0.0,\n",
       " 'tags=opp_hangup': -0.0,\n",
       " 'tags=ringing': -0.0,\n",
       " 'tags=shall_take_in_the_next_coming_month': -0.0,\n",
       " 'tags=still_thinking': -0.0,\n",
       " 'tags=switched_off': -0.0,\n",
       " 'tags=university_not_recognized': -0.0,\n",
       " 'tags=want_to_take_admission_but_has_financial_problems': -0.0,\n",
       " 'tags=will_revert_after_reading_the_email': 0.0,\n",
       " 'tags=wrong_number_given': -0.0,\n",
       " 'through_recommendations=no': -0.0,\n",
       " 'through_recommendations=yes': 0.0,\n",
       " 'total_time_spent_on_website': 0.001,\n",
       " 'totalvisits': -0.0,\n",
       " 'update_me_on_supply_chain_content=no': -0.0,\n",
       " 'what_is_your_current_occupation=businessman': 0.0,\n",
       " 'what_is_your_current_occupation=housewife': 0.0,\n",
       " 'what_is_your_current_occupation=other': 0.0,\n",
       " 'what_is_your_current_occupation=student': -0.0,\n",
       " 'what_is_your_current_occupation=unemployed': -0.0,\n",
       " 'what_is_your_current_occupation=working_professional': 0.0,\n",
       " 'what_matters_most_to_you_in_choosing_a_course=better_career_prospects': -0.0,\n",
       " 'what_matters_most_to_you_in_choosing_a_course=flexibility_&_convenience': 0.0,\n",
       " 'x_education_forums=no': -0.0}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(zip(dv.get_feature_names(), model.coef_[0].round(3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lead_number',\n",
       " 'tags=already_a_student',\n",
       " 'tags=busy',\n",
       " 'tags=closed_by_horizzon',\n",
       " 'tags=diploma_holder_(not_eligible)',\n",
       " 'tags=graduation_in_progress',\n",
       " 'tags=in_confusion_whether_part_time_or_dlp',\n",
       " 'tags=in_touch_with_eins',\n",
       " 'tags=interested__in_full_time_mba',\n",
       " 'tags=interested_in_next_batch',\n",
       " 'tags=interested_in_other_courses',\n",
       " 'tags=invalid_number',\n",
       " 'tags=lost_to_eins',\n",
       " 'tags=lost_to_others',\n",
       " 'tags=not_doing_further_education',\n",
       " 'tags=number_not_provided',\n",
       " 'tags=opp_hangup',\n",
       " 'tags=ringing',\n",
       " 'tags=shall_take_in_the_next_coming_month',\n",
       " 'tags=still_thinking',\n",
       " 'tags=switched_off',\n",
       " 'tags=university_not_recognized',\n",
       " 'tags=want_to_take_admission_but_has_financial_problems',\n",
       " 'tags=will_revert_after_reading_the_email',\n",
       " 'tags=wrong_number_given',\n",
       " 'totalvisits']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset = ['tags', 'lead_number', 'totalvisits']\n",
    "train_dict_small = df_train[subset].to_dict(orient='records')\n",
    "dv_small = DictVectorizer(sparse=False)\n",
    "dv_small.fit(train_dict_small)\n",
    "\n",
    "X_small_train = dv_small.transform(train_dict_small)\n",
    "\n",
    "dv_small.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression(random_state=1, solver=&#x27;liblinear&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(random_state=1, solver=&#x27;liblinear&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression(random_state=1, solver='liblinear')"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_small = LogisticRegression(solver='liblinear', random_state=1)\n",
    "model_small.fit(X_small_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.2483009809963478e-12"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_small.intercept_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\USER\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'lead_number': -0.0,\n",
       " 'tags=already_a_student': -0.0,\n",
       " 'tags=busy': 0.0,\n",
       " 'tags=closed_by_horizzon': 0.0,\n",
       " 'tags=diploma_holder_(not_eligible)': -0.0,\n",
       " 'tags=graduation_in_progress': -0.0,\n",
       " 'tags=in_confusion_whether_part_time_or_dlp': -0.0,\n",
       " 'tags=in_touch_with_eins': -0.0,\n",
       " 'tags=interested__in_full_time_mba': -0.0,\n",
       " 'tags=interested_in_next_batch': 0.0,\n",
       " 'tags=interested_in_other_courses': -0.0,\n",
       " 'tags=invalid_number': -0.0,\n",
       " 'tags=lost_to_eins': 0.0,\n",
       " 'tags=lost_to_others': -0.0,\n",
       " 'tags=not_doing_further_education': -0.0,\n",
       " 'tags=number_not_provided': -0.0,\n",
       " 'tags=opp_hangup': -0.0,\n",
       " 'tags=ringing': -0.0,\n",
       " 'tags=shall_take_in_the_next_coming_month': -0.0,\n",
       " 'tags=still_thinking': -0.0,\n",
       " 'tags=switched_off': -0.0,\n",
       " 'tags=university_not_recognized': -0.0,\n",
       " 'tags=want_to_take_admission_but_has_financial_problems': -0.0,\n",
       " 'tags=will_revert_after_reading_the_email': 0.0,\n",
       " 'tags=wrong_number_given': -0.0,\n",
       " 'totalvisits': -0.0}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(zip(dv_small.get_feature_names(), model_small.coef_[0].round(3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dict_small = df_val[subset].to_dict(orient='records')\n",
    "X_small_val = dv_small.transform(val_dict_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_small = model_small.predict_proba(X_small_val)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, ..., 0, 1, 1], dtype=int64)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "prospect = {\n",
    "'prospect_id': '2f302f24-3151-4763-afdc-a80d9750b3d8',\n",
    " 'lead_origin': 'lead_add_form',\n",
    " 'lead_source': 'reference',\n",
    " 'do_not_email': 'no',\n",
    " 'do_not_call': 'no',\n",
    " 'last_activity': 'sms_sent',\n",
    " 'country': 'india',\n",
    " 'specialization': 'select',\n",
    " 'how_did_you_hear_about_x_education': 'select',\n",
    " 'what_is_your_current_occupation': 'unemployed',\n",
    " 'what_matters_most_to_you_in_choosing_a_course': 'better_career_prospects',\n",
    " 'search': 'no',\n",
    " 'magazine': 'no',\n",
    " 'newspaper_article': 'no',\n",
    " 'digital_advertisement': 'no',\n",
    " 'through_recommendations': 'no',\n",
    " 'receive_more_updates_about_our_courses': 'no',\n",
    " 'tags': 'will_revert_after_reading_the_email',\n",
    " 'lead_quality': 'might_be',\n",
    " 'update_me_on_supply_chain_content': 'no',\n",
    " 'lead_profile': 'select',\n",
    " 'city': 'select',\n",
    " 'asymmetrique_activity_index': '02.medium',\n",
    " 'asymmetrique_profile_index': '02.medium',\n",
    " 'i_agree_to_pay_the_amount_through_cheque': 'no',\n",
    " 'a_free_copy_of_mastering_the_interview': 'no',\n",
    " 'last_notable_activity': 'sms_sent',\n",
    " 'newspaper': 'no',\n",
    " 'x_education_forums': 'no',\n",
    " 'get_updates_on_dm_content': 'no',\n",
    " 'lead_number': 627173,\n",
    " 'totalvisits': 0.0,\n",
    " 'total_time_spent_on_website': 0,\n",
    " 'page_views_per_visit': 0.0,\n",
    " 'asymmetrique_activity_score': 14.306252489048187,\n",
    " 'asymmetrique_profile_score': 16.344882516925527\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22892012087019842"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = dv.transform([prospect])\n",
    "model.predict_proba(X_test)[0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.0, 0.0, 0.0, 1.0, 0.0, 14.306252489048187, 0.0, 1.0, 0.0, 16.344882516925527, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 627173.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0]\n"
     ]
    }
   ],
   "source": [
    "print(list(X_test[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "prospect = {\n",
    "    'prospect_id': '2f302f24-3151-4763-afdc-a80d9750b3d8',\n",
    " 'lead_origin': 'lead_add_form',\n",
    " 'lead_source': 'reference',\n",
    " 'do_not_email': 'no',\n",
    " 'do_not_call': 'no',\n",
    " 'last_activity': 'sms_sent',\n",
    " 'country': 'india',\n",
    " 'specialization': 'select',\n",
    " 'how_did_you_hear_about_x_education': 'select',\n",
    " 'what_is_your_current_occupation': 'unemployed',\n",
    " 'what_matters_most_to_you_in_choosing_a_course': 'better_career_prospects',\n",
    " 'search': 'no',\n",
    " 'magazine': 'no',\n",
    " 'newspaper_article': 'no',\n",
    " 'digital_advertisement': 'no',\n",
    " 'through_recommendations': 'no',\n",
    " 'receive_more_updates_about_our_courses': 'no',\n",
    " 'tags': 'will_revert_after_reading_the_email',\n",
    " 'lead_quality': 'might_be',\n",
    " 'update_me_on_supply_chain_content': 'no',\n",
    " 'lead_profile': 'select',\n",
    " 'city': 'select',\n",
    " 'asymmetrique_activity_index': '02.medium',\n",
    " 'asymmetrique_profile_index': '02.medium',\n",
    " 'i_agree_to_pay_the_amount_through_cheque': 'no',\n",
    " 'a_free_copy_of_mastering_the_interview': 'no',\n",
    " 'last_notable_activity': 'sms_sent',\n",
    " 'newspaper': 'no',\n",
    " 'x_education_forums': 'no',\n",
    " 'get_updates_on_dm_content': 'no',\n",
    " 'lead_number': 627173,\n",
    " 'totalvisits': 0.0,\n",
    " 'total_time_spent_on_website': 0,\n",
    " 'page_views_per_visit': 0.0,\n",
    " 'asymmetrique_activity_score': 14.306252489048187,\n",
    " 'asymmetrique_profile_score': 16.344882516925527\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22892012087019842"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = dv.transform([prospect])\n",
    "model.predict_proba(X_test)[0, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7267316017316018"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_proba(X_val)[:, 1]\n",
    "converted = y_pred >= 0.5\n",
    "(converted == y_val).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7267316017316018"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_val, y_pred >= 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thresholds = np.linspace(0, 1, 11)\n",
    "thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00 0.382\n",
      "0.05 0.382\n",
      "0.10 0.382\n",
      "0.15 0.382\n",
      "0.20 0.382\n",
      "0.25 0.491\n",
      "0.30 0.628\n",
      "0.35 0.705\n",
      "0.40 0.730\n",
      "0.45 0.736\n",
      "0.50 0.727\n",
      "0.55 0.710\n",
      "0.60 0.685\n",
      "0.65 0.662\n",
      "0.70 0.647\n",
      "0.75 0.630\n",
      "0.80 0.619\n",
      "0.85 0.617\n",
      "0.90 0.618\n",
      "0.95 0.618\n",
      "1.00 0.618\n"
     ]
    }
   ],
   "source": [
    "thresholds = np.linspace(0, 1, 21)\n",
    "\n",
    "accuracies = []\n",
    "\n",
    "for t in thresholds:\n",
    "    acc = accuracy_score(y_val, y_pred >= t)\n",
    "    accuracies.append(acc)\n",
    "    print('%0.2f %0.3f' % (t, acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuC0lEQVR4nO3deZwU5bn3/8+XTRHZFHBBo/4SDQqKR0cEQRGjiCIRdzDq0SQSYjQxyXHLk814nkSj58TEfYmPGhkRNQghiBpxRhRRZgwCQ8QQNGHRgAsCA4LA9fuja7AZe6AHprunZ77v12tedFXdVfdVA/TVfV9VdykiMDMzq61FoQMwM7PGyQnCzMwycoIwM7OMnCDMzCwjJwgzM8vICcLMzDJygrCCkPRzSY/koZ+LJb20nftuNUZJ70g6cfujM2vcnCAsJyStTvvZJGlt2vLXCh1fMZF0QPI7vLPQsVjz4gRhORERu9b8AP8ChqWtG1OfY0lqlZsoi8ZFwEfACEk75bNjSS3z2Z81Lk4QVkhtJD0saZWkKkklNRuS4ZtrJM0GqiW1ktRX0nRJKyS9Ien4tPYXS1qYHOvt2t9SJN0i6aNk2ylp6/eWNFHSh5IWSLq0rmAlXSjpn5I+kPR/ttKur6T30t9cJZ2RnAuS+kiqkLRS0r8l/e82fk8XAT8GPgWG1errdEmzkmP9Q9KQZP1ukv6fpKXJeT+V9nt6qdYxQtKXktcPSrpL0mRJ1cAgSUMl/TXpY5Gkn9faf0Da38uipI+jknNrldbuLEmztnGu1og4QVghfRUYC3QCJgK319o+EhiabN8D+DPw38BuwH8BT0rqKqkd8DvglIhoDxwDzEo7ztHAfKAL8Gvg95KUbHsUWAzsDZwN/FLSV2oHKukQ4C7gwqTt7sA+mU4qImYA1cAJaavPB0qT178FfhsRHYAvAuMyHSfp99ikn7FJu4vStvUBHgauSn5HxwHvJJv/AOwC9AS6Ab+pq48Mzgf+L9AeeCk5l4uSPoYC35Y0PInhC8DTwG1AV+BwYFZEzAQ+AE5KO+4FSVxWJJwgrJBeiojJEbGR1BtH71rbfxcRiyJiLak3l8lJ+00R8RxQAZyatN0E9JLUNiLejYiqtOP8MyLuS/p5CNgL2EPSvsAA4JqI+CQiZgH3k0oCtZ0NTIqIFyNiHfCTpM+6PEoqwSGpfRLno8m2T4EvSeoSEauThFKX/wSejoiPSCWYUyR1S7Z9A3ggIp5LfidLIuJNSXsBpwCjI+KjiPg0Isq30kdtEyLi5eSYn0REWUTMSZZnJ+cxMGn7NeAvEfFo0s8Hye8RUr/rC5LfwW7AyXyWJK0IOEFYIb2X9noNsHOtesOitNf7AeckwxgrJK0g9ea+V0RUA+cBo4F3Jf1ZUo9M/UTEmuTlrqS+CXwYEavS2v4T6J4h1r3T40n6/GAr51YKnJnUDM4EXo+IfybbvgEcBLwpaaak0zIdQFJb4BxgTNLnK6TqOecnTfYF/pFh132T8/poK/FtTfrvHUlHS3pB0nJJH5P6PXfZRgwAjwDDJO0KnAtMi4h3tzMmKwAnCGvM0qcaXgT8ISI6pf20i4gbASLimYg4idS3gzeB+7I4/lJgt+QTfo0vAEsytH2X1JshAJJ2ITXMlDnwiHmkks0pbDm8RET8PSJGkhr6uQl4Ihkmq+0MoANwZ1LTeI9U8qoZZlpEaoiqtkXJeXXKsK2a1NBTzXnsmSn8WsulpIYA942IjsDdQM0QXV0xEBFLgFeS87gQDy8VHScIKxY1n0ZPltRS0s6Sjpe0j6Q9JH01eZNdB6wGNm7rgBGxCJgO/Co53mGkPt1nusrqCeC0pCDbBvgF2/7/Uwp8l1Rt4PGalZIukNQ1IjYBK5LVmeL9T+AB4FBSY/uHA/2BwyUdCvweuETSVyS1kNRdUo/kU/rTpBJLZ0mtJR2XHPMNoKekwyXtDPx8G+cAqVrEhxHxSVL3OD9t2xjgREnnKnUhwe6SDk/b/jBwdXIO47PoyxoRJwgrCsmb+enAj4DlpD65XkXq33AL4IekvhF8SGp8/LIsDz0S2D/Zdzzws6S+Ubv/KuA7pN703yV12enibRz7UeB4YGpEvJ+2fghQJWk1qYL1iIj4JH1HSd2BrwC3RsR7aT+VwBTgPyPiNeASUgXoj4FyUkNxkPrE/impb1PLgCuT83iLVHL7C/B3UkXobbkM+IWkVcBPSSuqR8S/SNVXfkjqdz+LLWtJ45OYxifDclZE5AcGmVkuSfoH8K2I+EuhY7H68TcIM8sZSWeRqmlMLXQsVn/N/Q5VM8sRSWXAIcCFSb3FioyHmMzMLKOcDjFJGiJpvlJTGFybYftVyTQBsyTNlbQxuaGmZqqFOcm2ilzGaWZmn5ezbxBKzUPzFqlb7RcDM4GRyfXhmdoPA74fEScky+8AJbWu/tiqLl26xP7777+DkZuZNR+VlZXvR0TXTNtyWYPoAyyIiIUAksaSukwxY4Igdbnho3Vsy8r+++9PRYW/bJiZZUvSP+valsshpu5secv+YjJPYVBzV+oQ4Mm01QE8K6lS0qi6OpE0SqmZMSuWL1/eAGGbmRnkNkEow7q6xrOGAS9HxIdp6/pHxBGkpir4TtqdoFseMOLeiCiJiJKuXTN+SzIzs+2QywSxmLS5a0hNWby0jrYjqDW8FBFLkz+Xkbobs08OYjQzszrkMkHMBA5U6nGJbUglgYm1G0nqSGpqhAlp69rVTKCWzK8zGJibw1jNzKyWnBWpI2KDpMuBZ4CWpOatr5I0Otl+d9L0DODZWvO07AGMT57p0goojYgpuYrVzMw+r0ndKFdSUhK+isnMLHuSKiOiJNM2z8VkZmYZeS4ms61YvXo1FRUVVFZWctBBBzF48GB22mmnQodllhdOEGaJTZs2MX/+fGbMmMGMGTN49dVXmTNnDps2fTbPXMeOHRk+fDjnnnsuJ554Im3atClgxGa55QRhzdaHH37Iq6++ukVC+PjjjwHo1KkTRx99NMOHD6dv3778x3/8B6+//jrjxo1j/PjxPPTQQ3Tu3JkzzjiD8847j0GDBtG6desCn5FZw3KR2pqNuXPnMm3atM0J4a233gKgRYsWHHroofTt23fzz0EHHUSLFplLdOvWreO5557jscceY8KECaxatYrdd9+ds846i3PPPZeBAwfSqpU/e1lx2FqR2gnCmryI4Prrr+f6668HYI899tgiGZSUlLDrrrtu17E/+eQTpkyZwrhx45g4cSLV1dV069aNs846i/POO48BAwbQsmXLhjwdswblBGHN1saNG7niiiu46667uPjii/nZz37GfvvtR3KPTYNas2YNTz/9NI899hiTJk1i7dq17LXXXpx99tl885vf5LDDDmvwPs12lBOENUvr1q3jggsu4IknnuCaa67hV7/6VU4SQybV1dVMmjSJcePGMXnyZNavX88VV1zBL37xCzp06JCXGMyy4fsgrNlZtWoVp556Kk888QT/8z//w4033pi35ADQrl07zjvvPJ588kmWLl3Kt771LX73u99x8MEH88QTT9CUPphZ0+UEYU3OsmXLGDRoEOXl5Tz88MP84Ac/KGg8nTt35s4772TGjBl069aNc845h1NPPZV//OMfBY3LbFucIKxJeeeddxgwYADz5s1jwoQJXHjhhYUOabM+ffowc+ZMbr31Vl566SV69erFf//3f7Nu3bpCh2aWkROENRlz5szhmGOOYfny5fzlL39h6NChhQ7pc1q1asX3vvc93nzzTYYNG8ZPfvITevfuzQsvvFDo0Mw+xwnCmoSXX36Z4447DklMmzaNY445ptAhbVX37t23KGCfcMIJXHjhhfz73/8udGhmmzlBWNGbNGkSJ554It26dWP69On06tWr0CFl7ZRTTqGqqoof//jHPPbYY/To0YN77rlni+k9zArFCcKK2kMPPcTw4cPp1asXL730Evvtt1+hQ6q3tm3bcsMNNzB79mwOP/xwRo8ezTHHHMOsWbMKHZo1c04QVrRuueUWLr74YgYNGsTUqVMp9meS9+jRg6lTp/KHP/yBhQsXcuSRR/KDH/yAVatWFTo0a6acIKzoRARXX301V111Feeeey6TJk2iffv2hQ6rQUjiggsuYP78+YwaNYpbb72Vww47jL///e+FDs2aIScIKyobNmzg61//OjfffDPf+c53KC0tbZLPZ+jcuTN33XUX06ZNY/Xq1QwYMIA33nij0GFZM+MEYUVj7dq1nHnmmTz44INcf/313HbbbU1+Irz+/fszbdo0dtppJwYOHMjLL79c6JCsGclpgpA0RNJ8SQskXZth+1WSZiU/cyVtlLRbNvta8/O///u//OlPf+LOO+/kpz/9aV6nziikHj168NJLL7HHHntw0kknMWXKlEKHZM1Ezibrk9QSeAs4CVgMzARGRsS8OtoPA74fESfUd98anqyv6YoIDj74YPbcc0/KysoKHU5BLFu2jJNPPpmqqirGjBnDOeecU+iQrAko1GR9fYAFEbEwItYDY4HTt9J+JPDodu5rTdysWbOYP38+559/fqFDKZhu3brxwgsv0KdPH0aMGMH9999f6JCsictlgugOLEpbXpys+xxJuwBDgCe3Y99RkiokVSxfvnyHg7bGqbS0lNatW3PWWWcVOpSC6tSpE88++yyDBw/m0ksv5eabby50SNaE5TJBZBogrms8axjwckR8WN99I+LeiCiJiJJivw7eMtu0aROPPvooQ4YMYffddy90OAW3yy67MGHCBM4991yuvvpqfvSjH3n6cMuJXD44dzGwb9ryPsDSOtqO4LPhpfrua03ctGnTWLJkCbfcckuhQ2k02rRpQ2lpKR07duRXv/oVK1as4Pbbb6/zOdpm2yOXCWImcKCkA4AlpJLA5waQJXUEBgIX1Hdfax5KS0tp164dw4YNK3QojUrLli2555576Ny5M7/+9a9ZsWIFDz30EK1bty50aNZE5CxBRMQGSZcDzwAtgQciokrS6GT73UnTM4BnI6J6W/vmKlZrvNavX8/jjz/O8OHDadeuXaHDaXQkcdNNN9G5c2euu+46Vq5cyeOPP07btm0LHZo1Abn8BkFETAYm11p3d63lB4EHs9nXmp8pU6bw0Ucf8bWvfa3QoTRq1157LZ06deKyyy5jyJAhTJw4kY4dOxY6LCtyHrC0Rq20tJQuXbpw4oknFjqURm/06NGMGTOG6dOnc8IJJ+Cr+mxHOUFYo7Vq1SomTpzIueee63H1LI0cOZKnnnqKefPmcdxxx7Fo0aJt72RWBycIa7QmTJjA2rVrm/XNcdtj6NChPPvssyxdupQBAwYwZ86cQodkRcoJwhqt0tJS9ttvP/r161foUIrOscceywsvvMCnn35Kv379GD9+fKFDsiLkBGGN0vLly3n22WcZOXKkr+3fTkcccQQVFRX07NmTM888k+uvv96PMrV68f88a5Qef/xxNm7c6OGlHbT33ntTXl7ORRddxM9//nPOPvtsVq9eXeiwrEg4QVijNGbMGHr16sWhhx5a6FCK3s4778yDDz7Ib37zGyZMmEC/fv1YuHBhocOyIuAEYY3O22+/zfTp033vQwOSxJVXXsmUKVNYsmQJRx11FFOnTi10WNbIOUFYozN27FgARowYUeBImp6TTjqJ1157jT333JPBgwdz2223eaI/q5MThDU6paWl9O/fn/3337/QoTRJX/rSl3jllVcYOnQo3/3ud7n00ktZt25docOyRsgJwhqVOXPmMHfuXBenc6xDhw6MHz+eH//4x/z+97/nhBNO4L333it0WNbIOEFYo1JaWkrLli39OM08aNGiBTfccAPjxo1j1qxZlJSU4Ef2WjonCGs0ah4MNHjwYPzwp/w555xzePnll2nVqhXHHnssY8aMKXRI1kg4QVijMX36dP75z396eKkADj/8cGbOnMnRRx/NBRdcwNVXX83GjRsLHZYVmBOENRqlpaW0bduW008/vdChNEtdu3blueee47LLLuPmm2/mtNNO84ywzZwThDUKn376KePGjeP000+nffv2hQ6n2WrdujV33HEH99xzD88//zxf/OIXufHGG1m7dm2hQ7MCcIKwRuG5557jgw8+8PBSIzFq1CjmzJnDoEGDuO666+jRowdjxozxXE7NjBOENQqlpaV07tyZk08+udChWOLLX/4yEyZM4IUXXqBLly5ccMEF9OnTh/Ly8kKHZnniBGEFV11dzVNPPcU555xDmzZtCh2O1XL88cczc+ZM/vCHP7Bs2TKOP/54hg8fzvz58wsdmuVYThOEpCGS5ktaIOnaOtocL2mWpCpJ5Wnr35E0J9nmi7ObsD/96U9UV1d7eKkRa9GiBRdccAHz58/nl7/8JVOnTqVnz55cfvnlLmQ3YTlLEJJaAncApwCHACMlHVKrTSfgTuCrEdETqH131KCIODwiSnIVpxXemDFj6N69O8cee2yhQ7FtaNu2Lddddx0LFizgW9/6FnfffTdf+tKXuOmmm/jkk08KHZ41sFx+g+gDLIiIhRGxHhgL1L5+8XzgjxHxL4CIWJbDeKwR+uCDD5gyZYofDFRkunXrxh133MGcOXMYOHAg1157LV/+8pcpLS11IbsJyeX/yO5A+hPTFyfr0h0EdJZUJqlS0kVp2wJ4Nlk/qq5OJI2SVCGpwl91i88TTzzBhg0bPLxUpA4++GAmTpzI1KlT6dKlC1/72tfo27cvL774YqFDswaQywShDOtqzyvcCjgSGAqcDPxE0kHJtv4RcQSpIarvSDouUycRcW9ElEREiadnKD6lpaUcfPDBHH744YUOxXbAoEGDmDlzJg8//DDvvvsuAwcO5LzzzvMEgEUulwliMbBv2vI+wNIMbaZERHVEvA+8CPQGiIilyZ/LgPGkhqysCVm0aBEvvvgi559/PlKmzxNWTFq0aMGFF17I/Pnz+cUvfsGECRM4+OCDue+++zzsVKRymSBmAgdKOkBSG2AEMLFWmwnAsZJaSdoFOBr4m6R2ktoDSGoHDAbm5jBWK4CaBwONHDmywJFYQ9pll134yU9+wuzZs+nduzejRo1i0KBBviy2COUsQUTEBuBy4Bngb8C4iKiSNFrS6KTN34ApwGzgNeD+iJgL7AG8JOmNZP2fI2JKrmK1wigtLeXoo4/mi1/8YqFDsRw46KCDeOGFF7j//vuZPXs2hx12GDfccAPr168vdGiWJTWlxw2WlJSE57MvDvPmzaNnz5789re/5bvf/W6hw7Ece++997jyyit57LHH6NmzJ/fddx/9+vUrdFgGSKqs61YCX1doBVFaWkqLFi0499xzCx2K5cGee+7J2LFjmTRpEitXrqR///5cfvnlrFy5stCh2VY4QVjeRQSlpaV85StfYc899yx0OJZHQ4cOpaqqiiuuuII777yTQw45hAkTJhQ6LKuDE4Tl3auvvsrbb7/tex+aqfbt2/Pb3/6WGTNmsNtuuzF8+HDOPvtsli6tfZGjFZoThOVdaWkpO+20E2eeeWahQ7EC6tOnD5WVlfzyl79k0qRJHHLIIdxzzz2+JLYRcYKwvNqwYQOPPfYYw4YNo0OHDoUOxwqsdevWXHfddcyZM4cjjzyS0aNHM3DgQObMmVPo0AwnCMuzqVOnsmzZMg8v2RYOPPBA/vKXv/DAAw9QVVXFYYcdxsCBA3nkkUf8NLsCcoKwvBo3bhwdOnTglFNOKXQo1shI4pJLLuGtt97ixhtvZMmSJVx44YV0796dK6+8kqqqqkKH2Ow4QVhezZgxgwEDBrDzzjsXOhRrpLp06cI111zDW2+9xfPPP8/gwYO588476dWrF/379+ehhx5izZo1hQ6zWXCCsLyprq7mb3/7G0ceeWShQ7Ei0KJFC0444QTGjh3LkiVLuOWWW3j//fe5+OKL2Xvvvbn88suZPXt2ocNs0pwgLG/eeOMNNm3a5ARh9da1a1d++MMf8uabb1JWVsZpp53G/fffT+/evenbty8PPPAA1dXVhQ6zyXGCsLyprKwEcIKw7SZpc/F6yZIl/OY3v2HVqlV84xvfYK+99uLb3/425eXlzJ8/n/fee89PudtBnovJ8ubiiy9mypQpvPvuu57e2xpMRDB9+nTuvfdexo0b97mk0KZNGzp27EinTp3o2LHj5p9My+3bt2fjxo18+umnm3/Wr1+/xXLtn5rtGzZsoFDvpx06dODWW2/drn23NhdTqx0Jyqw+KisrOfLII50crEFJon///vTv359bb72VV155hRUrVvDxxx9v/rP26/fee2/z8urVq+vdZ4sWLWjdujVt2rShdevWtG7dmlatWhXssbm5eliaE4TlRXV1NfPmzeOMM84odCjWhHXu3JlTTz21Xvts2LCBlStX8vHHH7Nq1Spatmy5+U0/PQGk/zSX56c7QVheuEBtjVWrVq3Ybbfd2G233QodSqPTPNKgFZwL1GbFxwnC8qKyspJu3brRvXv3QodiZllygrC8qKyspKSkxAVqsyLiBGE5t2bNGubNm+fhJbMik9MEIWmIpPmSFki6to42x0uaJalKUnl99rXi4AK1WXHK2VVMkloCdwAnAYuBmZImRsS8tDadgDuBIRHxL0ndst3XikfNzYtOEGbFJZffIPoACyJiYUSsB8YCp9dqcz7wx4j4F0BELKvHvlYkXKA2K065TBDdgUVpy4uTdekOAjpLKpNUKemieuwLgKRRkiokVSxfvryBQreG5DuozYpTLhNEpneD2hOVtAKOBIYCJwM/kXRQlvumVkbcGxElEVGSq9vNbfu5QG1WvHJ5J/ViYN+05X2ApRnavB8R1UC1pBeB3lnua0WgpkBdUpJxLjAza8S2+Q1C0mmStuebxkzgQEkHSGoDjAAm1mozAThWUitJuwBHA3/Lcl8rAr6D2qx4ZfPGPwL4u6RfSzo42wNHxAbgcuAZUm/64yKiStJoSaOTNn8DpgCzgdeA+yNibl371ufErHGoqKhwgdqsSGX1PAhJHYCRwCWkagH/D3g0IlblNrz68fMgGp9DDz2Ufffdl8mTJxc6FDPLYGvPg8hq6CgiVgJPkrrcdC/gDOB1SVc0WJTW5LhAbVbcsqlBDJM0HpgKtAb6RMQppIrJ/5Xj+KyI+Q5qs+KWzVVM5wC/iYgX01dGxBpJX89NWNYUuEBtVtyySRA/A96tWZDUFtgjIt6JiOdzFpkVvZo7qPfZZ59Ch2Jm2yGbGsTjwKa05Y3JOrOt8h3UZsUtmwTRKpkPCYDkdZvchWRNwZo1a6iqqvLwklkRyyZBLJf01ZoFSacD7+cuJGsKXKA2K37Z1CBGA2Mk3U5qjqRFwEVb38WaOxeozYrfNhNERPwD6CtpV1I31jWqm+OscaqsrKRr164uUJsVsawm65M0FOgJ7FxTcIyIX+QwLityLlCbFb9sbpS7GzgPuILUENM5wH45jsuK2Nq1a5k3b55ncDUrctkUqY+JiIuAjyLieqAfW07FbbaFN954g40bN7r+YFbkskkQnyR/rpG0N/ApcEDuQrJi52dQmzUN2dQg/iSpE3Az8Dqp2Vzvy2VQVtxcoDZrGraaIJIHBT0fESuAJyVNAnaOiI/zEZwVJxeozZqGrQ4xRcQm4H/Sltc5OdjW1BSoPbxkVvyyqUE8K+ks+eOgZcEFarOmI5saxA+AdsAGSZ+QutQ1IqJDTiOzolRzB7UvcTUrftncSd0+H4FY01BRUeECtVkTsc0EIem4TOtrP0Cojn2HAL8FWgL3R8SNtbYfD0wA3k5W/bHmDm1J7wCrSE0vvqGuZ6Za4+ICtVnTkc0Q01Vpr3cG+gCVwAlb20lSS+AO4CRgMTBT0sSImFer6bSIOK2OwwyKCM8cWyRqCtRf/epXt93YzBq9bIaYhqUvS9oX+HUWx+4DLIiIhcl+Y4HTgdoJwpoIF6jNmpZsrmKqbTHQK4t23UlNDZ6+X/cM7fpJekPS05J6pq0PUldQVUoaVVcnkkZJqpBUsXz58mzitxzxFN9mTUs2NYjbSL1ZQyqhHA68kcWxMw1CR63l14H9ImK1pFOBp4ADk239I2KppG7Ac5LezFT3iIh7gXsBSkpKah/f8qjmDup99/VUXWZNQTY1iIq01xuARyPi5Sz2W8yWk/rtAyxNbxARK9NeT5Z0p6QuEfF+RCxN1i+TNJ7UkNU2C+NWOC5QmzUt2SSIJ4BPImIjpIrPknaJiDXb2G8mcKCkA4AlwAjg/PQGkvYE/h0RIakPqW8oH0hqB7SIiFXJ68GAnz/RiK1du5aqqiqGDRu27cZmVhSySRDPAycCq5PltsCzwDFb2ykiNki6HHiG1GWuD0RElaTRyfa7gbOBb0vaAKwFRiTJYg9gfPJJtBVQGhFT6n12ljcuUJs1PdkkiJ0joiY5kNQLdsnm4BExGZhca93daa9vB27PsN9CoHc2fVjj4AK1WdOTzVVM1ZKOqFmQdCSpT/tmm1VWVtKlSxcXqM2akGy+QVwJPC6ppsC8F6lHkJpt5gK1WdOTzY1yMyX1AL5M6tLVNyPi05xHZkXDBWqzpmmbQ0ySvgO0i4i5ETEH2FXSZbkPzYrF7NmzXaA2a4KyqUFcmjxRDoCI+Ai4NGcRWdHxM6jNmqZsEkSL9IcFJZPwtcldSFZsXKA2a5qyKVI/A4yTdDepqTJGA0/nNCorKi5QmzVN2XyDuIbUzXLfBr4DzCZ1s5zZ5gK1h5fMmp5tJoiI2ATMABYCJcBXgL/lOC4rEi5QmzVddQ4xSTqI1PxJI4EPgMcAImJQfkKzYuBnUJs1XVurQbwJTAOGRcQCAEnfz0tUVjQqKipcoDZrorY2xHQW8B7wgqT7JH2FzM94sGbMBWqzpqvOBBER4yPiPKAHUAZ8H9hD0l2SBucpPmvEXKA2a9qyKVJXR8SYiDiN1EN/ZgHX5jowa/xcoDZr2ur1TOqI+DAi7omIE3IVkBUPT/Ft1rTVK0GYpausrGT33XfnC1/4QqFDMbMccIKw7VZZWUlJSYkL1GZNlBOEbZe1a9cyd+5cDy+ZNWFOELZdXKA2a/pymiAkDZE0X9ICSZ+78knS8ZI+ljQr+flptvtaYblAbdb0ZTOb63ZJpgW/AzgJWAzMlDQxIubVajotuYR2e/a1AnGB2qzpy+U3iD7AgohYGBHrgbHA6XnY1/LAd1CbNX25TBDdgUVpy4uTdbX1k/SGpKcl9aznvkgaJalCUsXy5csbIm7bhk8++YSqqipP0GfWxOUyQWT6aBm1ll8H9ouI3sBtwFP12De1MuLeiCiJiJKuXbtub6xWD7Nnz2bDhg2uP5g1cblMEIuB9Ck+9wGWpjeIiJURsTp5PRloLalLNvta4fgZ1GbNQy4TxEzgQEkHSGpD6tkSE9MbSNqz5nnXkvok8XyQzb5WOC5QmzUPObuKKSI2SLqc1DOtWwIPRESVpNHJ9ruBs4FvS9oArAVGREQAGffNVaxWPy5QmzUPOUsQsHnYaHKtdXenvb4duD3bfa3wVqxYwezZs/npT3+67cZmVtR8J7XVy7Rp04gIjj/++EKHYmY55gRh9VJWVsZOO+1E3759Cx2KmeWYE4TVS3l5OX379mXnnXcudChmlmNOEJa1FStW8Ne//tXDS2bNhBOEZe2ll15i06ZNThBmzYQThGWtrKyMNm3acPTRRxc6FDPLAycIy1pZWRl9+/albdu2hQ7FzPLACcKy8vHHH7v+YNbMOEFYVlx/MGt+nCAsKzX1B9//YNZ8OEFYVsrKyjj66KNdfzBrRpwgbJs+/vhjXn/9dQ8vmTUzThC2Ta4/mDVPThC2TeXl5a4/mDVDThC2TTX1h1122aXQoZhZHjlB2FatXLmSyspKDy+ZNUNOELZVNfWHgQMHFjoUM8szJwjbqrKyMlq3bk2/fv0KHYqZ5ZkThG1VeXm56w9mzVROE4SkIZLmS1og6dqttDtK0kZJZ6ete0fSHEmzJFXkMk7LzPUHs+atVa4OLKklcAdwErAYmClpYkTMy9DuJuCZDIcZFBHv5ypG27qXX36ZjRs3OkGYNVO5/AbRB1gQEQsjYj0wFjg9Q7srgCeBZTmMxbaD6w9mzVsuE0R3YFHa8uJk3WaSugNnAHdn2D+AZyVVShpVVyeSRkmqkFSxfPnyBgjbapSVldGnTx/XH8yaqVwmCGVYF7WWbwWuiYiNGdr2j4gjgFOA70g6LlMnEXFvRJREREnXrl13KGD7zKpVq1x/MGvmclaDIPWNYd+05X2ApbXalABjJQF0AU6VtCEinoqIpQARsUzSeFJDVi/mMF5L4/qDmeXyG8RM4EBJB0hqA4wAJqY3iIgDImL/iNgfeAK4LCKektROUnsASe2AwcDcHMZqtbj+YGY5+wYRERskXU7q6qSWwAMRUSVpdLI9U92hxh7A+OSbRSugNCKm5CpW+7yysjKOOuoo2rVrV+hQzKxAcjnERERMBibXWpcxMUTExWmvFwK9cxmb1W3VqlVUVFRwzTXXFDoUMysg30ltn+P6g5mBE4RlUF5eTqtWrTjmmGMKHYqZFZAThH1Ozf0Prj+YNW9OELaF1atXM3PmTA8vmZkThG2ppv7g5z+YmROEbaGsrMz1BzMDnCCslvLyco466ih23XXXQodiZgXmBGGbuf5gZumcIGyz6dOns2HDBicIMwOcICxNWVkZLVu2dP3BzAAnCEtTM/+S6w9mBk4Qlqiurnb9wcy24ARhgOsPZvZ5ThAGfFZ/6N+/f6FDMbNGwgnCgFSCKCkpcf3BzDZzgjCqq6t57bXXPLxkZltwgjDXH8wsIycIo7y83PUHM/scJwjbXH9o3759oUMxs0YkpwlC0hBJ8yUtkHTtVtodJWmjpLPru6/tGNcfzKwuOUsQkloCdwCnAIcAIyUdUke7m4Bn6ruv7bhXXnmFTz/91M9/MLPPyeU3iD7AgohYGBHrgbHA6RnaXQE8CSzbjn1tB/n+BzOrSy4TRHdgUdry4mTdZpK6A2cAd9d337RjjJJUIali+fLlOxx0c1NeXs6RRx5Jhw4dCh2KmTUyuUwQyrAuai3fClwTERu3Y9/Uyoh7I6IkIkq6du1a/yibsTVr1vDqq6+6/mBmGbXK4bEXA/umLe8DLK3VpgQYKwmgC3CqpA1Z7ms7qKb+4ARhZpnkMkHMBA6UdACwBBgBnJ/eICIOqHkt6UFgUkQ8JanVtva1HVdWVkaLFi1cfzCzjHKWICJig6TLSV2d1BJ4ICKqJI1OtteuO2xz31zF2lyVlZW5/mBmdcrlNwgiYjIwuda6jIkhIi7e1r7WcGrqD1deeWWhQzGzRsp3UjdTM2bMcP3BzLbKCaKZqqk/DBgwoNChmFkj5QTRTJWVlXHEEUe4/mBmdcppDaJYlJSUsHbt2kKHkVfz58/n+9//fqHDMLNGzAkC6NGjB+vWrSt0GHnVu3dvvvnNbxY6DDNrxJwggEceeaTQIZiZNTquQZiZWUZOEGZmlpEThJmZZeQEYWZmGTlBmJlZRk4QZmaWkROEmZll5ARhZmYZKSLjkzyLkqTlwD+3c/cuwPsNGE5j77eQffucm36/hezb51w/+0VExuc1N6kEsSMkVURESXPpt5B9+5ybfr+F7Nvn3HA8xGRmZhk5QZiZWUZOEJ+5t5n1W8i+fc5Nv99C9u1zbiCuQZiZWUb+BmFmZhk5QZiZWUbNKkFIGiJpvqQFkq7NsF2Sfpdsny3piDz23UPSK5LWSfqvPPb7teRcZ0uaLql3Hvs+Pel3lqQKSQPy0W9au6MkbZR0dkP0m03fko6X9HFyzrMk/TQf/ab1PUtSlaTyfPQr6aq0c52b/L53y1PfHSX9SdIbyTlfkqd+O0san/zbfk1Srwbq9wFJyyTNrWN7w79/RUSz+AFaAv8A/j+gDfAGcEitNqcCTwMC+gKv5rHvbsBRwP8F/iuP/R4DdE5en5Lnc96Vz+pghwFv5qPftHZTgcnA2Xk85+OBSQX4t90JmAd8oebfW75+12nthwFT83jOPwJuSl53BT4E2uSh35uBnyWvewDPN9A5HwccAcytY3uDv381p28QfYAFEbEwItYDY4HTa7U5HXg4UmYAnSTtlY++I2JZRMwEPm2A/urT7/SI+ChZnAHsk8e+V0fyLxtoBzTEFRPZ/D0DXAE8CSxrgD7r23dDy6bf84E/RsS/IPXvLU/9phsJPNoA/WbbdwDtJYnUh5EPgQ156PcQ4HmAiHgT2F/SHjvYLxHxIqlzqEuDv381pwTRHViUtrw4WVffNrnqOxfq2+83SH0CyVvfks6Q9CbwZ+Dr+ehXUnfgDODuBuivXn0n+iXDHk9L6pmnfg8COksqk1Qp6aI89QuApF2AIaSSckPIpu/bgYOBpcAc4HsRsSkP/b4BnAkgqQ+wHw33wWtHY6uX5pQglGFd7U+s2bTJVd+5kHW/kgaRShDX5LPviBgfET2A4cANeer3VuCaiNjYAP3Vt+/XSc190xu4DXgqT/22Ao4EhgInAz+RdFAe+q0xDHg5Irb2Cbih+z4ZmAXsDRwO3C6pQx76vZFUMp5F6pvqX9nxby7ZaPD3mVY7snORWQzsm7a8D6lPFvVtk6u+cyGrfiUdBtwPnBIRH+Sz7xoR8aKkL0rqEhE7MtlZNv2WAGNTIw90AU6VtCEintqBfrPqOyJWpr2eLOnOPJ3zYuD9iKgGqiW9CPQG3spxvzVG0HDDS9n2fQlwYzKMuUDS26RqAq/lst/k7/gSSBWOgbeTn1xr+PeZhiieFMMPqWS4EDiAz4pLPWu1GcqWRZ7X8tV3Wtuf03BF6mzO+QvAAuCYAvy+v8RnReojgCU1y/n4XSftH6ThitTZnPOeaefcB/hXPs6Z1FDL80nbXYC5QK98/K6BjqTGztvl+d/XXcDPk9d7JP++uuSh304kxXDgUlJ1gYY67/2pu0jd4O9fDRJ0sfyQqvK/ReoqhP+TrBsNjE5eC7gj2T4HKMlj33uS+gSwEliRvO6Qh37vBz4i9VV8FlCRx3O+BqhK+n0FGJCPfmu1fZAGShBZnvPlyTm/QeqigAZJzNmcM3AVqSuZ5gJX5rHfi4GxDfU7rsfvem/g2eT/8lzggjz12w/4O/Am8EeSqwQboN9HgXdJXciymNSQcE7fvzzVhpmZZdScitRmZlYPThBmZpaRE4SZmWXkBGFmZhk5QZiZWUZOENbsSdo9bcbR9yQtSV6vkDQvB/39XPWcsVfS6jrWP9iQs9GapXOCsGYvIj6IiMMj4nBS8zP9Jnl9OLDNuXskNacZCawZcYIw27qWku5LnifwrKS2AMmkd79MnqvwPUlHSipPJsJ7pmYWTUnflTQvmZ9/bNpxD0mOsVDSd2tWSvpB8tyEuZKurB1MMuf/7ckx/0xqmniznPAnH7OtOxAYGRGXShoHnAU8kmzrFBEDJbUGyoHTI2K5pPNIPdfj68C1wAERsU5Sp7Tj9gAGAe2B+ZLuIvVMjEuAo0ndFfuqpPKI+GvafmcAXwYOJTV9xDzggVycuJkThNnWvR0Rs5LXlaTmwqnxWPLnl4FewHPJBIAtSU2JADAbGCPpKbacufXPEbEOWCdpGak3+wHA+EhNqIekPwLHkpoNtMZxwKORmol2qaSpO36KZpk5QZht3bq01xuBtmnL1cmfAqoiol+G/YeSelP/Kqkptmue/1D7uK3IPF1zJp4fx/LCNQizHTcf6CqpH4Ck1pJ6SmoB7BsRLwBXk5rlc9etHOdFYLikXSS1IzWcNC1DmxGSWiZ1jkENfC5mm/kbhNkOioj1yaWmv5PUkdT/q1tJzfj5SLJOpK6OWpEMQ2U6zuuSHuSz5xXcX6v+ADAeOIHUbJ1vkap9mOWEZ3M1M7OMPMRkZmYZOUGYmVlGThBmZpaRE4SZmWXkBGFmZhk5QZiZWUZOEGZmltH/Dx/fiaZtcNmXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(thresholds, accuracies, color='black')\n",
    "\n",
    "plt.title('Threshold vs Accuracy')\n",
    "plt.xlabel('Threshold')\n",
    "plt.ylabel('Accuracy')\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6179653679653679"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "converted_small = y_pred_small >= 0.5\n",
    "(converted_small == y_val).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6179653679653679"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_val, converted_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size_val = len(y_val)\n",
    "baseline = np.repeat(False, size_val)\n",
    "baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6179653679653679"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(baseline, y_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_positive = ((y_pred >= 0.5) & (y_val == 1)).sum()\n",
    "false_positive = ((y_pred >= 0.5) & (y_val == 0)).sum()\n",
    "false_negative = ((y_pred < 0.5) & (y_val == 1)).sum()\n",
    "true_negative = ((y_pred < 0.5) & (y_val == 0)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[999, 143],\n",
       "       [362, 344]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table = np.array(\n",
    "     # predict neg    pos\n",
    "    [[true_negative, false_positive], # actual neg\n",
    "     [false_negative, true_positive]]) # actual pos\n",
    "\n",
    "confusion_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.54058442, 0.07738095],\n",
       "       [0.19588745, 0.18614719]])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table / confusion_table.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision and recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7063655030800822, 0.48725212464589235)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = true_positive / (true_positive + false_positive)\n",
    "recall = true_positive / (true_positive + false_negative)\n",
    "precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.54058442, 0.07738095],\n",
       "       [0.19588745, 0.18614719]])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table / confusion_table.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7063655030800822, 0.48725212464589235)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = true_positive / (true_positive + false_positive)\n",
    "recall = true_positive / (true_positive + false_negative)\n",
    "precision, recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC and AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TPR and FPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = []\n",
    "\n",
    "thresholds = np.linspace(0, 1, 101)\n",
    "\n",
    "for t in thresholds: #B\n",
    "    tp = ((y_pred >= t) & (y_val == 1)).sum()\n",
    "    fp = ((y_pred >= t) & (y_val == 0)).sum()\n",
    "    fn = ((y_pred < t) & (y_val == 1)).sum()\n",
    "    tn = ((y_pred < t) & (y_val == 0)).sum()\n",
    "    scores.append((t, tp, fp, fn, tn))\n",
    "\n",
    "df_scores = pd.DataFrame(scores)\n",
    "df_scores.columns = ['threshold', 'tp', 'fp', 'fn', 'tn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>469</td>\n",
       "      <td>451</td>\n",
       "      <td>237</td>\n",
       "      <td>691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>409</td>\n",
       "      <td>202</td>\n",
       "      <td>297</td>\n",
       "      <td>940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>344</td>\n",
       "      <td>143</td>\n",
       "      <td>362</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>220</td>\n",
       "      <td>97</td>\n",
       "      <td>486</td>\n",
       "      <td>1045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>90</td>\n",
       "      <td>36</td>\n",
       "      <td>616</td>\n",
       "      <td>1106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>697</td>\n",
       "      <td>1135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn\n",
       "0          0.0  706  1142    0     0\n",
       "10         0.1  706  1142    0     0\n",
       "20         0.2  706  1142    0     0\n",
       "30         0.3  469   451  237   691\n",
       "40         0.4  409   202  297   940\n",
       "50         0.5  344   143  362   999\n",
       "60         0.6  220    97  486  1045\n",
       "70         0.7   90    36  616  1106\n",
       "80         0.8    9     7  697  1135\n",
       "90         0.9    0     0  706  1142\n",
       "100        1.0    0     0  706  1142"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores['tpr'] = df_scores.tp / (df_scores.tp + df_scores.fn)\n",
    "df_scores['fpr'] = df_scores.fp / (df_scores.fp + df_scores.tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>469</td>\n",
       "      <td>451</td>\n",
       "      <td>237</td>\n",
       "      <td>691</td>\n",
       "      <td>0.664306</td>\n",
       "      <td>0.394921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>409</td>\n",
       "      <td>202</td>\n",
       "      <td>297</td>\n",
       "      <td>940</td>\n",
       "      <td>0.579320</td>\n",
       "      <td>0.176883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>344</td>\n",
       "      <td>143</td>\n",
       "      <td>362</td>\n",
       "      <td>999</td>\n",
       "      <td>0.487252</td>\n",
       "      <td>0.125219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>220</td>\n",
       "      <td>97</td>\n",
       "      <td>486</td>\n",
       "      <td>1045</td>\n",
       "      <td>0.311615</td>\n",
       "      <td>0.084939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>90</td>\n",
       "      <td>36</td>\n",
       "      <td>616</td>\n",
       "      <td>1106</td>\n",
       "      <td>0.127479</td>\n",
       "      <td>0.031524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>697</td>\n",
       "      <td>1135</td>\n",
       "      <td>0.012748</td>\n",
       "      <td>0.006130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn       tpr       fpr\n",
       "0          0.0  706  1142    0     0  1.000000  1.000000\n",
       "10         0.1  706  1142    0     0  1.000000  1.000000\n",
       "20         0.2  706  1142    0     0  1.000000  1.000000\n",
       "30         0.3  469   451  237   691  0.664306  0.394921\n",
       "40         0.4  409   202  297   940  0.579320  0.176883\n",
       "50         0.5  344   143  362   999  0.487252  0.125219\n",
       "60         0.6  220    97  486  1045  0.311615  0.084939\n",
       "70         0.7   90    36  616  1106  0.127479  0.031524\n",
       "80         0.8    9     7  697  1135  0.012748  0.006130\n",
       "90         0.9    0     0  706  1142  0.000000  0.000000\n",
       "100        1.0    0     0  706  1142  0.000000  0.000000"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA230lEQVR4nO3deXhN1/rA8e+bgRASxCxiDoomalaKqhqqlKLVlltDPeq2RUtx/Yr29qqOXK3SW9c1VA0taq7eGmpW1ExjHqJFk8YURIb1++Oc5AYJJ7JzdnLyfp7nPD3nrH32enca79lZe+31ijEGpZRSOZ+X3QEopZSyhiZ0pZTyEJrQlVLKQ2hCV0opD6EJXSmlPIQmdKWU8hCa0JW6TyJyUkQeszsOpZJpQle2EpGrqR5JInI91evnRWSMiMQ7X18Ukc0i0sj52RdFJNHZdllE9ohIe7uPCUBEpovIzduO7xln28lUx3leRP4jIgWcbetE5IazLUpEFopIKXuPRuUUmtCVrYwxBZIfwGngyVTvzXZuNs/ZXgzYCCwUEXG2bXG2FQI+B+aKSCH3HkW6Pkh9fMaYeanannTG/RBQD/i/VG2vONsqAwWAj9wXssrJNKGrHMMYEw/MAEoCQbe1JQGzAH+gSlqfF5HCIrJMRP4QkRjn8+BU7etE5O8isklErojIDyJSNFV7DxE5JSLRIjLSomM6C6wEaqbRdhH4Dgi3oi/l+TShqxxDRPICLwKRxpio29q8gV5APHAqnV14Af8BygEhwHXgs9u2ec65n+JAHmCIc/8PAJOBHkBpHF8owWSSiJQF2gG70mgLAjoDRzPbj8odNKGrnKCbiFwEzgB1gKdStTV0tt3AMTTxgjHmQlo7McZEG2MWGGOuGWOuAP8Amt222X+MMYeNMdeB+fzv7LgLsMwYs94YEwe8BSTdI+4hznH/iyISdVvbd864NwI/AWNTtU0UkUtAFFAUePUe/SgFaEJXOcN8Y0whY0xxY8yjxpidqdq2GmMKAYWBJUDT9HYiIvlF5AvnsMllYD1QyHl2n+xcqufXcIxhg+Os/ExygzEmFoi+R9wfOeMuZIwpelvbU873yxljBji/QJK9ZowJBB50Hlem/xJQuYMmdOURjDFXgQFADxGpnc5mbwBVgQbGmADgEef7ks72qf0OlE1+ISL5uW0c32rGmH3Au8CkVBeBlUqXJnTlMYwx0cBUYFQ6mxTEMW5+UUSKAKMzsPtvgfYi0kRE8gDv4J5/PzNwjOd3cENfKofThK48zQSgnYg8mE5bPhxj01uB713dqTHmAPBX4GscZ+sxQGQmY3Wl35vARBxj9krdlWiBC6WU8gx6hq6UUh5CE7pSSnkITehKKeUhNKErpZSH8LGr46JFi5ry5cvb1b1SSuVIO3fujDLGFEurzbaEXr58eXbs2GFX90oplSOJSHprFemQi1JKeQpN6Eop5SE0oSullIewbQxdKaWsEB8fT2RkJDdu3LA7FEv5+fkRHByMr6+vy5/RhK6UytEiIyMpWLAg5cuXx1MWpTTGEB0dTWRkJBUqVHD5c/ccchGRaSJyQUT2p9MuIjJRRI6KyF4ReSgDcSulVKbcuHGDoKAgj0nmACJCUFBQhv/qcGUMfTrQ5i7tbXHUcKwC9MNRpksppdzGk5J5svs5pnsOuRhj1otI+bts0hGYaRzLNm4VkUIiUsoY83uGo3HB/v37effdd/ntt99ued/Ly4tHHnHUKzh06BDnz5+/pd3X15eHH344ZR9RUbdWBPPz86Nhw4YA7Nmzh5iYmFva/f39qVevHgC//PILly9fvqU9ICCAhx5y/HHy888/c+3atVvaCxcuTFhYGABbtmwhLi7ulvZixYpRo0YNADZu3EhCQsIt7SVLlqRatWoAHDx4kIkTJ1K6dOk7f0BKqdzLGHPPB1Ae2J9O2zKgSarXq4G66WzbD9gB7AgJCTH3Y/78+QZI8yEiRkQ8uj35uZeXl+nTp485cuTIff0clfIUBw8etLX/qKgoExYWZsLCwkyJEiVM6dKlU14DJiwszNSoUcN06dLFxMbGGmOM8fLySnm/ffv2JiYmJs19p3VswA6TTq62YtpiWn8XpLnIujHmX8aYusaYusWKpXnn6j117do13S+epKQkkpKSPLrdGMPo0aNJSkpi2rRphIeH88cff9zXz1IplXlBQUHs3r2b3bt3079/fwYPHpzy2t/fn927d7N//37y5MnDlClTAMiXL1/K+0WKFGHSpEmWxGJFQo8kVa1FHAVtf0tnW2WBMWPGMH/+fLy9vYmNjdUlFJTKAZo2bcrRo0fveL9Ro0acPXvWkj6smLa4BHhFROYCDYBLJovGz9X/dO3alcjISF5//XVWr15N27Zt7Q5JKdsNGjSI3bt3W7rP8PBwJkyYkKl9JCQksHLlStq0uXV+SWJiIqtXr6ZPnz6Z2n8yV6YtzgG2AFVFJFJE+ohIfxHp79xkBXAcOAp8iaPyunKDvn37kidPHpKSkuwORSmVhuvXrxMeHk7dunUJCQlJSdzJ7wcFBfHnn3/SqlUrS/pzZZZL93u0GxzFc5WbFSxYkNDQ0DT/jFMqN8rsmbTVksfK03v/0qVLtG/fnkmTJvHaa69luj9dyyWHK1CgAJs3b7Y7DKXUfQgMDGTixIl89NFHxMfHZ3p/mtA9QHR0NLGxsXaHoZS6D7Vr1yYsLIy5c+dmel+6lksOFx4eztatW/nxxx/p2LGj3eEolauNGTPmltdXr15Nc7vb31+6dKkl/esZeg6XfHfs2rVrbY5EKWU3Teg5XHJC37lzp82RKKXspgk9hytdujReXl53rF2jlMp9dAw9hxMRatWqpQt1KaX0DN0ThIaGcuTIEbvDUErZTBO6ByhTpgxHjx7lm2++sTsUpZSNNKF7gJo1awKwcuVKmyNRKnfy9vYmPDw85XHy5EnWrVtHYGAgtWvXpnr16rz99tsAt7xfrVo1hgwZYlkcOobuAZILX+iqi0rZI61b/E+ePEnTpk1ZtmwZsbGxhIeH0759e4CU969fv07t2rXp1KlTSgGezHDpDF1E2ohIhLNu6PA02guLyCJnTdGfRaRmpiNTLqtSpQoAR44cSS4kopTKRvz9/alTpw7Hjh275f18+fIRHh7uvuVzRcQbmAS0wrH2+XYRWWKMOZhqs78Bu40xnUSkmnP7lpZEqO6pWLFi+Pn5cePGDU6ePJmhKuFKeZrmzZvf8V63bt0YMGAA165do127dne0v/jii7z44otERUXRpUuXW9rWrVt3zz6TV08EqFChAosWLbqlPTo6mq1bt/LWW2/dUpAmJiaGI0eOpNxPklmuDLnUB44aY44DONc97wikTugPAO8BGGN+FZHyIlLCGKOTo91ARChfvjwXLlzIcJVwpVTmpbeq4oYNG6hduzZeXl4MHz6cGjVqsG7dOjZs2MCDDz5IREQEw4cPp2TJkpbE4UpCLwOcSfU6Ekchi9T2AJ2BjSJSHyiHo3LRLQldRPrhqCtKSEjIfYas0pK8pkv16tXtDkUpW93tjDp//vx3bS9atKhLZ+SuSh4rT+/9w4cP06RJEzp16pRyhp8Zroyhu1IzdBxQWER2A68Cu4CEOz5kQU1RlbYqVapw+vRp4uLi7A5FKeWi0NBQRowYwfvvv2/J/lxJ6PesGWqMuWyM6WWMCQd6AsWAE5ZEqFwSGhpKUlISffv2tTsUpVQG9O/fn/Xr13PiROZTpitDLtuBKiJSATgLPAs8l3oDESkEXDPG3AT6AuuNMZczHZ1yWfJQy4EDB2yORKncJ61lcps3b57mBdrb38+XL5/7ZrkYYxJE5BVgFeANTDPGHEiuKWqMmQJUB2aKSCKOi6XWVDxVLktO6Fb9Yiilch6XbiwyxqzAUQw69XtTUj3fAlSxNjSVEfnz5ycgIICYmBi7Q1FK2URv/fcgpUuXJj4+nitXrtgdilJu5Yk31N3PMWlC9yDh4eGISLplr5TyRH5+fkRHR3tUUjfGEB0djZ+fX4Y+p2u5eJB27doxd+5cLl68SKlSpewORym3CA4OJjIy8pY7MD2Bn58fwcHBGfqMJnQPUqNGDQD27NmjNxipXMPX11eXu3DSIRcPUrlyZQCmTJlyjy2VUp5IE7oHCQgIwNvbm1OnTtkdilLKBprQPUzBggU9bixRKeUaTegepnjx4sTGxhIfH293KEopN9OE7mGSLw4dPnzY5kiUUu6mCd3DJJe42rdvn82RKKXcTRO6h+nTpw9eXl5ERETYHYpSys2sqikaKCJLRWSPiBwQkV7Wh6pc4efnR7ly5dKsnqKU8mxW1RT9K3DQGPOkiBQDIkRktnM5XeVGkZGRnDhxQkvRKZULuXKGnlJT1Jmgk2uKpmaAgiIiQAHgT9KoWKSyXqlSpRARzp07x82b+n2qVG7iSkJPq6Zomdu2+QzHmui/AfuAgcaYpNt3JCL9RGSHiOzQudJZw8fHh8KFC2OM0ZkuSuUyVtUUbQ3sBkoD4cBnIhJwx4e0pqhbJBfg3rZtm82RKKXcyZKaokAvYKFxOIqjnmg1a0JUGRUaGoq3tzcbNmywOxSllBu5ktBTaoqKSB4cNUWX3LbNaaAlgIiUAKoCx60MVLmuV69ehIWFsXHjRrtDUUq50T0TujEmAUiuKXoImJ9cUzS5rijwd6CxiOwDVgPDjDFRWRW0urs2bdrw/PPPc+zYMX7//Xe7w1FKuYlVNUV/Ax63NjR1v+Li4ihRogQAGzdupGvXrjZHpJRyB71T1AMdOHCAF154gTx58ug4ulK5iCZ0D1SpUiXAMdtFE7pSuYcmdA8UGBhIkSJFCAwMZO/evVy6dMnukJRSbqAJ3UMln6UnJSWxZcsWm6NRSrmDJnQPValSJf7880+8vb11+qJSuYRLs1xUzjNo0CAuX77MyJEjdRxdqVxCz9A9VIMGDWjVqhVNmjTh559/Ji4uzu6QlFJZTBO6h4qNjWXp0qVUrVqVGzduMGHCBIy5fQkepZQn0YTuof788086dOhAQkICTz31FMOHD6dfv366pK5SHkwTuocqXbo0efLk4fTp0yxYsICRI0cydepUHnvsMc6fP293eEqpLGBVCbqhIrLb+dgvIokiUsT6cJWrvL29qVChAseOHcPLy4t3332X2bNns337dsLCwvjxxx/tDlEpZbF7JvRUJejaAg8A3UXkgdTbGGM+NMaEG2PCgRHAT8aYP7MgXpUBlSpV4tixYymvn3vuObZv305QUBCPP/44f/vb30hI0MJSSnkKq0rQpdYdmGNFcCpzKlasyPHjx2+5GFqzZk22b99Onz59eO+99+jbt69eLFXKQ1hVgg4AEckPtAEWZD40lVmvv/46mzdvvuP9/Pnz8+WXXzJmzBhmzJjBiBEjbIhOKWU1V24scqUEXbIngU3pDbeISD+gH/yvTJrKOhUqVLhr+6hRozh//jzvv/8+JUuWZNCgQe4JTCmVJawqQZfsWe4y3KI1Rd0rNjaWzz//nF9++SXNdhHh008/5emnn2bw4MGMHj1ab0BSKgezqgQdIhIINAMWWxuiul8iwl//+ldWrFiR7jbe3t589dVXvPDCC7zzzjs89NBDbN261Y1RKqWsYlUJOoBOwA/GmNisCVVlVP78+SlVqtQtM13S4ufnx6xZs1i2bBlXrlyhcePGvP/++3qxVKkcxpISdM7X04HpVgWmrFGpUiWOH3etXvcTTzzBgQMHeOmllxg+fDh//PEHH3zwAV5eev+ZUjmBrrbo4SpVqpShm4gKFizI119/TbFixfj444+Jiopi6tSp+Pjor4pS2Z2eenm4SpUqcfbsWW7cuOHyZ7y8vJg4cSJvv/02M2bMoHv37sTHx2dhlEopK2hC93CvvvoqFy9exM/PL0OfExFGjRrFJ598wrfffssLL7ygd5Uqlc3p39EerlChQpn6/ODBg0lMTGTo0KF4e3szc+ZMHX5RKpvSf5m5wKRJk4iKimL06NH39fkhQ4aQmJjI8OHD8fLyYsaMGXh7e1scpVIqs3TIJRfYsWMHEyZMyNSQybBhwxg7diyzZ8+md+/eJCYmWhihUsoKmtBzgSeeeIKLFy+yZcuWTO1nxIgRvPPOO8ycOZO+ffuSlJRkUYRKKStoQs8FWrVqhY+PD8uXL8/0vt566y1Gjx7N9OnTad26NXv37rUgQqWUFTSh5wKBgYE0bdrUkoQOMHr0aCZNmsTOnTsJDw+nd+/e/P7775bsWyl1/zSh5xJPP/00wcHBGZqPnh4RYcCAARw9epTBgwcze/ZsqlevzpdffqnLBShlI7HrH2DdunXNjh07bOlbWevIkSP069ePdevW0bx5c7744gtCQ0PtDkspjyQiO40xddNqs6SmqHOb5s6aogdE5KfMBKyyzpUrVyzfZ5UqVVizZg3/+te/2LVrFzVq1ODVV1/ljz/+sLwvpVT6LKkpKiKFgM+BDsaYGkBX60NVmfXBBx9QqlQprl+/bvm+RYSXXnqJiIgI+vbty+TJk6lcuTLvvfdelvSnlLqTVTVFnwMWGmNOAxhjLlgbprJCWFgYsbGxrF27Nsv6KFGiBJMnT2b//v00b96cv/3tb4SGhjJz5kyd5qhUFrOqpmgoUFhE1onIThHpmdaORKSfiOwQkR3657j7NWvWjPz587Ny5cos76tatWosXryYdevWUbJkSf7yl79QoUIF+vXrx4IFC7h48WKWx6BUbuNKQnelpqgPUAd4AmgNvCUid1wV0xJ09vLz86Nly5asWLHCbbNRmjVrxrZt25g3bx516tRh3rx5dOnShaCgIBo3bsyYMWPYsmWLLvyllAWsqikaCXxvjIk1xkQB64Ewa0JUVmrbti3Hjx/n8OHDbuvTy8uLbt26sXDhQqKiotiwYQMjR44kMTGRd955h8aNG1OsWDG6dOnC9OnTuXz5sttiU8qT3HPaooj4AIeBlsBZHDVGnzPGHEi1TXXgMxxn53mAn4FnjTH709uvTlu0x9mzZ1myZAndunUjKCjI7nCIjo5m9erVrFq1ilWrVnH27Fny5ctHp06deO6552jevDn+/v52h6lUtnG3aYsuzUMXkXbABMAbmGaM+UdyPdHkUnQiMhToBSQBU40xE+62T03o6nbGGLZt28bMmTOZO3cuMTEx5MmTh6ZNm/Lkk0/Sq1cvAgIC7A5TKVtlOqFnBU3o9omOjmbx4sV069aNAgUK2B1OmuLi4li/fn3Kmfv+/fsJCAjg5ZdfZuDAgZQqVcruEJWyRaZvLFKeZc+ePfTp04c1a9bYHUq68ubNS6tWrfjoo4/Yt28f27dvp02bNnz44YeUL1+evn378uuvv9odplLZiib0XKhJkyYUKFCAFStW2B2Ky+rWrcu8efM4fPgwL730El9//TXVq1enY8eObNq0ye7wlMoWNKHnQnny5KFVq1Zunb5olUqVKvHZZ59x6tQpRo0axaZNm2jSpAmNGzdm4cKFWsxa5Wqa0HOpdu3acebMGQ4ePGh3KPelWLFivP3225w6dYpPP/2Uc+fO8fTTT1OmTBkGDhzIrl277A5RKbfThJ5LtW7dGiDTVYzs5u/vzyuvvMLhw4dZsmQJzZs3Z8qUKTz00EMMGTKEmzdv2h2iUm6js1xysfPnz1OiRAm7w7BcTEwMb731FpMmTaJu3brMnTuXSpUq2R2WUpbQWS4qTZ6YzAEKFy7MZ599xoIFCzh69Ci1a9fmiy++0MXBlMfThJ6LHTp0iM6dO3Po0CG7Q8kSnTt3Zvfu3dSrV4/+/fvz6KOPunXJA6XcTRN6Lubn58eiRYuydDldu5UrV44ff/yRf//73+zZs4ewsDD++9//2h2WUllCE3ouVr58eYKDg/npJ88uMCUi9O7dm4MHDxIaGkrnzp3R6zfKE2lCz8VEhGbNmrF+/focNx/9fpQqVYqVK1dStGhR2rVrx5EjR+wOSSlLWVJT1FlP9JKzpuhuERllfagqKzzyyCOcO3cu1yS30qVLs2rVKowxtG7dmp07d9odklKWsaSmqNMGY0y48/GOxXGqLNK8eXPq1q1LTEyM3aG4TWhoKCtWrODq1avUrVuXF154gVOnTtkdllKZZlVNUZVDhYaGsn37dho0aGB3KG5Vr149jhw5wogRI1iwYAFVq1blq6++sjsspTLFqpqiAI1EZI+IrBSRGmntSGuKZl+58Y7KwMBAxo4dy+HDh2nUqBE9evRg4sSJdoel1H2zqqboL0A5Y0wY8CnwXVo70pqi2dPChQspVKgQJ0+etDsUW5QtW5aVK1fSqVMnBg4cyFtvvZUrLhIrz2NJTVFjzGVjzFXn8xWAr4gUtSxKlaUqV67M9evXs/X66FnNz8+P+fPn06dPH959911efPFFbty4YXdYSmWIKwl9O1BFRCqISB7gWWBJ6g1EpKSIiPN5fed+o60OVmWNWrVqERISwsKFC+0OxVY+Pj58+eWXvP3228ycOZNHH32Uc+fO2R2WUi67Z0I3xiQArwCrgEPAfGPMARHpn1xXFOgC7BeRPcBEHAWi9W/WHEJE6NKlCz/88AMXL160OxxbiQijRo3im2++Yffu3dSvX5+IiAi7w1LKJbraogJg69atNGrUiBkzZtCzZ0+7w8kWdu3aRevWrQkKCuLnn3+mYMGCdoeklK62qO6tQYMGjBs3jsaNG9sdSrZRu3btlLJ3ffr00QulKtvThK4Ax1DDsGHDqFy5st2hZCstWrTgvffe45tvvmHChAl2h6PUXWlCVykSExNZunSpFl2+zdChQ3nqqacYOnQoq1evtjscpdKlCV2lEBEGDBjABx98YHco2YqIMH36dKpVq0aHDh08fnVKlXNpQlcpvLy86NKlC6tWreLy5ct2h5OtBAYGsnr1akJCQnjiiSfYuHGj3SEpdQdN6OoWXbt2JS4ujiVLltx741ymRIkSrFmzhjJlytC2bVs9U1fZjiZ0dYuGDRsSHBzMN998Y3co2VKpUqVYu3YtZcuW5bHHHmPq1Kl2h6RUCk3o6hZeXl507dqVQ4cOkZCQYHc42VLp0qXZvHkzLVu25KWXXmLQoEH6s1LZgt5YpO4QGxtLvnz58PLS7/u7SUhI4M0332T8+PG0b9+e+fPnky9fPrvDUh5ObyxSGeLv74+Xl5feSHMPPj4+fPLJJ3z++ecsX76c1q1b5/qlE5S9NKGrNC1atIiQkBD+/PNPu0PJ9l5++WXmzp3L1q1bad68uS7opWxjSU3RVNvVE5FEEeliXYjKDmXLliUyMpLvvvvO7lByhG7durFs2TKOHDlCkyZNOH78uN0hqVzIspqizu3ex7Eqo8rh6tSpQ8WKFZk3b57doeQYjz/+OKtXryYmJoaHH36YvXv32h2SymWsrCn6KrAAuGBhfMomIkK3bt1YvXo1Wi7QdQ0bNmTDhg14e3vzyCOPsHXrVrtDUrmIJTVFRaQM0AmYcrcdaU3RnOWZZ54hMTGRBQsW2B1KjvLAAw+wadMmihQpwvPPP8/169ftDknlElbVFJ0ADDPGJN5tR1pTNGcJCwtj+PDh1K9f3+5Qcpxy5coxdepUjh8/zvvvv293OCqXuOc8dBFpBIwxxrR2vh4BYIx5L9U2J/hf4i8KXAP6GWO+S2+/Og9d5Qbdu3dn0aJFHDhwgEqVKtkdjvIAmZ2Hfs+aosaYCsaY8saY8sC3wIC7JXOVs+zatUtnu9ynjz76CF9fX1577TWd16+ynFU1RZUHe+edd+jfvz/x8fF2h5LjlClThrfffpsVK1awePFiu8NRHk5v/Vf3tHTpUjp06MDixYvp0KGD3eHkOPHx8dStW5fz58+zd+9eihcvbndIKgfTW/9VprRt25aSJUsybdo0u0PJkXx9fZk9ezaXLl3iL3/5C0lJSXaHpDyUJnR1Tz4+PvTs2ZNly5bpbe33qWbNmnz88cd8//33/POf/7Q7HOWhNKErl/Tq1Yv8+fPz66+/2h1KjvXyyy/TsWNHhg0bxq5du+wOR3kgHUNXLrt06RKBgYF2h5GjRUdHExYWhr+/Pzt37qRAgQJ2h6RyGB1DV5YIDAzEGMORI0fsDiXHCgoK4quvvuLIkSMMHDjQ7nCUh9GErjJk2LBh1K1bV5fVzYTmzZszcuRIpk2bpoufKUtpQlcZ0qNHD65cucLHH39sdyg52ujRo2nUqBH9+vXjxIkTdoejPIQmdJUhtWrV4plnnuGf//wnFy7owpr3y8fHh6+//hpwrKWuC3gpK2hCVxn29ttvc/36dT766CO7Q8nRypcvz6xZs9i5cye9evXSpQFUpmlCVxkWGhpKly5dWLFihSahTOrQoQPjxo1j3rx5/P3vf7c7HJXDWVKCTkQ6isheEdntXO+8ifWhquzks88+Y9euXYiktbqyyoihQ4fyl7/8hdGjRzN//ny7w1E5mCvL53oDh4FWOIpbbAe6G2MOptqmABBrjDEi8iCOBbyq3W2/Og/dM8THx+Pj46OJPZPi4uJ49NFH2bdvH/v27aNcuXJ2h6SyqczOQ79nCTpjzFXzv28Gf+4sgKE80L59+6hQoQJr1661O5QcL2/evMyePRuA3r1763ov6r5YUoIOQEQ6icivwHKgtzXhqeysSpUqxMXFMXHiRLtD8Qjly5dn/PjxrFmzhs8//9zucFQOZFUJOowxi5zDLE8BaV7d0ZqinsXPz49+/fqxZMkSnUttkd69e9OuXTvefPNNvSNXZZgrCT0SKJvqdTDwW3obG2PWA5VEpGgabVpT1MO8/PLLeHl56QqCFhERvvzyS/z8/OjYsSM7d+60OySVg1hSgk5EKovzqpiIPATkAaKtDlZlP8HBwfTo0YPJkyfz+++/2x2ORyhdujTffPMNMTExNGjQgDfffJNr167ZHZbKAawqQfc0sF9EdgOTgGeMTlDONd555x2+++47SpYsaXcoHqNly5YcOnSIXr168eGHH/Lwww/r3aTqnnT5XGUpY4xOYbTYokWL6Ny5My+//LJeLFW6fK5yj3fffZcePXrYHYbH6dSpE2+88QaTJ09m0aJFdoejsjFN6Moyxhhmz57Ntm3b7A7F44wdO5Y6derQt29fIiMj7Q5HZVOa0JVlBg8eTPHixXnjjTd0jReL5cmThzlz5hAXF8fzzz9PQkKC3SGpbEgTurJMgQIF+Mc//sGmTZuYM2eO3eF4nCpVqjB58mTWr1/PyJEj7Q5HZUN6UVRZKikpiQYNGnDhwgWOHTuGj4+P3SF5nJdffpkpU6awYMECOnfubHc4ys3udlFUE7qy3J49e/Dy8qJWrVp2h+KR4uLieOSRRzh06BDbt2+natWqdoek3EhnuSi3CgsLS0nmN2/etDkaz5M3b16++eYb8uTJQ+fOnbl8+bLdIalsQhO6yjL9+vWjadOmWlA6C4SEhDB37lwiIiJ47rnnSExMtDsklQ1oQldZpl27duzevZumTZvqVLss8NhjjzFx4kSWL1/OsGHD7A5HZQOa0FWWeeqpp/j+++85c+YMjRs35tChQ3aH5HEGDBjAK6+8wscff8y0adPsDkfZTBO6ylItWrTgp59+4ubNm7Rr107H1LPA+PHjefzxx+nfvz9bt261OxxlI6tqij7vrCm6V0Q2i0iY9aGqnKp27dqsWrWK//znP+TJk8fucDyOj48Pc+fOJTg4mK5du6K1BnKveyZ0Z03RSUBb4AGgu4g8cNtmJ4BmxpgHcRS3+JfVgaqcLSwsjObNmwOwefNmLbFmscKFC7NgwQL++OMPunfvrhdJcymraopuNsbEOF9uxVEEQ6k7bN26lYcffpjBgwcTFxdndzgepXbt2nz++eesXr2aUaNG2R2OsoFlNUVT6QOsTKtBS9CpBg0a8MorrzBx4kTCw8NZv3693SF5lN69e9O3b1/Gjh3LkiVL7v0B5VEsqykKICItcCT0NOdQaQk6JSJ8+umnrFixghs3btCsWTOGDBlid1ge5dNPP+Whhx6iZ8+eHDt2zO5wlBtZVlNURB4EpgIdjTFafk7dVdu2bTlw4ADDhg2jevXqACQkJGhVHgv4+fnx7bff4uXlxdNPP63l63IRq2qKhgALgR7GmMPWh6k8Uf78+Rk3bhx9+vQBYOrUqVSsWJGRI0dy4sQJm6PL2SpUqMDs2bPZu3cvAwYM0OWMcwmraoqOAoKAz0Vkt4joqlsqw2rWrEndunUZN24cFStWpGPHjpw5c+beH1Rpatu2LW+99RYzZsxg3rx5doej3EBXW1TZTmRkJFOnTuXDDz+kTp06euE0ExISEmjSpAmHDx/mwIEDlCpVyu6QVCbp8rkqRzp27Bg3btygRo0aREREsHHjRtq0aUOZMnebZKVuFxERQXh4OC1btmTp0qVaxDuH0+VzVY5UqVIlatSoAcDixYvp27cvwcHBhIWFMWzYMNatW6el2FxQtWpVxo0bx/Lly/nPf/5jdzgqC+kZusoRjDHs37+flStXsnLlSjZu3Iivry9//vknfn5+TJ8+ncuXL9OjRw8KFy5sd7jZTlJSEi1btmTnzp0sX76cpk2b2h2Suk96hq5yPBGhVq1avPnmm6xdu5bo6Gh+/PFH/Pz8AFi0aBEDBw6kbNmyDBo0SGfJ3MbLy4uZM2dSunRpWrZsyb///W+7Q1JZQBO6ypECAgJo3LhxyuvFixezZ88eOnfuzKRJk6hYsSIjRoxIaY+KirIjzGylbNmybN26lRYtWtC3b19ef/11XVPHw2hCVx7jwQcfZObMmZw4cYIPPviAVq1aAXDgwAFKlCjBY489xr///e9cvYZMoUKFWL58Oa+99hrjx49n0KBBOkfdg2hCVx4nODiYoUOH8uijjwKOJDZixAhOnTpF3759qVGjBt99912uTWQ+Pj5MmDCB119/nU8//ZRx48bZHZKyiF4UVbmGMYZVq1bxxhtvEBkZyYkTJyhSpAhLlizB19eXOnXqULx4cbvDdJukpCR69uzJ7NmzmTp1asoduyp703noSqWSkJDA/v37CQ8PB6BKlSocPXoUgMqVK9OoUSPat29Pt27dbIzSPW7evEmHDh3473//y/Lly2nTpo3dIal70ISu1F389ttvHDt2jG3btrF582a2bNnCU089xeTJk0lISKBhw4Y0atSINm3aULFiRQIDAylSpEjKDJuc7urVqzRp0oQTJ06wZcsWHnjg9vo1KjvRhK5UBsXHx+Pr68v58+fp3bs3a9euvWUlyIkTJ/Lqq69y8+ZN4uPj8ff3tzHazDt9+jT169fH39+fbdu2UbRoUbtDUunI9Dx0F2qKVhORLSISJyK6uLXK8Xx9fQEoUaIEy5cvJzo6mjVr1jBnzhymTJlCixYtAMf899KlS9OnTx9++OGHHHvnakhICIsXL+bs2bM8/fTT3Lhxw+6Q1H245xm6s6boYaAVjrXRtwPdjTEHU21THCgHPAXEGGM+ulfHeoauPMGePXsYP348Cxcu5MqVKxQtWpRmzZoxd+5cfHx8OHr0KHFxcQQEBFCqVCl8fHzsDvmu5syZw3PPPUejRo1YuHAhJUuWtDskdZvMnqG7UlP0gjFmOxCf6WiVykHCwsKYPn06Fy5cYOHChTz++ONERUWlJO433niDmjVrEhISQunSpRkwYAAbNmywOer0de/enW+//ZY9e/ZQv359du3aZXdIKgNcOUPvArQxxvR1vu4BNDDGvJLGtmOAq+mdoYtIP6AfQEhISJ1Tp05lLnqlsrnt27dz8uRJYmJiWLNmDUuXLqVBgwasWbMGgMmTJ9OkSRNq1aplc6S32rVrFx07diQ6OppRo0bx2muvkS9fPrvDUtz9DB1jzF0fQFdgaqrXPYBP09l2DDDkXvs0xlCnTh2jVG5z9epVc+zYsZTn3t7eRkRM7969zW+//WZzdLc6d+6cad++vQFM2bJlzaxZs0xiYqLdYeV6wA6TTl61rKaoUure/P39qVixYsrz8+fP88YbbzBr1iyqVKnC4MGD+eOPP2yO0qFEiRIsXbqUNWvWUKxYMXr06EG3bt2IjY21OzSVDktqiiql7k9QUBAffvghBw8epHXr1kyZMoW8efMCMG3aNAYNGsSqVatsnXXSokULtm/fzocffsiiRYto0qSJlgbMpiypKSoiJUUkEngd+D8RiRSRgKwMXClPUrlyZRYsWMClS5cICHD804mIiOCLL76gTZs2BAYG0rhxY0aPHp3ymaioKLdNk/Ty8mLIkCEsXbqU48ePU69ePbZs2eKWvpXr9MYipbKxa9eu8dNPP7Fu3To2bdpEYGAgy5cvBxxFtc+dO8ebb77JX//6V7fd3HTw4EE6dOjAmTNn+PLLL+nZs6db+lUOeqeoUh7CGJNSE3TWrFl8/fXXfP/99xQvXpyhQ4fSo0cPSpQokeVxREdH07VrV9auXcubb77J2LFj8fb2zvJ+lVYsUspjpC7w3KNHD1auXMmmTZuoWbMmQ4cOZfPmzQDcuHGD+Pisuy0kKCiIVatWMWDAAD744AOeffbZXL3OfHahCV2pHK5x48asXr2agwcP0rZtWwDGjh1LrVq1WL58eZat++7r68ukSZP45JNP+Pbbb3niiSe4cuVKlvSlXKMJXSkPUb169ZQVIBs1aoQxhvbt29OsWTP+7//+jx9//DFL+h08eDAzZsxg3bp1PProo5w/fz5L+lH3pgldKQ/Utm1b9u3bx4QJE4iKimLcuHHMmzcvpf3mzZuW9tezZ08WLVrE/v37qVGjBrNmzcq1FaHspAldKQ+VJ08eBg4cyMGDB4mNjeX9998HYPny5dSoUYNNmzZZ2t+TTz7Jzp07CQ0NpWfPnrRr147Nmzfn2BUocyJN6ErlAnnz5qVIkSIABAYGkpCQQJMmTWjQoAHjx4+37EahBx54gA0bNjBx4kQ2bNjAww8/TLFixejSpYvlXyDqTjptUalc6MqVK0yePJm5c+eya9cuHn74YTZu3AhA//798fPzo0aNGlSvXp18+fJRpEgRKlSokKE+YmJi+PHHH/nhhx9YtmwZUVFRTJ48mb59+2bFIeUamVqcK6seujiXUtlDRESEWblyZcrrli1bGn9/fwOkPHr27GmMMSYpKcl88skn5tSpUxnqIyYmxrRu3doAZuDAgSY+Pt7SY8hNuMviXHqGrpS6Q1JSEqdOnSIiIoL4+HiCg4OpXbs2+/btIywsDGMMJUuWJCQkhLJlyzJkyBAaNmzImTNniIyMpGHDhrfMmQdHce6hQ4cyYcIE6taty8cff8wjjzxi0xHmXHc7Q8/e5VOUUrbw8vKiQoUKdwyz1KpVixMnTjBnzhyOHDnCmTNnOHjwIJcvXwZg7969tG/fnpo1a/LSSy9Rr149AgMDqVq1Kj4+PowfP5769eszdOhQmjVrRocOHRg9ejS1a9e+4wtAZZxLZ+gi0gb4J+CNY230cbe1i7O9HXANeNEY88vd9qln6Ep5nqtXrzJ37ly++OILUv/7vnr1Kv7+/syZM4fDhw9TuXJlNm7cyKxZs4iNjaVWrVr07NmT559/nlKlStl4BNlfptZycbGmaDvgVRwJvQHwT2NMg7vtVxO6Up7t4MGDnDp1ikuXLvHMM88gIgwYMIApU6bcMke9UKFCVKtWja1btyIiVK1alaZNm9KwYUOCg4MpWbIkDz74IACHDx8mMTGRgIAAAgMDyZcvHyKCl5djwl5SUhLgWCLBU8/4M5vQGwFjjDGtna9HABhj3ku1zRfAOmPMHOfrCKC5Meb39ParCV2p3Ony5cscP36cM2fOcPr0aW7evMngwYOJiIigZcuWnD179pbtfX19qVy5MiLCsWPH7lgzxt/fn3LlygFw5MiRlDVsvLy88Pb2pmDBginFrk+cOJGS9JMVKFAgZUGzY8eO3RFvQEAAxYoVwxjD8ePH72gvVKgQQUFBJCYmcvLkyTvaixQpQuHChYmPj+f06dPkzZuX0aNH8/rrr7v4E7tVZsfQywCpJ6lG4jgLv9c2ZYBbEvptNUVd6Fop5WkCAgIIDw8nPDz8lverVq2ackb/ww8/sGvXLs6ePcvFixdT6pkGBQURFxdHQkIC8fHxJCYm3pLQfX19iY+PJykpKWWbgIAAqlatCkBsbOwdNzoVK1aMypUrA47pnLcn/JIlS1KhQgWSkpJSrhWkVqZMGUJCQoiPj+fq1at3tIeEhFCmTBlu3LjBtWvXbvkCsZorCT2tv1tuP613ZRuMMf8C/gWOM3QX+lZK5SLe3t4UKVKEZ599lmeffdbucHIcq2qKat1RpZSymVU1RZcAPcWhIXDpbuPnSimlrHfPIRdjTIKIJNcU9QamGWdNUWf7FGAFjhkuR3FMW+yVdSErpZRKi0s3FhljVuBI2qnfm5LquQH+am1oSimlMkJXW1RKKQ+hCV0ppTyEJnSllPIQmtCVUspD2LZ8roj8AZy6z48XBaIsDCcn9K3HnDv6zm392tl3Tj3mcsaYYmk12JbQM0NEdqS3loGn9q3HnDv6zm392tm3Jx6zDrkopZSH0ISulFIeIqcm9H/lwr71mHNH37mtXzv79rhjzpFj6Eoppe6UU8/QlVJK3UYTulJKeYhsndBFpI2IRIjIUREZnka7iMhEZ/teEXnITf1WE5EtIhInIkOs6DMDfT/vPNa9IrJZRMLc1G9HZ5+7RWSHiDSxol9X+k61XT0RSRSRLu7oV0Sai8gl5zHvFpFRVvTrSt+p+t8tIgdE5Cd39CsiQ1Md737nz7uIm/oOFJGlIrLHecyWrNrqQr+FRWSR8/f7ZxGpaVG/00TkgojsT6fd+vxljMmWDxxL9R4DKgJ5gD3AA7dt0w5YiaNiUkNgm5v6LQ7UA/4BDHHzMTcGCjuft3XjMRfgf9dcHgR+ddcxp9puDY5VP7u46ZibA8ts+t0uBBwEQpJ/59z1s061/ZPAGjce89+A953PiwF/Annc0O+HwGjn82rAaouO+RHgIWB/Ou2W56/sfIZeHzhqjDlujLkJzAU63rZNR2CmcdgKFBKRUlndrzHmgjFmOxCfyb7up+/NxpgY58utOKpDuaPfq8b5Wwj4k0aJwazq2+lVYAFwwc39ZgVX+n4OWGiMOQ2O3zk39Ztad2COBf262rcBCoqI4DiB+BNIIHNc6fcBYDWAMeZXoLyIZLropzFmPY5jSI/l+Ss7J/T0Ck9ndJus6DerZLTvPji+4d3Sr4h0EpFfgeVAbwv6dalvESkDdAKmYB1Xf9aNnEMAK0Wkhhv7DgUKi8g6EdkpIj3d1C8AIpIfaIPjS9QKrvT9GVAdR/nKfcBAY0wSmeNKv3uAzgAiUh8ohzUnSlbEliHZOaFbVpw6C/rNKi73LSItcCT0Ye7q1xizyBhTDXgK+LsF/bra9wRgmDEm0aI+Xe33FxzrZoQBnwLfubFvH6AO8ATQGnhLRELd0G+yJ4FNxpi7nWFa3XdrYDdQGggHPhORADf0Ow7Hl+duHH8J7iLzfxm4wvJc41LFIpvYVZzazoLXLvUtIg8CU4G2xphod/WbzBizXkQqiUhRY0xmFzdype+6wFzHX+IUBdqJSIIx5rus7NcYcznV8xUi8rkbjzkSiDLGxAKxIrIeCAMOZ3G/yZ7FuuEWV/vuBYxzDu0dFZETOMa0f87Kfp3/n3uB40IlcML5yGrW5xorBv+z4oHjy+Y4UIH/Xcyocds2T3DrRYWf3dFvqm3HYO1FUVeOOQRH7dbGbu63Mv+7KPoQcDb5tbt+3s7tp2PNRVFXjrlkqmOuD5x21zHjGHpY7dw2P7AfqOmOnzUQiGPs19/Nv2OTgTHO5yWcv2NF3dBvIZwXX4GXcIxrW3Xc5Un/oqj1+cuqwLPigeMq8GEcV6lHOt/rD/R3PhdgkrN9H1DXTf2WxPHtehm46Hwe4Ka+pwIxOP403Q3scFO/w4ADzj63AE3c9f/5tm2nY0FCd/GYX3Ee8x4cF6Ct/BK95zEDQ3HMdNkPDHJjvy8Cc6061gz8vEsDPzj/Le8HXnBTv42AI8CvwEKcs8gs6HcO8DuOyROROIZIszR/6a3/SinlIbLzRVGllFIZoAldKaU8hCZ0pZTyEJrQlVLKQ2hCV0opD6EJXeUIIhKUahXAcyJy1vn8oogczIL+xkgGV9IUkavpvD/dqhUilbobTegqRzDGRBtjwo0x4TjWdBnvfB4O3HO9DxHJzndFK2UJTejKE3iLyJfONbR/EJF8AM6FrcY61xIfKCJ1ROQn52JXq5JXthOR10TkoHNN6rmp9vuAcx/HReS15DdF5HXnWuH7RWTQ7cE417n+zLnP5TiWW05uG5eqr4+y6geicic9a1GeoArQ3RjzkojMB54GvnK2FTLGNBMRX+AnoKMx5g8ReQbHeva9geFABWNMnIgUSrXfakALoCAQISKTcawF3wtogONOv20i8pMxZleqz3UCqgK1cNzCfhCY5iwU0QmoZowxt/WlVKZpQlee4IQxZrfz+U4c62ckm+f8b1WgJvBf5yJf3jhuywbYC8wWke+4dUXF5caYOCBORC7gSM5NgEXGsWgWIrIQaIpjhb5kjwBzjGN1yN9EZI3z/cvADWCq88x92f0fslJ30iEX5QniUj1P5NYTlVjnfwU4kDwOb4ypZYx53Nn2BI41NeoAO1ONt6e137SWPE1LWssPJ+BY5GsBjiWIv3dxX0q5RBO6yi0igGIi0ghARHxFpIaIeAFljTFrgTdxrLxX4C77WQ88JSL5RcQfxxDKhjS2eVZEvJ3j9C2cfRYAAo0xK4BBOC7oKmUZHXJRuYIx5qZz6uBEEQnE8bs/AccqfF853xMcs2cuOodl0trPLyIynf+t0T31tvFzgEXAozhW0DuMY+weHGPxi0XEz9nXYIsOTykAXW1RKaU8hQ65KKWUh9CErpRSHkITulJKeQhN6Eop5SE0oSullIfQhK6UUh5CE7pSSnmI/wdib4FTrG6Z8wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(df_scores.threshold, df_scores.tpr, color='black', linestyle='solid', label='TPR')\n",
    "plt.plot(df_scores.threshold, df_scores.fpr, color='black', linestyle='dashed', label='FPR')\n",
    "plt.legend()\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "plt.yticks(np.linspace(0, 1, 11))\n",
    "\n",
    "plt.xlabel('Thresholds')\n",
    "plt.title('TPR and FPR')\n",
    "\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tpr_fpr_dataframe(y_val, y_pred):\n",
    "    scores = []\n",
    "\n",
    "    thresholds = np.linspace(0, 1, 101)\n",
    "\n",
    "    for t in thresholds:\n",
    "        tp = ((y_pred >= t) & (y_val == 1)).sum()\n",
    "        fp = ((y_pred >= t) & (y_val == 0)).sum()\n",
    "        fn = ((y_pred < t) & (y_val == 1)).sum()\n",
    "        tn = ((y_pred < t) & (y_val == 0)).sum()\n",
    "\n",
    "        scores.append((t, tp, fp, fn, tn))\n",
    "\n",
    "    df_scores = pd.DataFrame(scores)\n",
    "    df_scores.columns = ['threshold', 'tp', 'fp', 'fn', 'tn']\n",
    "\n",
    "    df_scores['tpr'] = df_scores.tp / (df_scores.tp + df_scores.fn)\n",
    "    df_scores['fpr'] = df_scores.fp / (df_scores.fp + df_scores.tn)\n",
    "\n",
    "    return df_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>636</td>\n",
       "      <td>1030</td>\n",
       "      <td>70</td>\n",
       "      <td>112</td>\n",
       "      <td>0.900850</td>\n",
       "      <td>0.901926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>568</td>\n",
       "      <td>918</td>\n",
       "      <td>138</td>\n",
       "      <td>224</td>\n",
       "      <td>0.804533</td>\n",
       "      <td>0.803853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>506</td>\n",
       "      <td>798</td>\n",
       "      <td>200</td>\n",
       "      <td>344</td>\n",
       "      <td>0.716714</td>\n",
       "      <td>0.698774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>437</td>\n",
       "      <td>693</td>\n",
       "      <td>269</td>\n",
       "      <td>449</td>\n",
       "      <td>0.618980</td>\n",
       "      <td>0.606830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>382</td>\n",
       "      <td>574</td>\n",
       "      <td>324</td>\n",
       "      <td>568</td>\n",
       "      <td>0.541076</td>\n",
       "      <td>0.502627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>312</td>\n",
       "      <td>455</td>\n",
       "      <td>394</td>\n",
       "      <td>687</td>\n",
       "      <td>0.441926</td>\n",
       "      <td>0.398424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>231</td>\n",
       "      <td>338</td>\n",
       "      <td>475</td>\n",
       "      <td>804</td>\n",
       "      <td>0.327195</td>\n",
       "      <td>0.295972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>155</td>\n",
       "      <td>241</td>\n",
       "      <td>551</td>\n",
       "      <td>901</td>\n",
       "      <td>0.219547</td>\n",
       "      <td>0.211033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>83</td>\n",
       "      <td>119</td>\n",
       "      <td>623</td>\n",
       "      <td>1023</td>\n",
       "      <td>0.117564</td>\n",
       "      <td>0.104203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn       tpr       fpr\n",
       "0          0.0  706  1142    0     0  1.000000  1.000000\n",
       "10         0.1  636  1030   70   112  0.900850  0.901926\n",
       "20         0.2  568   918  138   224  0.804533  0.803853\n",
       "30         0.3  506   798  200   344  0.716714  0.698774\n",
       "40         0.4  437   693  269   449  0.618980  0.606830\n",
       "50         0.5  382   574  324   568  0.541076  0.502627\n",
       "60         0.6  312   455  394   687  0.441926  0.398424\n",
       "70         0.7  231   338  475   804  0.327195  0.295972\n",
       "80         0.8  155   241  551   901  0.219547  0.211033\n",
       "90         0.9   83   119  623  1023  0.117564  0.104203\n",
       "100        1.0    0     0  706  1142  0.000000  0.000000"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "y_rand = np.random.uniform(0, 1, size=len(y_val))\n",
    "df_rand = tpr_fpr_dataframe(y_val, y_rand)\n",
    "df_rand[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8eklEQVR4nO3deXhMZ/vA8e+dxBJbtOGliNrXiNipLWiLWBqCWiq186pqaUu9lraquqBUo5bX/lNU7ZQoaqnal9gSuyKofV+L5/fHnHgHQdRkJpncn+uay8zZnvtMxj1nnnPOc4sxBqWUUkmfh6sDUEop5Ria0JVSyk1oQldKKTehCV0ppdyEJnSllHITmtCVUspNaEJX8SYif4rIqy6OwVtEFojIJRH5OYHa+FREpiTEth1NRIJEJMbVcTwvEVkpIu3iuawRkXwJHVNSpAndyUTkqt3jnojcsHvdwkomf1uvL4rIWhGpYK3bSkTuWvMui8h2Eanr6n0CEJGJInL7of1705r3p91+nhKRCSKSzpq3UkRuWvPOishsEXnpCU01ArIAvsaYxg6I2y0SolKgCd3pjDHpYh/AUaCe3bQfrcV+suZnBtYAs0VErHnrrHkZgR+A6SKS0bl78Vjf2O+fMeYnu3n1rLhLAmWAPnbzuljz8gHpgMFPaONlYJ8x5s6zBiciXs+6jqOJiKerY1DuSxN6ImaM+RuYBGQFfB+adw/4PyAtkD+u9UXkBRFZKCJnROSC9TyH3fyVIvK5iPwhIldE5FcRyWQ3v6WIHBGRcyLS20H7dBxYDPjHMe8iMBcIfMz+fAb0A960jujbioiHiPSx4jwtIpNFxMdaPpf187ytiBwFfntoe2mtWLLZ/arIZs1OaW3riojsFpHSdutlE5FZ1vt6WES6Pm5/rV8uI0VkkYhcA6qJSB0R2Wb9yjomIp/aLR8b89sictT61dLbbr63tc0LIhKF7cvRvr3C1t/1ohV3/Ydi+UFEFlv7+oeIZBWRYdb29ohIiSfsixGRziKy33pfPheRvCKyztqXGSKS0m759iJyQETOi8h8u/cWEXnNau+SiIQD8lBbbUQk2opriYi8/Li41P9oQk/ERCQV0AqIMcacfWieJ9Aa+Bs48phNeAATsB3V5gRuAOEPLdPc2s6/gJTAh9b2iwAjgZZANmxfKDl4TiLiBwQD2+KY5ws0BA7Eta4x5hNgINYvGGPMOGzvTyugGpAH2xH+w/tYFSgM1Hxoe9eA2sAJu18VJ6zZ9YHp2H4JzY/dpoh4AAuA7UB2oAbwvog8sO2HNAe+ANJj+8V1DQiztl0H+LeIhDy0TiWgoLX9fiJS2Jr+CZDXetQE3o5dQURSWLH9iu3v+S7wo4gUtNtuE2y/jjIBt4B1wFbr9Uzg2yfsB0AtoBRQHugBjAFaAH7YvqSbWbFUB7602nsJ22d0ujUvEzDLLo6DQEW7/QgB/oPts5AZ+B2Y9pS4FIAxRh8uegB/Aq8+NO1T4DZwETiN7aiylDWvFXDHmvc3tgTd5BnaCwQu2L1eCfSxe90ZiLCe9wOm281La8X16mO2PRG4acV2ETj70H5etaYfwdZV5G0Xw3XgEmCASCDnE/bhU2CK3evlQGe71wWt98YLyGVtM88TtheE7Qvz4TaW2b0uAtywnpcDjj60fC9gwhPel8lP+bsMA4Zaz2NjzmE3fyPQ1Hp+CKhlN69DbPxAZeAvwMNu/jTgU7tY/ms3710g2u51MeDiE+I0QEW711uAnnavhwDDrOfjsHXBxc5LZ/1dcmH7MltvN0+AGKCd9Xox0NZuvof1GXnZLo58zvg/mtQeeoSeOM0wxmQ0xvzLGFPdGLPFbt56Y0xG4AVsR46VH7cREUkjIqOt7ojLwGog40P9uH/ZPb+O7T8e2I7Kj8XOMLaj2XNPiXuwFXdGY0ymh+aFWNNfNsZ0NsbcsJvX1RjjAwRY+/UsvwSy8eAvlCPYknkWu2nHeHYPvy+prT74l7F10VyMfWA7mswSxzbibF9EyonICqvL5hLQCduR6pPaj/PvwoP7ng04Zmzdcfbzs9u9PmX3/EYcr9PxZPFd/4G/izHmKrbPT/aH98HYsrT9Pr0MfGf3/p7HlvTt90PFQRN6EmX9B+kMtHxCv+cH2I5YyxljMgBVrOnymOXtncT2M9q2gkgaHurHdzRjzE5gADBCROITI8AJbAkgVk5sv2LsE82ThhR91uFGjwGH7b64Mhpj0htjgp+hjanYvoz9rC+yUcTvbwIP/V2w7W+sE4Cf1S1kP/94PLftSA/8XazzFb5WLA9/toQH9+kY0PGh99jbGLPWOaEnXZrQkzBjzDlgLLbukbikx3bUdFFEXsTW/xpfM4G6IlLJOtHVH+d8XiZh6/+t/7QFLdOAbiKSW2yXQsb2scf3KphTgG/sidR42AhcFpGe1glKTxHxF5EyT13zf9ID540xN0WkLLY+9viaAfQS2wnvHNi6TWJtwNY/30NEUohIEFAPq+/ayaYCrUUk0DoXNBDYYIz5E/gFKCoiDa1fPV2xnfiPNQrbPhYFEBEfEXnuS1STA03oSd8wIFhEAh4zzxs4C6wHIuK7UWPMbuAdbP8xTwIXsPVzJihjzG1gONA3nquMx3a1z2rgMLZ+/HefuMaD7e3B9qVwyPqJn+0py9/FliQDrfbOYvtSje8XAth+WfUXkSvYvoxnPMO6n2HryjiM7eTn/9nFdhvbF2FtK64fgDBrH53KGLMc299wFrbPT16gqTXvLNAY+ApbN0x+4A+7decAX2O7JPcysAvbPqmnEOskg1JKqSROj9CVUspNaEJXSik3oQldKaXchCZ0pZRyEy4brChTpkwmV65crmpeKaWSpC1btpw1xmSOa57LEnquXLnYvHmzq5pXSqkkSUQeN3aTdrkopZS70ISulFJuQhO6Ukq5CZdXcFFKqefx999/ExMTw82bN10dikOlTp2aHDlykCJFinivowldKZWkxcTEkD59enLlykX8B+lM3IwxnDt3jpiYGHLnzh3v9Z7a5SIi48VW2mvXY+aLiAy3Sk3tEJGSzxC3Uko9l5s3b+Lr6+s2yRxARPD19X3mXx3x6UOfiK3s1OPUxjZaWn5s1VNGPlMESin1nNwpmcf6J/v01IRujFmNrWLI47yBrcSWMcasx1YR56VnjiSezp49S6tWrTh06FBCNaGUUkmSI65yyc6D5aNieEypKBHpICKbRWTzmTNn/lFjy5cvZ9KkSeTLl49///vfnD179ukrKaVUAjl37hyBgYEEBgaSNWtWsmfPfv+1iBAYGIi/vz+NGzfm+vXrAHh6et6fXq9ePS5evOiQWByR0OP6XRDnIOvGmDHGmNLGmNKZM8d55+pTvfnmm4wePRpPT09GjRrFyy+/zOrVq//RtpRS6nn5+voSGRlJZGQknTp1olu3bvdfp02blsjISHbt2kXKlCkZNWoUAN7e3venv/jii4wYMcIhsTgiocfwYD3AHNjqCSaYDh06EBkZSfbs2blx4wa1atVi1644z9kqpVSiULlyZQ4cOPDI9AoVKnD8uGPKvjrissX5QBcRmQ6UAy4ZY046YLtPVLRoUSIjI6lbty4bNmxg4sSJfPPNN3h46L1SSiVX77//PpGRkQ7dZmBgIMOGDXuubdy5c4fFixdTq9aD15fcvXuX5cuX07Zt2+fafqz4XLY4DVgHFBSRGBFpKyKdRKSTtcgi4BBwAPgvtnqJTpEpUyZWr15Nhw4dGDJkCHXq1GHjxo3Oal4ppZ7oxo0bBAYGUrp0aXLmzHk/ccdO9/X15fz587z22msOae+pR+jGmGZPmW+wFRN2idh+qWLFivHuu++yZMkS2rdvz9ChQ0mTJo2rwlJKucDzHkk7Wmxf+eOmX7p0ibp16zJixAi6du363O25Rf+EiNClSxcWLVpEmjRpGDNmDH5+fkyaNIm7d++6OjyllIqTj48Pw4cPZ/Dgwfz999/PvT23SOixateuzZEjRwgICOD8+fO0atWKEiVK8Ouvv7rdOA9KKfdQokQJihcvzvTp0597W2LrMXG+0qVLm4QqcHHr1i169OjBtGnT8Pb25ujRo4Dt2zB79uz06NGDsLAwt7y7TKnkJjo6msKFC7s6jAQR176JyBZjTOm4lnerI/RYqVKl4rvvvuPQoUPs27ePCRMmUK9ePd566y3Spk1Lq1atCAsL48qVK64OVSmlHMYtE3qsdOnSkSpVKlKkSMGCBQvYtWsXM2fOpH///kydOpWSJUsSFRXl6jCVUsoh3Dqhx2rRogWTJk1i8+bNlCpVikKFCvHbb79x9epVqlWrRnR0tKtDVEqp55YsEjpAWFgYW7ZsIWfOnDRp0oSIiAhWrlyJh4cH1atXZ+/eva4OUSmlnkuySegABQsWZMOGDXz77bc0a9aMggUL8sMPP3D9+nWqVq3K7t27XR2iUkr9Y8kqoQN4eXnRrVs3AgICAFi3bh2XL1/m1KlTFC9enCFDhuCqK3+UUup5JLuE/rAvv/yS9evX07FjR4wxfPjhh5QoUYLLly+7OjSlVBIROxxu7OPPP/9k5cqV+Pj4UKJECQoXLsxnn30G8MD0QoUK8eGHHzosjmSf0D09PSlXrhyjRo1i3759+Pv7s337dooWLcrChQtdHZ5SKgmIvZU/9pErVy7ANsLitm3b2Lx5M1OmTGHLli0PTN+2bRsLFy7kjz/+cEgc8UroIlJLRPZadUM/jmP+CyIyx6opulFE/B0SnZPlzZuXnTt3sm7dOnx8fKhXrx5BQUGcPJngg0cqpdxY2rRpKVWqFAcPHnxgure3N4GBgc4bPldEPIERwGvYxj7fJCLzjTH2F3D/B4g0xjQQkULW8jUcEqELlC9fnq1bt1K4cGFWrVpF9uzZKVOmDO+//z6NGzfGy8sRow4rpRJCUFDQI9OaNGlC586duX79OsHBwY/Mb9WqFa1ateLs2bM0atTogXkrV658apuxoycC5M6dmzlz5jww/9y5c6xfv56+fftiX63twoUL7N+/nypVqjx9x+IhPkfoZYEDxphDxpjbwHRsdUTtFQGWAxhj9gC5RCSLQyJ0kZQpU7Jjxw7efvtt0qdPz8aNG2nevDl58uRh7dq1rg5PKZWI2He52Cfz33//nRIlSvD666/z8ccfU7Ro0fvTAwICyJo1K3Xr1iVr1qyOCcQY88QH0AgYa/e6JRD+0DIDgW+t52WBO0CpOLbVAdgMbM6ZM6dJSnbu3Gnq1atnfHx8DGDKly9vJkyYYI4fP+7q0JRK1qKiolwdgkmbNu0j01asWGHq1KnzxOl79+41mTNnNtu2bYtzu3HtG7DZPCZfx+cIPT41Q78CXhCRSOBdYJuV1B/+8njumqKu4u/vz/z58zl+/DjfffcdO3bsoHXr1mTPnp2AgABGjBihlzsqpZ5JgQIF6NWrF19//bVDthefhP7UmqHGmMvGmNbGmEAgDMgMHHZIhIlM2rRpeeedd+jRowfe3t6ICCdPnqRLly7Ur1+fs2fPujpEpVQS0qlTJ1avXs3hw8+fMp86fK6IeAH7sJ3kPA5sApobY3bbLZMRuG6MuS0i7YHKxpiwJ203IYfPdZYzZ87wzTffMGLECG7fvo2IkCVLFqZNm0blypVdHZ5SyYIOn/s/Tz1CN8bcAboAS4BoYIYxZvdDdUULA7tFZA9QG3jvOfYhycicOTODBg3iwIEDdOjQgSVLlpAmTRqqVavG6NGjXR2eUiqZccsCF650+vRp8ufPz+XLl+nYsSPh4eF6maNSCUiP0P8n2d8p6mi+vr588MEHiAijR4+mcuXKXL161dVhKeXW3PGChH+yT5rQHczT05N+/fqxatUqMmbMyPr16ylcuDCnTp1ydWhKuaXUqVNz7tw5t0rqxhjOnTtH6tSpn2k97QtIIJUrV+bgwYOEhISwZs0agoKCWLZsGdmzZ3d1aEq5lRw5chATE/PAHZjuIHXq1OTIkeOZ1tE+dCdYuHAhzZs3J3Xq1GTLlo0ePXoQGhpKqlSpXB2aUiqJ0T50F6tbty5r167F19eX7du306JFC/z8/Bg+fDg3b950dXhKKTehCd1J/P392blzJ1999RUpUqTgzJkzvPfee/zrX/8iIiLCrfr/lFKuoQndiby8vOjZsyd79+6lf//++Pv7c/XqVWrXrk2+fPn47LPPOHLkiKvDVEolUdqH7mKXLl1i/vz5fP/992zatInUqVMze/Zsateu7erQlFKJkPahJ2I+Pj60bNmSrl274unpCUC9evWYOnWqiyNTSiU1mtATibfeeoslS5bg4+PDvXv3aNGiBUOHDnV1WEqpJEQTeiJSo0YNdu/eTWhoKADdu3fn3Xff5c6dR0YiVkqpRziqpqiPiCwQke0isltEWjs+1OTB19eXn3/+mXnz5vH+++8THh5O1apVdVhepdRTPTWh29UUrY2t1FwzESny0GLvAFHGmOJAEDBERFI6ONZkpX79+gwdOpTw8HDWrl1Ljhw5aNu2rV4Fo5R6LEfVFDVAehERIB1wnjgqFqln17lzZz7++GPu3bvH+PHjyZUrFwEBAezYscPVoSmlEpn4JPTswDG71zHWNHvh2MZEPwHsBN4zxtx7eEMi0kFENovIZncbdyGhiAhffvklly9fZuDAgfj6+rJz505KlizJp59+yq1bt1wdolIqkXBUTdGaQCSQDQgEwkUkwyMrJeGaoq6WOnVqevXqxenTp/n5559p0qQJn332Gbly5WLmzJmuDk8plQg4pKYo0BqYbRWlPoCtnmghx4So7Hl4eNCoUSOmTp3K/PnzOXPmDI0bN6ZZs2bcu/fIjyKlVDISn4S+CcgvIrmtE51NgfkPLXMUW81RRCQLUBA45MhA1aPq1avH9u3byZQpE9OnTydnzpzs3bvX1WEppVzEUTVFPwdeEZGdwHKgpzFGr7NzgqJFi3Ly5EmCg4M5fvw4hQsXpnfv3ly5csXVoSmlnEzHcnEjEyZM4Pvvv2fbtm1kzpyZVq1aERYWhr+/v6tDU0o5iI7lkky0bt2arVu3sn79evLly8eQIUMoVqwYpUuXZuvWra4OTymVwDShu6GyZcuSKlUqPDw8eOuttzh16hSVKlXSq2GUcnOa0N2QiDBv3jyqVKnClClTaNu2LSVKlKBx48Z89tlnWkxDKTelCd1NZciQgUWLFt1P4uXLl+ftt9/m008/pUuXLprUlXJDXq4OQCWcVKlSMW3aNDJnzsyZM2eYMGECWbJk4ZtvvsHDw4Phw4djG61BKeUONKG7OU9PT8LDw7l37x6enp507tyZCxcuEB4ejqenJ9988w0pU+o4akq5A+1ySQZEBE9PT27fvk3NmjUZP348efLk4bvvviNt2rQEBgbSqlUr1q5d6+pQlVLPQRN6MpIyZUp++eUXPvroI65evQpA5syZ8fb2Zv78+VSsWJFGjRpx4MABF0eqlPonNKEnM3nz5uXLL7/k2LFjTJw4ERGhY8eOHD16lE8//ZSIiAgKFy5M27ZtiY6OdnW4SqlnoHeKJnPXrl3D29sbDw8Pxo0bx4kTJzh58iQTJkzg5s2b1K9fnx49elCxYkVXh6qUwgF3isajBN1HIhJpPXaJyF0RefF5A1cJL23atHh42D4Gy5Yto1+/fmzYsIFly5bxySef8Mcff1CpUiUqVqzI3LlzdURHpRIxh5SgM8YMMsYEGmMCgV7AKmPM+QSIVyWgqVOn8tNPPxETE0OdOnWoVKkSR44c4fvvv+fkyZM0aNCAGjVqcOLEw6MnK6USA0eVoLPXDJjmiOCUc4kITZo0YePGjfj5+REcHMzp06fp0qUL+/btY8yYMWzcuJHixYuzaNEiV4erlHqIo0rQASAiaYBawKznD025yssvv8wff/zBpEmTyJ07NwC7d++mXbt2bNmyhWzZslGnTh26dOmiw/QqlYg4qgRdrHrAH4/rbtGaoklHhgwZaNasGQCbN28mMDCQV155hRMnTrBhwwa6du3KDz/8QNGiRVm8eLGLo1VKgeNK0MVqyhO6W7SmaNJUuHBhwsPD+euvv6hRowYdO3akT58+rFmzhnTp0hEcHEzjxo05ePCgq0NVKllzVAk6RMQHqArMc2yIytXSpk3LO++8Q1RUFH369GHatGmUK1eOsmXLsm3bNj777DMWLVpE4cKFef/99zl/Xs+HK+UK8boOXUSCgWGAJzDeGPNFbPk5Y8woa5lWQC1jTNP4NKzXoSddUVFR7Nu3j5CQkPvTTp48ySeffMK4cePInz8/K1euJGvWrK4LUik39aTr0PXGIvVcxo0bh4eHB61btwZg9erVBAcHkzNnTlasWEGWLFlcHKFS7kVL0KkEYYxh5syZtGnThu7du3PgwAGqVKnCokWLOHLkCDVq1OD06dOuDlOpZEMTuvrHRIQ5c+bw9ttvM3z4cPLnz0+NGjVIlSoVv/zyC4cOHSIoKIiYmBhXh6pUsqAJXT2X1KlTM3HiRI4dO8YXX3xBTEwMGTNmJCgoiIiICI4fP07FihXZu3evq0NVyu1pH7pyKGPM/SpIbdu2xcfHh//7v/8D4P/+7/+oUaMGKVKkcGWISiVp2oeunCY2mV+/fp1jx44xdOhQbt26xe3bt6lduza+vr6EhIQwd+5c1waqlBvShK4SRJo0afj111/ZsGEDNWvWvF9QI/ba9QYNGtC2bVuuXbvm4kiVch+a0FWCKlu2LD///DMHDhxgyJAhTJ48mYMHD9KgQQPGjx9PqVKl2LJli6vDVMotaEJXTpE7d266d+9OtmzZ8PLyuj/o18GDByldujSlS5dm+PDhnD171sWRKpV0aUJXLjFkyBAWLFiAr68vIsKff/7Je++9R548eRg1apQW0lDqH9CErlymbt26REVF0bVrVy5dusR3331HmTJl+Pe//021atXYt2+fq0NUKknRhK5c6sUXX2TYsGHs37+fd999l2XLltG7d28iIyPx9/enW7dunDt3ztVhKpUkOKSmqLVMkFVTdLeIrHJsmMrd5cqVCxFh7969fP3111SvXp233nqL4cOHkzdvXvr378/27dtx1X0TSiUFDqkpKiIZgR+A+saYokBjx4eqkoOCBQsyYMAA5s6dy6FDh5gzZw4VK1bkk08+ITAwkGzZstGuXTt27tzp6lCVSnQcVVO0OTDbGHMUwBijIzKpf0RE6NmzJ5MnTyYyMpKGDRvi7+9PTEwMEyZMICgoiOnTpxMQEEBwcDArV67Uo3alLI6qKVoAeEFEVorIFhEJi2tDWoJOxVfLli3Zv38/HTt2JGPGjGTPnp1WrVoxbdo0jh49yoABA9iyZQvVqlW7f6373bt3XR22Ui7lqJqiXkApoA5QE+grIgUeWUlL0KlnkDlzZkaMGEGvXr0AmD9/Ph9++CHp0qWjd+/e/Pnnn4wePZpLly7RpEkTihYtqmXwVLLmqJqiMUCEMeaaMeYssBoo7pgQlbLZsGEDQ4YMoWrVqly4cAFvb286dOhAdHQ0s2bN4syZM1SrVo3Dhw+7OlSlXMJRNUXnAZVFxEtE0gDlgGjHhqqSuy+++IIZM2awdetWatasyaVLlwDw9PSkYcOGLFu2jKtXr1KtWjWOHDni4miVcr6nJnRjzB2gC7AEW5KeYYzZLSKd7OqKRgMRwA5gIzDWGLMr4cJWyVXjxo2ZOXMm27Zto3bt2ly/fv3+vBIlSrB06VIuXbpEpUqV6Nu3L7///jt///23CyNWynl0PHSVJM2ePZu1a9cyaNCg+0P2xtqyZQvvvfce69at4969e7z44osMHDiQ9u3b4+Gh99KppE3HQ1dup2HDhgwePBgRYdWqVXzwwQdcvHgRgFKlSrFmzRrOnTvHrFmzKF68OJ06daJ69eo6nIBya5rQVZK3bt06hg4dSp48eejevTvR0bbTNxkzZqRhw4YsX76csWPHEhkZSUBAACNGjNBr15Vb0oSukryPP/6YrVu38uqrrxIeHk6RIkUIDg6+f126iNC2bVuio6OpVq0aXbp0ITQ0lAsXLrg4cqUcS/vQlVs5ffo0EydOxN/fn+Dg4Efm37t3j2+//ZZevXqRNWtWWrVqRc2aNSlfvjxeXl4uiFipZ/OkPnRN6Mqt/fTTT5QuXZq8efM+MH3jxo107979/olTHx8fQkNDCQsLo3LlynryVCVaelJUJUuXL1+ma9euVK5cmXnz5j3Qb162bFnWrFnD2bNnmTlzJiEhIcyYMYOgoCDy5s3LmDFjuHPnjgujV+rZaUJXbitDhgwsX76cjBkzEhISQo0aNdi+ffsDy7zwwguEhoYyceJETp06xY8//ki2bNno2LEjAQEBLFy4UE+gqiRDE7pya/7+/mzfvp3w8HC2b99O+fLluXz5cpzLpkmThubNm7NmzRrmzJnDnTt3qFevHq1bt+bWrVtOjlypZ6cJXbm9FClS8M4773DgwAGmTJlChgwZAGjUqBHjx49/5AhcRAgJCWH37t3069ePSZMmUb16dU6dOuWK8JWKN03oKtmI7V4BuHnzJqdOnaJt27YEBQWxZ8+eR5ZPkSIFn332GTNmzGDbtm2UKVOGlStXOjlqpeJPE7pKllKnTs2qVasYO3YsO3fupHjx4kyaNCnOZRs3bsyaNWvw8PCgWrVq1KtX7/7NS0olJg6pKWrVE71k1RSNFJF+jg9VKcfy8PCgbdu27Nmzh8qVK9O7d2+uXr0a57IlS5YkOjqar776itWrV+Pv789rr73G4MGD2blzp544VYnCU69Dt2qK7gNewzbu+SagmTEmym6ZIOBDY0zd+Das16GrxOTWrVscP36cPHnycO/ePYDHXot+5swZvv32WxYsWMDu3bsBqFatGmPGjCFfvnxOi1klT897HXp8aooqlaSlSpWKPHnyYIzhgw8+ICgoiKioqDiXzZw5M19++SW7du3i2LFjDBs2jC1btlCsWDEGDRqk168rl3FUTVGACiKyXUQWi0jRuDakNUVVUhAQEMCuXbsIDAykb9++3Lhx47HL5siRg/fee4/o6Ghq1apFjx49eOONN/QyR+USjqopuhV42RhTHPgemBvXhrSmqErsRITWrVuzZ88emjZtyoABAyhUqBCrVq164nrZsmVj9uzZjBw5kkWLFtGoUSNN6srpHFJT1Bhz2Rhz1Xq+CEghIpkcFqVSTvavf/2LyZMns2LFCrJmzcpLL7301HVEhE6dOjFy5EgWLlxIkyZNuH37thOiVcrGITVFRSSrWGVjRKSstd1zjg5WKWcLCgpi/fr1FChQAGMM7dq1Y9asWU9cp1OnToSHhzN//nyKFSvGwIEDOXr0qJMiVsmZQ2qKAo2AXSKyHRgONDV6HZdyE7El7q5cucLu3btp1KgR//nPf+6Ptx6Xd955h1mzZpE1a1Z69+5Nrly5ePPNN/nrr7+cFbZKhnT4XKWewa1bt+jatStjxowhNDSUqVOnkjJlyieuc/jwYcaNG8fgwYPx9vZm8ODBtGnT5pFaqErFhw6fq5SDpEqVitGjRzNkyBBmzZpF69atn7pO7ty5GTBgADt27CAgIIB27drx6quvcvDgQSdErJITTehK/QPdu3dn1KhRdOzYMd7rFChQgBUrVjBq1Cg2b95MsWLFGDx4sF63rhxGu1yUcoCePXuSIUMG/v3vf/Piiy8+dfnjx4/TuXNn5s+fT5UqVZg3bx4ZM2ZM+EBVkqddLkoloHv37rFnzx769OmDn58fX3/99VPHdsmePTtz585l0qRJrFu3jqCgID1hqp6bJnSlnpOHhwfz5s1jx44dvP7663z88cc0btyYK1euPHE9ESEsLIyFCxeyf/9+KlWqxKFDh5wUtXJHmtCVcpBixYoxe/ZsBg8ezMqVK+NdEOP111/nt99+48KFC5QvX54//vgjgSNV7koTulIOJCJ88MEHHDx4kHz58mGMoU+fPhw4cOCJ65UrV461a9fi4+NDtWrVmDBhgpMiVu5EE7pSCcDHxweAqKgovv32WwoVKkTnzp25ePHiY9cpWLAgGzZsoEqVKrRp04b//Oc/TopWuQtN6EoloKJFi3Lo0CE6derE6NGjKVy4MDNmzHjsSdMXX3yRxYsX065dO7788ktGjhzp5IhVUqaXLSrlJFu2bKFDhw5cuHCBqKgoUqdO/dhl7969S/369VmyZAkRERG8+uqrToxUJWZPumxRE7pSTnTnzh2OHTtG7ty5uX37NkePHn1slaPLly9TsWJFYmJi2LBhAwUKFHBytCoxeu7r0J9WU9RuuTIicldEGv3TYJVyZ15eXuTOnRuAPn36ULJkSWbOnBnnshkyZGDBggWkSJGCSpUqMXDgQC5cuODMcFUS89SEbtUUHQHUBooAzUSkyGOW+xrbqIxKqad49913KVSoEI0bNyYkJIRjx449skyuXLlYunQpJUuWpHfv3vj5+dG7d2/+/vtvF0SsEjtH1hR9F5gFnHZgfEq5LT8/P/744w+++eYbfv31V4oUKcLs2bMfWa548eJEREQQGRlJ/fr1GThwILVr1+b8+fMuiFolZg6pKSoi2YEGwKgnbUhriir1oBQpUvDRRx+xe/duatSoQdGitnK89+7de2TZ4sWLM3XqVCZOnMjvv/9OuXLliI6OdnbIKhFzVE3RYUBPY8zjR/xHa4oq9Ti5c+dm7ty5FCxYEICmTZvSrVs3rl+//siyb7/9NitWrODy5cuUKVOG77///onFNlTy4ZCaokBpYLqI/ImtetEPIhLiiACVSm7u3LmDr68vw4YNo2TJkmzYsOGRZV555RU2b95MpUqV6Nq1K5UrVyYqKsoF0arExCE1RY0xuY0xuYwxuYCZQGdjzFxHB6tUcuDl5cXIkSNZunQp169f55VXXuHzzz9/5GYkPz8/Fi9ezOTJk9m7dy/FihWjadOmbNmyxUWRK1dzVE1RpZSDvfrqq+zcuZOmTZsSHh4e52BfIkLLli2Jjo7mww8/ZPHixZQuXZrXXnsNvc8j+dEbi5RK5IwxnDhxguzZs2OMeWIt0kuXLjFmzBgGDRrEmTNnaN68OV988QW5cuVyXsAqQWmBC6WSMBG5n8x79erFJ5988tiToD4+Pnz00UccOHCA3r17M3v2bIoWLapD8iYTmtCVSiKMMZw8eZL+/ftToUIFIiMjH7tshgwZGDBgAPv27SNHjhzUrVuX3bt3Oy9Y5RKa0JVKIjw8PJg4cSJTp07lyJEjlC5dmm7duj3xBiM/Pz+WLFmCt7c3NWvW5OjRo06MWDmbJnSlkhARoVmzZuzZs4e2bdsyevRorl279sR1cuXKRUREBFeuXKFmzZpxDjGg3IMmdKWSoBdeeIHRo0dz5MgR/Pxst4k0adKEYcOGcfv27UeWDwgIYMGCBRw/fpyyZcvGeW27Svo0oSuVhMXecX3jxg0uXbpEt27d8Pf3Z968eY9ct16lShXWrVtHmjRpqFq1KlOmTHFFyCoBaUJXyg14e3sTERHBL7/8goeHByEhIRQrVuyRu0eLFi3Khg0bKF++PC1btqROnTp6stSNaEJXyk2ICMHBwezcuZPJkyfj6+tLzpw5Adi3bx+3bt0CIFOmTCxdupRBgwaxdu1aAgICaN++PWfPnnVl+MoB9MYipdzc3bt3KVKkCDdu3KBv3760adMGT09PAM6dO8eAAQMIDw8nY8aMDB8+nKZNmz7x5iXlWnpjkVLJmIeHB+Hh4WTLlo0OHTo8cA27r68vQ4cOZdu2beTJk4fmzZtTr149vRImiXJICToReUNEdohIpDXeeSXHh6qU+idEhNdee41169Y9cA37unXr7i/j7+/P2rVrGTp0KCtWrKBIkSKMGDEiznHZVeL11C4Xq7TcPuA1bEPpbgKaGWOi7JZJB1wzxhgRCcA2gFehJ21Xu1yUco0LFy4watQoevbsiYfHo8d0hw8fpmPHjixdupSKFSsyduxYChV64n9n5UTP2+Xy1BJ0xpir5n/fDGl5tACGUiqReOGFF+jVqxceHh4cPnyYDz744IGxYXLnzs2SJUuYNGkSUVFRFC9enAEDBsR5fbtKXBxSgg5ARBqIyB7gF6CNY8JTSiWkRYsW8e2339KmTRtu3Lhxf7qIEBYWRnR0NA0aNKBv376ULl2ahQsXajdMIuaoEnQYY+ZY3SwhwOdxbkhriiqVqLzzzjv079+fyZMn8/LLL9O/f/8HLl/MkiUL06dPZ968eVy+fJl69epRrFgxJkyYwN9//+3CyFVcHFWC7j5jzGogr4hkimOe1hRVKpHp27cvK1eupFy5cnzyySdMnjz5kWXq16/P/v37mTJlCl5eXrRp04aiRYsyZ86cR+5IVa7jkBJ0IpJPrAtXRaQkkBI45+hglVIJo2rVqixYsICoqCjat28PwOTJkwkNDWX9+vUApEiRghYtWhAZGcmCBQtIkSIFDRs2pHLlyhw8eNCV4SuLo0rQhQK7RCQSGAG8afRrW6kkp3DhwqRPnx6Aq1ev8ttvv1GhQgUqVarEvHnzuHfvHiJC3bp12b59O6NHjyYqKor69etz9epVF0ev9E5RpdRjXb16lfHjxzN06FD+/PNPQkNDmTlz5gPLLFu2jJo1a9KkSROmTp2qd5kmML1TVCn1j6RLl46uXbuyf/9+pk2bRrt27QC4fv06c+bMAWzFrD///HOmT59OeHi4K8NN9jShK6WeysvLi6ZNm1KrVi0ARowYQcOGDWnTpg3Xrl3j448/pl69enTv3p2IiAgXR5t8aUJXSj2zbt260adPHyZOnEiZMmWIiopi8uTJFCpUiODgYPr06cOdO3dcHWayowldKfXMvLy8+Pzzz1m6dCnnz5+nTJkyrFixgvXr19O6dWu++OILgoKCiImJcXWoyYomdKXUP1ajRg22b99OcHAwxYoVI23atPz3v//lxx9/ZPv27VSoUIHo6GhXh5lsaEJXSj2XLFmyMGvWLPLlywdAnTp1mD9/PkOGDOH27dtUrlyZTZs2uTjK5EETulLKYe7du0ehQoX49ddf6dixI7lz58bb25tq1aoxcuRILl686OoQ3ZomdKWUw3h4eDB06FCOHz/OyJEjiYqK4uLFi/j5+dG5c2eyZs1K06ZN2bJli6tDdUua0JVSDuft7U2nTp3YuXMnjRs3ZtOmTWzcuJG2bduydOlSypcvz5AhQ3TkRgfTO0WVUk5x48YNypUrR0hICDt27GDevHnUrl2bSZMmoYP1xZ/eKaqUcrnr169ToEABPv/8c3bt2kVYWBjLly+nevXqXLlyxdXhuQVH1RRtYdUU3SEia0WkuONDVUolZb6+vsycOZNffvmF9OnTM3nyZLJkyUJUVBTNmzd/oGqS+meemtCtmqIjgNpAEaCZiBR5aLHDQFVjTAC24hZjHB2oUso9BAcHs2XLFmbOnMkbb7zBd999x8KFC/n440eOFdUz8orHMvdrigKISGxN0ftFoo0xa+2WX4+tCIZSSsXJw8OD0NBQQkNDAdi8eTODBw/GGEOzZs0oUaJEnAWs1ZM5rKaonbbA4rhmaAk6pVRcihe39dIOGTKE0qVLkzVrVrp3765968/IYTVFAUSkGraE3jOu+VqCTikVl27dujF58mReeuklwFakeujQofj7+7N4cZzHhyoODqspKiIBwFjgDWOMlp9TSj2Tli1bcujQIYYNG4aIEBoaStq0aQkODqZ+/frMmzeP27dvuzrMRO2p16GLiBewD6gBHMdWY7S5MWa33TI5gd+AsIf60x9Lr0NXSj3O1atXMcaQMmVKunTpwk8//cSVK1fIlCkTnTp1om/fvqRMmdLVYbrEc12HHs+aov0AX+AHEYkUEc3USql/LF26dKRPn55UqVJx69Ytrly5gpeXF15eXgwYMICKFStqYeo46J2iSqlEzRjD5s2bmT59Oj/99BMnT57E29sbDw8Pxo4dS5MmTVwdolPpnaJKqSRLRChTpgxDhgxhz549vPLKK3z99dcULVqUN998k969e+uYMBY9QldKJSn37t3Dw8OD27dv07FjRyZOnMgbb7zBlClTSJcunavDS3B6hK6UchuxNxz98ssvzJo1i6CgIObPn0+ZMmVYtGgRrjpITQw0oSulkqQyZcoQGhrK6tWrSZUqFadOnaJOnTq8+uqrbN261dXhuYQmdKVUkpQjRw4mTJhAdHQ0ISEhXLhwgfz58xMZGUmpUqUICwvj6NGjrg7TqTShK6WStAIFCjBt2jRmzpxJ//79OXjwID179mTGjBkUKFCAnj178tdff7k6TKfQk6JKKbczZcoUrl27xpo1a/jxxx9JmTIlYWFhfPjhhxQoUMDV4T0XPSmqlEo27t69yw8//MD7779P586d2bNnD61atWLy5MkUK1aMRYsWuTrEBKMJXSnlVjw9PZk3bx45cuSgXr16AIwaNYo///wTf39/GjRoQEREhIujTBia0JVSbidz5swsXrwYEeHVV19lyZIlZM2alaVLl1KkSBFCQkL49ddfXR2mw2lCV0q5pXz58hEREUHatGk5dOgQAC+++CLLli2jUKFCvPHGGyxbtszFUTqWo2qKFhKRdSJyS0Q+dHyYSin17EqVKsWOHTto3749AGPHjqV3795MmzaN/PnzU79+fX777TcXR+k4jqopeh7oCgx2eIRKKfUcUqRIgZeXrdrmiRMnGDduHOXLlyc0NJTcuXNTt25dVq1a5eIoHSM+R+j3a4oaY24DsTVF7zPGnDbGbAL+ToAYlVLKIfr168euXbuoXLkyn376KVmyZMHPz4/atWvz+eefc+3aNVeH+FwSoqboY2lNUaWUqxUsWJCFCxcyevRo1q5dS79+/ahduzb9+vWjQIECjB8/PsmO3ujQmqJPozVFlVKJRYcOHTh48CAtWrRg1qxZ/PTTT/j5+dG2bVuqVq3Knj17XB3iM3NYTVGllEpqsme3dTZs3LiRFi1aUKVKFcaOHcvu3bspXrw4AwYM4OLFi64N8hnEJ6FvAvKLSG4RSQk0BeYnbFhKKeU8gYGBtG/fnkGDBjF9+nRWrVpFSEgIffv2JWvWrLz55ptJYmheh9QUFZGsIhIDdAf6iEiMiGRIyMCVUspRUqZMyQ8//MC4ceNYs2YNQUFBNG7cmE2bNtGhQweWL19OnTp1aNGiBTdu3HB1uI+lg3MppZSdPXv2EBYWRsOGDfn4Y9ttN7dv32bQoEH06dOHMmXKMHfuXLJly+aS+J40OJcmdKWUesidO3cQETw9Pdm/fz958uTB09OTuXPn8tZbb+Hj48PPP//MK6+84vTYdLRFpZR6Bl5eXnh6enLs2DFKlizJe++9hzGGkJAQ1q5dS6pUqahSpQpfffVVorrEURO6Uko9hp+fH506dWLEiBEMHmy7ET4gIIBt27YRGhpKr169qFWrFidOJI4L/zShK6XUE3z99de8+eab9OjRg4EDB2KMwcfHh+nTpzNmzBjWrFlD4cKFGTNmjMuP1jWhK6XUE3h4eDBp0iSaN29O7969mThxIgAiQvv27dmxYwelSpWiY8eOVK9enZiYGNfF6rKWlVIqiUiVKhVTpkxh2rRptGzZEoBTp04BtmF6ly9fzn//+1+2bt1KUFCQy5K6JnSllIoHEaFp06Z4eXlx9epVihUrxuuvv8769esREdq1a8fSpUs5ffo01atXd0m/uiZ0pZR6Rl5eXvTs2ZNt27ZRoUIFatSoQUREBGXLliUiIoKTJ09SrVo1Tp486dS4NKErpdQzSp06NR988AGHDx/mm2++Yc+ePdSuXZuNGzfyyiuvEBERwfHjx6levTp//fWX0+LShK6UUv9QunTp+Oijjzh8+DBz586lbNmygK17ZsGCBRw9epQaNWpw+vRpp8SjCV0ppZ5TypQpeeONNxARDh48SNWqVenfvz/jxo3j8OHDVK9e3SlJ3VE1RUVEhlvzd4hISceHqpRSiV/evHkZN24cmzZtonXr1jRo0ICDBw9Srlw5du3alaBtO6qmaG0gv/XoAIx0cJxKKZVkhIWFER0dTWhoKFOnTuWll17i5s2bVKhQgQULFiRYuw6pKWq9nmxs1gMZReQlB8eqlFJJhp+fH1OmTGH9+vV06NCBzZs3U6hQIerXr8/w4cMTpE2veCwTV03RcvFYJjvwwDU7ItIB2xE8OXPmfNZYlVIqySlXrhzlytlS5tKlS6lVqxb58+dPkLbik9DjU1M0XnVHjTFjgDFgGz43Hm0rpZTbyJgxI+vXr0+w7TuqpqjWHVVKKRdzVE3R+UCYdbVLeeCSMca5t0gppVQy99QuF2PMHRGJrSnqCYyPrSlqzR8FLAKCgQPAdaB1woWslFIqLvHpQ8cYswhb0rafNsruuQHecWxoSimlnoXeKaqUUm5CE7pSSrkJTehKKeUmNKErpZSbENv5TBc0LHIGOPIPV88EnHVgOEmhbd3n5NF2cmvXlW0n1X1+2RiTOa4ZLkvoz0NENhtjSientnWfk0fbya1dV7btjvusXS5KKeUmNKErpZSbSKoJfUwybFv3OXm0ndzadWXbbrfPSbIPXSml1KOS6hG6Ukqph2hCV0opN5GoE7qrilPHo91CIrJORG6JyIeOaPMZ2m5h7esOEVkrIsWd1O4bVpuRIrJZRCo5ot34tG23XBkRuSsijZzRrogEicgla58jRaSfI9qNT9t27UeKyG4RWeWMdkXkI7v93WW93y86qW0fEVkgItutfXbIqK3xaPcFEZljfb43ioi/g9odLyKnRSTOytAJkr+MMYnygW2o3oNAHiAlsB0o8tAywcBibBWTygMbnNTuv4AywBfAh07e51eAF6zntZ24z+n43zmXAGCPs/bZbrnfsI362chJ+xwELHTRZzsjEAXkjP3MOeu9tlu+HvCbE/f5P8DX1vPMwHkgpRPaHQR8Yj0vBCx30D5XAUoCux4z3+H5KzEfobuqOPVT2zXGnDbGbAL+fs62/knba40xF6yX67FVh3JGu1eN9SkE0hJHicGEatvyLjALOO3kdhNCfNpuDsw2xhwF22fOSe3aawZMc0C78W3bAOlFRLAdQJwH7jih3SLAcgBjzB4gl4hkec52McasxrYPj+Pw/JWYE/rjCk8/6zIJ0W5Ceda222L7hndKuyLSQET2AL8AbRzQbrzaFpHsQANgFI4T3/e6gtUFsFhEijqx7QLACyKyUkS2iEiYk9oFQETSALWwfYk6QnzaDgcKYytfuRN4zxhzzwntbgcaAohIWeBlHHOg5IjYnkliTugOK06dAO0mlHi3LSLVsCX0ns5q1xgzxxhTCAgBPndAu/FtexjQ0xhz10FtxrfdrdjGzSgOfA/MdWLbXkApoA5QE+grIgWc0G6sesAfxpgnHWE6uu2aQCSQDQgEwkUkgxPa/Qrbl2cktl+C23j+Xwbx4fBcE6+KRS7iquLUrix4Ha+2RSQAGAvUNsacc1a7sYwxq0Ukr4hkMsY87+BG8Wm7NDDd9kucTECwiNwxxsxNyHaNMZftni8SkR+cuM8xwFljzDXgmoisBooD+xK43VhNcVx3S3zbbg18ZXXtHRCRw9j6tDcmZLvW37k12E5UAoetR0JzfK5xROd/QjywfdkcAnLzv5MZRR9apg4PnlTY6Ix27Zb9FMeeFI3PPufEVrv1FSe3m4//nRQtCRyPfe2s99tafiKOOSkan33OarfPZYGjztpnbF0Py61l0wC7AH9nvNeAD7a+37RO/oyNBD61nmexPmOZnNBuRqyTr0B7bP3ajtrvXDz+pKjj85ejAk+IB7azwPuwnaXubU3rBHSyngswwpq/EyjtpHazYvt2vQxctJ5ncFLbY4EL2H6aRgKbndRuT2C31eY6oJKz/s4PLTsRByT0eO5zF2uft2M7Ae3IL9Gn7jPwEbYrXXYB7zux3VbAdEft6zO839mAX63/y7uAt5zUbgVgP7AHmI11FZkD2p0GnMR28UQMti7SBM1feuu/Ukq5icR8UlQppdQz0ISulFJuQhO6Ukq5CU3oSinlJjShK6WUm9CErpIEEfG1GwXwLxE5bj2/KCJRCdDep/KMI2mKyNXHTJ/oqBEilXoSTegqSTDGnDPGBBpjArGN6TLUeh4IPHW8DxFJzHdFK+UQmtCVO/AUkf9aY2j/KiLeANbAVgOtscTfE5FSIrLKGuxqSezIdiLSVUSirDGpp9ttt4i1jUMi0jV2ooh0t8YK3yUi7z8cjDXOdbi1zV+wDbccO+8ru7YGJ9QbopInPWpR7iA/0MwY015EZgChwBRrXkZjTFURSQGsAt4wxpwRkTexjWffBvgYyG2MuSUiGe22WwioBqQH9orISGxjwbcGymG702+DiKwyxmyzW68BUBAohu0W9ihgvFUoogFQyBhjHmpLqeemCV25g8PGmEjr+RZs42fE+sn6tyDgDyy1BvnyxHZbNsAO4EcRmcuDIyr+Yoy5BdwSkdPYknMlYI6xDZqFiMwGKmMboS9WFWCasY0OeUJEfrOmXwZuAmOtI/eF/3yXlXqUdrkod3DL7vldHjxQuWb9K8Du2H54Y0wxY8zr1rw62MbUKAVssetvj2u7cQ15Gpe4hh++g22Qr1nYhiCOiOe2lIoXTegqudgLZBaRCgAikkJEioqIB+BnjFkB9MA28l66J2xnNRAiImlEJC22LpTf41imqYh4Wv301aw20wE+xphFwPvYTugq5TDa5aKSBWPMbevSweEi4oPtsz8M2yh8U6xpgu3qmYtWt0xc29kqIhP53xjdYx/qPweYA1THNoLePmx992Dri58nIqmttro5aPeUAtDRFpVSyl1ol4tSSrkJTehKKeUmNKErpZSb0ISulFJuQhO6Ukq5CU3oSinlJjShK6WUm/h/kwfEm1MSlHgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(df_rand.threshold, df_rand.tpr, color='black', linestyle='solid', label='TPR')\n",
    "plt.plot(df_rand.threshold, df_rand.fpr, color='black', linestyle='dashed', label='FPR')\n",
    "plt.legend()\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "plt.yticks(np.linspace(0, 1, 11))\n",
    "\n",
    "plt.xlabel('Thresholds')\n",
    "plt.title('TPR and FPR for the random model')\n",
    "\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ideal baseline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>706</td>\n",
       "      <td>1142</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>706</td>\n",
       "      <td>957</td>\n",
       "      <td>0</td>\n",
       "      <td>185</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.838004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>706</td>\n",
       "      <td>772</td>\n",
       "      <td>0</td>\n",
       "      <td>370</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.676007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>706</td>\n",
       "      <td>587</td>\n",
       "      <td>0</td>\n",
       "      <td>555</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.514011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>706</td>\n",
       "      <td>403</td>\n",
       "      <td>0</td>\n",
       "      <td>739</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.352890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>706</td>\n",
       "      <td>218</td>\n",
       "      <td>0</td>\n",
       "      <td>924</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.190893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>706</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>1109</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.028897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>555</td>\n",
       "      <td>0</td>\n",
       "      <td>151</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.786119</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>370</td>\n",
       "      <td>0</td>\n",
       "      <td>336</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.524079</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>185</td>\n",
       "      <td>0</td>\n",
       "      <td>521</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.262040</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>705</td>\n",
       "      <td>1142</td>\n",
       "      <td>0.001416</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn       tpr       fpr\n",
       "0          0.0  706  1142    0     0  1.000000  1.000000\n",
       "10         0.1  706   957    0   185  1.000000  0.838004\n",
       "20         0.2  706   772    0   370  1.000000  0.676007\n",
       "30         0.3  706   587    0   555  1.000000  0.514011\n",
       "40         0.4  706   403    0   739  1.000000  0.352890\n",
       "50         0.5  706   218    0   924  1.000000  0.190893\n",
       "60         0.6  706    33    0  1109  1.000000  0.028897\n",
       "70         0.7  555     0  151  1142  0.786119  0.000000\n",
       "80         0.8  370     0  336  1142  0.524079  0.000000\n",
       "90         0.9  185     0  521  1142  0.262040  0.000000\n",
       "100        1.0    1     0  705  1142  0.001416  0.000000"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_neg = (y_val == 0).sum()\n",
    "num_pos = (y_val == 1).sum()\n",
    "\n",
    "y_ideal = np.repeat([0, 1], [num_neg, num_pos])\n",
    "y_pred_ideal = np.linspace(0, 1, num_neg + num_pos)\n",
    "\n",
    "df_ideal = tpr_fpr_dataframe(y_ideal, y_pred_ideal)\n",
    "df_ideal[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+6ElEQVR4nO3de5zMdf//8cdr165dx3U+pM1xSWJz6IpKdV0KG5EkRVk6I6dsclpnyoqcvild8auulA5UIqGLTlSUc7gil5DKIiuE3ffvj5l1rT3O7n5m3jOzr/vtNjcz8/7M5/38zK73fubz+cz7JcYYlFJKBb4Q2wGUUko5Qwd0pZQKEjqgK6VUkNABXSmlgoQO6EopFSR0QFdKqSChA7pylIjsF5E2ljNEisiHIvKHiLztpT7GisjrDq1rnoiMzqXdiEhdB/pZKyIPFXY9Bej3ZhE56OGyjr2vRZEO6H5IRE5luKWJyJkMj3u4f+nPux+fEJGvRKSl+7XxIpLqbjspIltEpIPtbQIQkYUici7T9t3jbtufYTt/FZEFIlLK3bZWRM66246KyHsiUi2XrroCVYAKxpi7Hcjt8YBUEMaYx4wxE7y1flV06IDuh4wxpdJvwAGgY4bn/uVe7C13eyXgC+A9ERF323p3WxTwf8CbIhLl263I0dSM22eMeStDW0d37qZAC2BUhrb+7ra6QClgWi59XAHsMcZcyG84ESmW39co5S90QA9wxpjzwP8DqgIVMrWlAa8BJYF62b1eRMqJyDIR+V1Ejrvv18jQvlZEJojIlyKSIiKfiEjFDO33i8h/RSRZREY6tE2HgBVAo2zaTgBLgdgctmcckAjc496jf1BEQkRklDvnbyLyqoiUdS9f031I40EROQB8mml9Jd1Zqmf4VFHd3RzuXleKiOwQkeYZXlddRN51v68/iciAnLbX/cllYobHCSLyi4gcFpE+mZYtLiLTROSA+5PMPBGJdLfl+rPMjftT39si8rp7e7aJSIyIDHe/Zz+LyG2Ztu8DETkmIj+KyMMZ2iLd23RcRHbi+uNMptd69N6o/NEBPcCJSHEgHjhojDmaqS0U6A2cB/6bwypCgAW49mqjgTPAnEzL3OdeT2UgHBjqXn9D4AXgfqA6rj8oHg0geWzT5UAc8H02bRWALsCP2b3WGDMGmIz7E4wx5p+43p944BagNq49/MzbeBNwJdA20/r+BNoDhzN8qjjsbr4DeBPXJ6EP0tcpIiHAh8AW4DLgH8AgEblk3Tlseztc7++tuP4IZz4f8SwQg+sPWl33+hPdbZ78LHPTEdcOQDlc7/1K9zovA8YDL2ZYdhFwENfPvSswWUT+4W4bA9Rx39oCvTJsX4HfG+UBY4ze/PgG7AfaZHpuLHAOOAH8hmuvspm7LR644G47j+s/dbd89BcLHM/weC0wKsPjvsDH7vuJwJsZ2kq6c7XJYd0LgbPubCeAo5m285T7+f/iOlQUmSHDaeAPwACbgehctmEs8HqGx2uAvhke13e/N8WAmu511s5lfTfj+oOZuY/VGR43BM647/8NOJBp+eHAglzel4nu+68Az2Roi3HnqwsI8CdQJ0N7S+CnfPwsH8rlPVuV4XFH988j1P24tDtHFHA5kAqUzrD8FGCh+/4+oF2GtkfS37+83pvMPzu95e+mxwsD12JjTM8c2jYYY25wn1T8J3AjsDi7BUWkBDADaIdrzwygtIiEGmNS3Y+PZHjJaVx7uODaO/s5vcEY86eIJOeRe5oxZlQObZ2NMatzaBtgjHlZRK4GluH6JHAgj77SVefSTyj/xTWYV8nw3M/kX+b3JcJ9DP4KXIdoTmRoDwU+9zDrpkxZ01UCSgCb/ne6BHGv29OfZW5+zXD/DK4/uKkZHoPrZ18dOGaMScmUM/2Q0yW/F5m2oTDvjcqDHnIJYsaYU7j2qO8XkWtyWOxJXHusfzPGlAFau5+XHJbP6Bdce2uuF7gGlAo5L154xphtwERgboaTwHk5jGsgSReN61NMxgEst2lH8zsl6c+49pqjMtxKG2PiPHjtJe+pO2u6o7gG1qsyrLescZ0shsL9LPPjMFBeREpnynnIg20ozHuj8qADepAzxiQDL/O/46yZlcY1SJwQkfK4jn966h2gg4jcICLhuI6z+uJ36v/hOp5/h4fLLwIGi0gt96eW9GPsnl4F8ytQIf1Eqge+AU6KyDD3CcJQEWkkIi3yfKXrk1S8iDR0/4G8+PMwrpPc84EZIlIZQEQuy3D8uTA/S48ZY34GvgKmiEiEiDQGHgTSr8BaDAx3n6StATyR4eWFeW9UHnRALxqeB+Lc//Gya4vEtfe3AfjY05UaY3YA/YA3cO2VHcd1osyrjDHngFlAjl/GyeQVXCf7PgN+wnUc/4lcX3Fpf7tw/VHYJ67r/qvnsXwqrmPQse7+juL6o5rnHwRjzApcP5NPcZ34/TTTIsPcz28QkZPAalx75VCIn2UB3Ivr/MNhYAkwxhizyt02Dtdhlp+AT3C990Dh3huVN3GfiFBKKRXgdA9dKaWChA7oSikVJHRAV0qpIKEDulJKBQlrXyyqWLGiqVmzpq3ulQooKSkplC5dOu8FVdDbtGnTUWNMpezarA3oNWvWZOPGjba6Vyqg/Pzzz1x++eV5L6iCnojkNC+THnJRKhAkJ+c1o4JSHgzoIvKKe/rM7Tm0i4jMck+huVVEmjofU6mi7cSJE7YjqADgyR76QlyT/eSkPa5pPuvhmlXthcLHUkpl1KxZM9sRVADI8xi6MeYzEamZyyKdgFeN6yunG0QkSkSqGWN+cSpkRkePHmXXrl3eWLVSfuXyyy/niitcc4pt2rSJm2++2W4gP3X+/HkOHjzI2bNnbUdxVEREBDVq1CAsLMzj1zhxUvQyLp0q86D7Oa8M6GvWrKF79+7eWLVSfuemm26iT58+1KuXbcEpBRw8eJDSpUtTs2ZNPJ+A078ZY0hOTubgwYPUqlXL49c5MaBn9w5mO0GMiDyC67AM0dHR2S2SpxtuuIHrrruOb775hsTERK6//voCrUcpf2aM4dtvv+WVV16hV69e1K1bl23bthEREWE7mt85e/ZsUA3mACJChQoV+P333/P3Qk+qYOCaVW17Dm0vAvdmeLwbqJbXOps1a2YK6s8//zR/+9vfTEREhPniiy8KvB6l/F1aWppZtGiRAczkyZNtx/FLO3futB3Ba7LbNmCjyWFcdeKyxQ+AB9xXu1wH/GG8dPw8XYkSJVi2bBnR0dF07NiRHTt2eLM7pawREbp37067du2YNGkShw4dyvtFqsjy5LLFRcB6oL6IHBRXdfTHROQx9yLLcdUQ/BHX5Pt9vZY2g4oVK/Lxxx8TFRXF/v37fdGlUtY8+uijnD9/nqefftp2FJVJcnIysbGxxMbGUrVqVS677LKLj0WE2NhYGjVqxN13383p06cBCA0Nvfh8x44dHbss1dp86M2bNzdOfFP03LlzhIeHA66z3fk5I6xUoFi7di2ffPIJU6ZM4auvvqJly5a2I/mNH374gSuvvNJ2DADGjh1LqVKlGDp0KAClSpXi1KlTAPTo0YNmzZoxZMiQS57v1asXMTExjBw5Msv6sts2EdlkjGmeZWGC4Jui6YP5v/71L5o2bcrRo0ctJ1LKec2aNWPEiBFUr16dAQMGkJaWZjuSyqcbb7yRH3/8McvzLVu2dOxQmrW5XJxWo0YN/vOf/9CxY0fWrFlDiRIlbEdSyjHp16FPmTKFXr168eabb3LffffZjuV3Bg0axObNmx1dZ2xsLM8//3yh1nHhwgVWrFhBu3aXfkczNTWVNWvW8OCDDxZq/ekCfg893U033cQbb7zB119/zb333suFC57W/1XK/1WoUAGAnj17Ehsby4gRI4LuizTB6MyZM8TGxtK8eXOio6MvDtzpz1eoUIFjx45x6623OtJf0OyhA3Tp0oXZs2fTv39/+vXrx7x584Lq2lRVdKVPnRsSEsK0adNo06YNc+bMuXisVrkUdk/aaZGRkdl+Ykh//o8//qBDhw7MnTuXAQMGFLq/oNlDT9evXz+GDx9OtWrVbEdRyjEZr+T6xz/+Qfv27Zk0aRLHjh2zF0oVWtmyZZk1axbTpk3j/PnzhV5f0A3oAJMmTWLs2LGIiP7Cq6CQ+UqHqVOncvLkSSZOnGgpkXLKNddcQ5MmTXjzzTcLva6gHNDTD7Ps3r2bmJgY5s+fbzmRUoWT+bsWjRo1ok+fPsyZM4c9e/bYCaWyGDt27CWHwdIvTcws8/Mffvgh999/f6H7D8oBPV2tWrW49tprefTRR3n77bdtx1GqwM6cOZPluQkTJhAREcGQIUMsJFL+KKgH9PDwcN59912uv/56evbsybp162xHUqpAspsPvWrVqiQmJvLRRx+xYsUKC6mUvwnqAR1cZ5Pff/996tSpQ6dOndi9e7ftSErl26ZNm7J9fsCAAcTExDBo0CDOnTvn41TK3wT9gA5Qvnx5Pv74Y7p3717gaXuVsqly5crZPh8eHs6MGTPYs2cPs2fP9nEq5W+KxIAOrvnX582bR2RkJMePH+e3336zHUkpj6VPcZGduLg42rdvz7hx4/I/f7YKKh4N6CLSTkR2uwtBZ5nuTUTKicgSd5Hob0SkkfNRnWGMoUOHDrRt21YL76qAcfDgwVzbp0+fzunTpxk/fryPEil/5Mn0uaHAXFzFoBsC94pIw0yLjQA2G2MaAw8AM50O6hQRITExkR07dhAXF5fjZUVK+ZNGjXLfR2rQoAEPP/ww8+bN08sYLUifDjf9tn//ftauXUvZsmW55ppruPLKKxk3bhzAJc83aNDA0W/7erKHfi3wozFmnzHmHPAmrsLQGTUE1gAYY3YBNUWkimMpHda2bVsWLVrE119/Tbdu3Rz5hpZS3uTJID127FgiIiIYPny4DxKpjNK/yp9+q1mzJuCaYfH7779n48aNvP766xdPbqc///3337Ns2TK+/PJLR3J4MqDnVAQ6oy1AFwARuRa4AqiReUUi8oiIbBSRjbaP9d11113MmzePFStWZDsPsVL+JDU1Nc9lqlSpwlNPPcV7773n2AChnFGyZEmaNWvG3r17L3k+MjKS2NhYn06f60kR6GeAmSKyGdgGfA9kme7QGPMS8BK4ClzkK6kXPPzww6SmptKhQwfbUZTK1dVXX+3RckOGDOGFF15g6NChfPXVV0Vycrqbb745y3PdunWjb9++nD59mri4uCzt8fHxxMfHc/ToUbp27XpJ29q1a/PsM332RHB9oXHJkiWXtCcnJ7NhwwZGjx59yYnr48eP85///IfWrVvnvWEe8GQP/SBweYbHNYDDGRcwxpw0xvQ2xsTiOoZeCfjJkYRe9thjj1GjRg1SU1P597//bTuOUtnydI7vkiVLMnHiRDZs2MCiRYu8G0pdlPGQS8bB/PPPP+eaa67htttu4+mnn+aqq666+Hzjxo2pWrUqHTp0oGrVqs4Eyal6dPoN1178PqAWEI7r8MpVmZaJAsLd9x8GXs1rvc2aNXO0OnZhPffccwYwr732mu0oSmWxa9cuj5dNTU01zZo1M5dddpk5deqUF1P5h507d9qOYEqWLJnluX//+9/m9ttvz/X53bt3m0qVKpnvv/8+2/Vmt23ARpPDuJrnHrox5gLQH1gJ/AAsNsbsyFQo+kpgh4jswnU1zEBn/tz4Tt++fbnllluIj49n6dKltuMoVWAhISHMmjWLQ4cOMWXKFNtxVC5iYmIYPnw4zz77rCPr8+g6dGPMcmNMjDGmjjFmkvu5ecaYee77640x9YwxDYwxXYwxxx1J50MRERG8//77NG/enHvuuYc1a9bYjqTURb/88ku+lm/VqhU9evRg2rRp7Nu3z0uplBMee+wxPvvsM376qfBHqcW1B+97zZs3Nxs3brTSd26OHTvGTTfdxO+//86+ffu0NqnyCydOnCAqKipfrzl06BD169fntttu47333vNOMD/www8/ZJkvPlhkt20isskY0zy75YvMV/89Vb58eVasWMHSpUt1MFd+Y9u2bfl+zWWXXcbIkSNZsmSJfuIsInRAz0aNGjW47rrrAHjzzTdJTk62nEgVdaGhoQV63eDBg6lduzYDBw7UwulFgA7oufj555+Jj48nLi6OlJQU23FUERYTE1Og10VERPDcc8+xY8cOXnjhBYdT+Q9bh469qSDbpAN6Li6//HLeeustNm3aROfOnTl79qztSKqI2r59e4Ff26lTJ9q0aUNiYiJHjx51MJV/iIiIIDk5OagGdWMMycnJRERE5Ot1elLUA6+//jr3338/HTt25N133yUsLMx2JFXE/Pjjj9StW7fAr9+5cyeNGzfmoYceYt68eQ4ms+/8+fMcPHgw6Ha4IiIiqFGjRpbxJreTop589b/I69mzJydPnqRfv36sWLGCO+64w3YkVcQUthpRw4YN6devH7Nnz6Zv3740btzYoWT2hYWFUatWLdsx/IIecvFQ37592bJliw7mygonCrKMHTuWqKgonnrqKQcSKX+kA3o+pO/VfPnll7z00kuW06iiJLsi0flVrlw5Ro0axcqVK1m1apUDqZS/0QG9AObMmcNjjz3G22+/bTuKKiJyKhKdX/369aNWrVokJCR4NCWvCiw6oBfAK6+8cvGr1StXrrQdRxUBkZGRjqynePHiTJ48mS1btvD66687sk7lP5yqKVpWRD4UkS0iskNEejsf1X9ERkaybNkyrrrqKu68806++OIL25FUkEuvgOOEe+65hxYtWjBq1ChOnz7t2HqVfU7VFO0H7DTGNAFuBp4TkZzLlAeBqKgoVq5cSXR0NP/v//0/23FUkPvhhx8cW5eI8Nxzz3Hw4EGSkpIcW6+yz6maogYoLa7yKKWAY2RTsSjYVK5cmc8//zzorutV/sfJPXRw1bTs1q0bzz77LAcOHHB03coep2qKzsE1J/phXCXoBhpj0hxJ6OcqVapEaGgoBw4coFOnTtiulaqCkzemnkjfO09ISHB83coOTwZ0T2qKtgU2A9WBWGCOiJTJsiI/KhLttMOHD/PJJ59w++238+eff9qOo4KMNyaIi46OZtiwYSxevJh169Y5vn7le47UFAV6A++5KyT9iKueaIPMKzLGvGSMaW6MaV6pUqWCZvZL11133cV5X7p27Vrob/YplZET16FnJyEhgejoaAYOHKiXMQYBTwb0b4F6IlLLfaKzO/BBpmUOAP8AEJEqQH1cdUiLlDvuuIMXX3yRjz/+mJ49e+p/EOUYp65Dz6xEiRJMmzaNLVu2sGDBAq/0oXzHqZqiE4BWIrINWAMMM8YE37RuHnjooYeYNm0ae/bs4eTJk7bjqCBRqlQpr627a9eutGrVisTERD1cGOB0tkUvOXv2LBEREaSlpRESot/fUoVz+PBhqlev7rX1f/XVV1x//fWMGzeOxMREr/WjCk9L0FkQERHB2bNn6dy5M9OnT7cdRwW4PXv2eHX9rVq14q677mLq1KkcOXLEq30p79EB3YvCwsIoXrw4Tz75JIsWLbIdRwWwOnXqeL2PKVOm8NdffzF27Fiv96W8Qwd0LwoNDeW1116jdevW9OrVSwv1qgLzRV3bevXq8fjjjzN//nx27Njh9f6U83RA97KIiAiWLl1KTEwMnTt35ttvv7UdSQWgEydO+KSfxMREypQpw+DBg4OqpFtRoQO6D5QrV46VK1dy5ZVX6glSVSDeug49s4oVKzJu3DhWrVrFBx9kvjpZ+Tu9ysWHjDG4pruBP/74g7Jly1pOpALF2rVrufnmm33S1/nz54mNjeXs2bPs2LEj34WKlXfpVS5+In0wnzp1KrGxsRw8eNByIhUooqKifNZXWFgYM2fOZN++fcyYMcNn/arC0wHdgr///e8kJydz66236mReyiMVKlTwaX9t2rShc+fOTJo0icOHM8/0ofyVDugWNG/enGXLlrF//37atm3LH3/8YTuS8nN79+71eZ/PPfcc58+fZ/To0T7vWxWMDuiWtG7dmnfffZdt27Zxxx13kJZWJGYbVgUUExPj8z5r165N//79WbBgAVu3bvV5/yr/dEC3KC4ujtdff51HHnlEr35RubJ12GPkyJGULVuWp556ykr/Kn90FLHsnnvuoUePHgD85z//0Wt/VbZOnTplpd/y5cszatQoVq5cyapVq6xkUJ5zqkh0gohsdt+2i0iqiJR3Pm7w2rx5M40aNeLZZ5+1HUX5IV9dh56d/v37U7NmTYYOHapTQvs5R4pEG2OSjDGxxphYYDiwzhhzzAt5g1bjxo3p2rUrw4cP56WXXrIdR/kZb82H7onixYszZcoUtm7dqgXR/ZxTRaIzuhfQmajyKSQkhAULFhAXF8djjz2mk3mpS/j6ssXM7rnnHlq2bMmIESN0nn8/5lSRaABEpATQDng3h/agrSnqhPDwcN555x1at27NAw88wPfff287kvITpUuXttq/iDBr1ix+++03JkyYYDWLyplTRaLTdQS+zOlwSzDXFHVKZGQkH3zwAUlJSTRp0sR2HOUn9u/fbzsCzZs3p3fv3sycOdPr87OrgnGqSHS67ujhlkIrU6YMgwYNIiQkhP379/PDDz/YjqQsu/LKK21HAGDy5MlERkYyePBg21FUNpwqEo2IlAVuAt53NmLRZYyhS5cutG3blkOHDtmOoyzyhz10gCpVqpCYmMjy5ctZsWKF7TgqE6eKRAPcCXxijNEqsw4REf75z39y/Phx4uLiOH78uO1IypIzZ87YjnDRE088Qb169UhISODChQu246gMPLoO3Riz3BgTY4ypY4yZ5H5unjFmXoZlFhpjunsraFF1zTXXsGTJEnbt2sXtt99u7Qsmyi6b16FnFh4ezpQpU9ixYwcLFy60HUdloN8UDQBt2rRh0aJFfP3111qRvYiyeR16drp06UKrVq0YPXq07mT4ER3QA0SXLl1YtmwZ48aNsx1FWVC5cmXbES4hIkybNo0jR47w3HPP2Y6j3HRADyDt27endOnSnDp1irlz5+q8L0VIeHi47QhZtGzZkq5du5KUlMSRI0dsx1HogB6QXn31Vfr3769f8ChC/LW61ZQpUzh37hwjR460HUWhA3pAevzxx3nggQcYM2YM//znP23HUT7QqFEj2xGyVbduXQYOHMiCBQsoajWC/ZEO6AFIRHj55Zdp27Ytjz76KEuXLrUdSXmZP38zc/To0VSuXJkBAwboYUDLdEAPUGFhYbzzzjs0b96cQYMG8ddff9mOpLzIn6etLVOmDFOmTGH9+vW88cYbtuMUaWLrL2rz5s2NfkQrvGPHjnH8+HHq1KljO4ryohMnThAVFWU7Ro7S0tL429/+xuHDh9m9ezelSpWyHSloicgmY0zz7Np0Dz3AlS9fnjp16mCMITExkW3bttmOpLxg8+bNtiPkKiQkhFmzZnH48GGeeeYZ23GKLB3Qg8TRo0f55z//Sfv27fn555/zfoEKKNWqVbMdIU8tW7ake/fuTJ8+XeceskQH9CBRqVIlVqxYQUpKCu3atdN5X5QVkydPJjU1ldGjR9uOUiQ5UlPUvczN7pqiO0RknbMxlScaN27M0qVL+fHHH+nUqZNfTeikCueXX36xHcEjtWrV4oknnmDhwoVs3brVdpwix5GaoiISBfwfcIcx5irgbuejKk/ccsstvPrqq3zzzTd6XXAQiY2NtR3BYyNHjiQqKoqnnnrKdpQix6maovcB7xljDgAYY35zNqbKj3vuuYe9e/dy44032o6iHBJIJ7vLlSvHqFGjWLlyJZ988ontOEWKUzVFY4ByIrJWRDaJyANOBVQFc9llrh/RokWL6N+/v37hI8CFhobajpAv/fr1o1atWgwZMkTnTPchp2qKFgOaAbcDbYHRIhKTZUVaJNrntm/fzty5c/UkVYCLicny38mvFS9enOeee44dO3bwwgsv2I5TZDhVU/Qg8LEx5k9jzFHgMyBLhWMtEu17EydO5KGHHmLSpEn6HyuAbd++3XaEfOvcuTP/+Mc/SExM5OjRo7bjFAlO1RR9H7hRRIqJSAngb7jK1SnLRIQXXniB22+/nf79++u8LwGqRo0atiPkm4gwc+ZMUlJS9BOijzhSU9QY8wPwMbAV+AZ42RgTeLsUQapYsWK89dZbNG/e3O+/caiyd+7cOdsRCuSqq66iX79+vPTSS2zZssV2nKCnc7kUIWfPniUiIgJwTfYUaCfairK1a9dy8803245RIMePH6devXo0adKE1atXI5LdaTnlKZ3LRQFcHMw3b95M48aN+eEHPSoWKPypSHR+lStXjjFjxvDpp5+yYsUK23GCmg7oRVDJkiVJTk7m1ltvZf/+/bbjKA/4W5Ho/Hr00UepW7cuCQkJehmjF+mAXgTVq1ePVatW8eeff9KuXTuSk5NtR1J5iIyMtB2hUMLDw3nmmWfYuXMnCxYssB0naOmAXkRdffXVfPDBB+zfv5+OHTvqvC9+rmbNmrYjFFqXLl1o1aoViYmJnDp1ynacoKQDehF244038q9//YtKlSqRlpZmO47KRTCc7xARkpKSOHLkCNOmTbMdJyjpVS4KYwwiwp9//kmJEiX0KgQ/tH///qDYSwfo1q0by5YtY9euXURHR9uOE3D0KheVKxHh1KlT3HjjjTz55JM674sfSklJsR3BMUlJSRhjSEhIsB0l6OiArgDXlS833ngjM2bMYPz48bbjqEyC6cT1FVdcwbBhw1i8eDHr1mnpBCfpIRd1UVpaGg8++CALFy5k5syZDBgwwHYk5ZaSkkLp0qVtx3DM6dOnadCgAeXLl2fTpk36Jbd80EMuyiMhISHMnz+fTp06MWjQIJYsWWI7knIL9OvQMytRogTTpk1jy5YtzJ8/33acoKEDurpEsWLFWLRoEb169aJp06a24yi3UqVK2Y7guLvvvpvWrVuTmJjIyZMnbccJCjqgqywiIyNZsGABV1xxBWlpaRw8eNB2pCKvevXqtiM4TkSYNm0av//+O1OnTrUdJyg4UiTaXSD6D3eR6M0ikuh8VGXD4MGDadGiBXv37rUdpUjbs2eP7Qhe0aJFC+69916mT5/OoUOHbMcJeI4UiXb73BgT677pZRJB4tFHH+X8+fO0adNG99QtqlOnju0IXjN58mRSU1N1znQHOFUkWgWphg0bsnLlyouTeWnpQDuC6bLFzGrWrMkTTzzBwoUL2bp1q+04Ac2pItEALUVki4isEJGrsluR1hQNTM2aNWPZsmXs37+fu+66S794ZMGJEydsR/CqkSNHEhUVxdChQ/X3qxCcKhL9HXCFMaYJMBtYmt2KtKZo4GrdujVvv/02EydO1KkBLAjk+dA9Ua5cOcaOHcuqVav44IPMFS6VpxwpEm2MOWmMOeW+vxwIE5GKjqVUfqFDhw60bt0agC+//FL3pHwo2K5Dz87jjz9Ow4YNGTJkCGfPnrUdJyA5UiRaRKqKe7dNRK51rzd4D/oVcZ999hk33HADo0aNsh2lyIiKirIdwevCwsKYOXMm+/btY8aMGbbjBCRHikQDXYHtIrIFmAV0N7r7FrRuvPFGHn74YSZPnkxSUpLtOEVChQoVbEfwiTZt2tC5c2cmTZqklzEWgM7logokNTWVHj168NZbbzFv3jweffRR25GCWiAXic6vffv20bBhQ7p168arr75qO47f0blclONCQ0N59dVXiYuL4/HHH2fnzp22IwW1mJgY2xF8pnbt2gwePJjXXnuN7777znacgKIDuiqw8PBw3nnnHd5++20aNszuu2bKKYcPH857oSDy9NNPU7FiRRISEvTkez7ogK4KJTIykrvuuguA9evXs3btWruBglRRq8FZtmxZEhMT+fTTT1mxYoXtOAFDj6ErRxhjaNmyJTt27GDNmjVce+21tiMFlWCbD90T586d46qrrqJ48eJs3ryZYsWK2Y7kF/QYuvI6EWHJkiVUrlyZuLi4oJ1MypaicB16ZuHh4TzzzDPs2LGDhQsX2o4TEHRAV46pVq0aK1euJCQkhHbt2nHkyBHbkYJGUblsMbMuXbrQqlUrRo0apXOme0AHdOWounXr8tFHH/Hrr7/y/PPP244TNIra4ZZ0IsLzzz/Pr7/+ysSJE23H8Xs6oCvHtWjRgvXr1zNp0iTbUYLG/v37bUewpkWLFvTu3Zvnn39eD+XlQQd05RWNGzcmNDSUw4cP8+STT3LhwgXbkQLalVdeaTuCVZMnTyYiIoIhQ4bYjuLXdEBXXvXJJ58wffp0+vTpQ1pamu04Aaso76EDVK1alcTERD766CO9jDEXOqArr4qPj2fixIm89tprDBw4UL8kUkBnzpyxHcG6AQMGEBMTw6BBgzh37pztOH7JkZqiGZZrISKpItLVuYgq0I0YMYInn3ySOXPm8Mwzz9iOE5CCfT50T4SHhzNjxgz27NnD7NmzbcfxS47VFHUv9yyuWRmVukhEmDp1Kvfddx9vvPGGznVdAEXxOvTsxMXFERcXx/jx4/n1119tx/E7TtYUfQJ4F/jNwXwqSISEhLBgwQI+//xzIiIibMcJOJUrV7YdwW/MmDGDM2fOMGLECNtR/I4jNUVF5DLgTmCec9FUsAkPDycqKoq//vqLnj17sn79etuRAkZ4eLjtCH4jJiaGgQMH8sorr/Dtt9/ajuNXnKop+jwwzBiTmuuKtEi0Ak6ePMk333xDXFwcW7ZssR0nIBw8eNB2BL8yevRoqlSpwoABA/REewaO1BQFmgNvish+XNWL/k9EOmdekRaJVgCVKlVi9erVlC5dmttuu02/LOKBRo0a2Y7gV8qUKcOkSZPYsGED77zzju04fsORmqLGmFrGmJrGmJrAO0BfY8xSp8Oq4BEdHc2qVaswxnDrrbdqubE86B+9rOLj42nUqBHDhw/XyxjdnKopqlS+1a9fn5UrVyIiRa6AQ36lpuZ6NLNICg0NJSkpib179/LCCy/YjuMXdD50Zd25c+cunvRLS0sjJES/75bZiRMniIqKsh3D7xhjuO222/juu+/Yu3dvkXiPdD505dfSB/MJEyZw33336d5oNjZv3mw7gl8SEZKSkjh+/DhTpkyxHcc6HdCV34iIiOCtt96ib9++euVCJtWqVbMdwW/FxsbywAMP8Pzzz7Nv3z7bcazSAV35jYSEBJ5++mleeuklhg0bpoO68tjkyZMJCwvjySeftB3FKh3QlV+ZPHkyffv2JSkpicmTJ9uO4zd++eUX2xH8WvXq1Rk5ciRLly5l9erVtuNYowO68isiwuzZs4mPj9fDDBnExsbajuD3Bg8eTO3atRk4cCDnz5+3HccKHdCV30mf96VPnz4AnDp1ynIi+7Zt22Y7gt+LiIhg+vTp7Ny5s8hexqgDuvJrq1evpmbNmnz99de2o1gVGhpqO0JAuOOOO7j11lsZM2YMycnJtuP4nA7oyq9dffXVlClThttvv52dO3fajmNNTEyM7QgBQUSYPn06J0+eLJI1bXVAV36tSpUqfPLJJxQrVoxbb72Vn376yXYkK7Zv3247QsBo1KgRvXv3Zs6cOUXuMkYd0JXfq1u3LqtWreLMmTO0adOGo0eP2o7kczVq1LAdIaCMHz+esLCwIjdnug7oKiBcffXVrFixgjvvvJPy5cvbjuNzOvlU/lSvXp0nn3ySt956i2+++cZ2HJ/RuVxUQDp06BAVKlQoMtWP1q5dy80332w7RkBJSUmhbt261K9fn3Xr1iGSXWmHwFPouVzyKhItIp1EZKuIbHYXsLihsKGVysmpU6do2bIlPXv2LDLzvmiR6PwrXbo048eP5/PPPy8yc6Y7VSR6DdDEGBML9AFedjinUheVKlWKwYMH8+677zJo0KAiMUWAFokumIceeogmTZowdOhQTp8+bTuO1zlSJNoYc8r8739VSbKWqFPKUYMHD2bIkCHMmTOHcePG2Y7jdZGRkbYjBKTQ0FBmzZrFgQMHSEpKsh3H6xwpEg0gIneKyC7gI1x76VloTVHlpKSkJOLj4xk3bhyvvfaa7TheVbNmTdsRAlbr1q3p1q0bzz77LAcOHLAdx6ucKhKNMWaJMaYB0BmYkN2KtKaoclJISAjz589n3LhxdOrUKe8XBLAffvjBdoSAlr53npCQYDmJdzlVJPoiY8xnQB0RqVjIbErlqVixYiQmJlKmTBlOnz7Nl19+aTuSV+geeuFER0czbNgwFi9eHLS/I+BQkWgRqSvua4JEpCkQDhS9iRSUVQkJCbRp04avvvrKdhTHpaSk2I4Q8IYOHUq1atUYOnRo0J5Id6pI9F3AdhHZjOuKmHtMsL5jym+NHTuWyy+/nI4dO7Jr1y7bcRxVFCeaclrJkiWZMGECGzZsCNrLGPWLRSqo7Nu3j1atWhEREcGXX37JZZdlOX8fkFJSUihdurTtGAEvNTWV2NhYzpw5w86dOy/Wsw0kWiRaFRm1a9dm+fLlHDt2jJ49e9qO4xi9Dt0ZoaGhTJ06lb179wblnOk6oKug07RpU5YvX868efNsR3FMqVKlbEcIGu3ataNNmzaMHz+e48eP247jKB3QVVC64YYbqF+/PsYY5s2bF/DfEqxevbrtCEFDRJg2bRonTpxg7NixtuM4Sgd0FdS+++47+vbty7333suFCxdsxymwPXv22I4QVJo0acIjjzzC3Llz2bFjh+04jtEBXQW1Zs2aMWvWLD744AP69+8fsJer1alTx3aEoDNhwgRKly4dVPMB6YCugl7//v15+umnefHFFwO2LJletui8ihUrMn78eFavXs37779vO44j9LJFVSQYY4iPj2fRokXs2rWL2rVr246ULzofundcuHCB2NhYTp8+zc6dOwNifn29bFEVeSLCyy+/zGeffRZwgznofOjeUqxYMZ5//nl++ukn5syZYztOoemAroqMsLAwrrvuOgAWL17Mhx9+aDmR5/Q6dO9p06YN7du3Z9KkSQF/aEsHdFXkpKWlMWPGDO6++27WrVtnO45HoqKibEcIalOnTuXkyZNMnDjRdpRC0QFdFTkhISEsW7aM2rVr06lTJ7Zv3247Up4qVKhgO0JQa9SoEX369GHu3Lns3bvXdpwCc6qmaA93TdGtIvKViDRxPqpSzqlQoQIrVqygZMmStG/fnp9//jnvF1kUyINMoBg/fjxhYWGMGDHCdpQCc6qm6E/ATcaYxriKW7zkdFClnHbFFVewYsUKTp48ydtvv207Tq5iYmJsRwh66VPrLl68mPXr19uOUyB5XrYoIi2BscaYtu7HwwGMMVNyWL4csN0Yk+s0d3rZovIXBw4cIDo62naMXG3cuJHmzbO9Uk056NSpU9SvX5/q1avz9ddfExLif0elC3vZokc1RTN4EFjheTyl7EofzLdu3UqPHj04e/as5URZnTp1ynaEIqFUqVJMnTqVjRs3snDhQttx8s2xmqIAInILrgF9WA7tWiRa+a2tW7fyxhtv0L17d7+b90WvQ/ed++67j1atWjF8+HD++OMP23HyxbGaoiLSGHgZ6GSMyfZiTi0SrfxZz549mT17Nu+//z59+vQhLS3NdqSL9Dp03xERZs2axe+//x5wlzE6VVM0GngPuN8Yo9PCqYDVv39/Jk2axGuvvcYTTzzhN5M26WWLvtWsWTN69+7NzJkzA2qmS6dqiiYCFYD/E5HNIqJnO1XAGj58OAkJCfz444+cO3fOdhwALT9nweTJkylevDhPP53lSm2/pZNzKZUNYwznz58nPDyc1NRUQkNDrebRybnsmDBhAomJiXz++efccMMNtuMAOjmXUvkmIoSHh5OcnMz111/PG2+8YTXPlVdeabX/omrIkCFUq1aNhIQEvzn8lhsd0JXKRcmSJYmIiCA+Pp41a9ZYy7F//35rfRdlJUuWZMKECWzYsIF33nnHdpw86YCuVC4iIiJYunQpMTEx3HnnnWzevNlKjjNnzljpV0F8fDyNGjVi+PDhfnNOJSc6oCuVh6ioKD7++GOioqK47bbb2L17t88z6HXo9oSGhpKUlMTevXuZPXu27Ti50gFdKQ/UqFGD1atXU69ePYoXL+7z/vU6dLvatWtH+/btGT9+PL/++qvtODnSAV0pD8XExPDFF19Qs2ZN0tLSOH78uM/6rly5ss/6UtmbMWMGZ86c8evZGHVAVyofRFwzYfTv35+///3vnDx50if9hoeH+6QflbP69eszcOBAFixYgL9ecq0DulIF0LFjR7Zt28Zdd93lkxNlBw8e9HofKm+jR4+mcuXKDBgwwC8vY9QBXakCaN++PS+//DKrV6/mwQcf9Pp/7kaNGnl1/cozZcqUYcqUKaxfv55FixbZjpOFDuhKFVB8fDwTJ07k9ddf9/px1UCaTyTY9erVi2uuuYYRI0b43VTLOqArVQgjRozgqaee4rbbbvNqP6mpqV5dv/JcSEgISUlJ/Pe//2XOnDm241xC53JRykG7d++mfv36jq/3xIkTREVFOb5eVXBxcXF89dVX7N2716ezYRZ6LhcPikQ3EJH1IvKXiAwtbGClAtFHH31Ew4YNefXVVx1ft61vqKqcTZ06lZSUFL+aM92pItHHgAHANMcTKhUg2rRpwy233MKDDz7IypUrHV13tWrVHF2fKrxGjRrRu3dv5s6dy969e23HATzbQ78W+NEYs88Ycw54E+iUcQFjzG/GmG+B817IqFRAKF68OO+99x5XXXUVd911l367swgYP3484eHhDB3qHwcmvFEkOkdaU1QFuzJlyrB8+XIqVqxIXFwcx44dc2S9v/zyiyPrUc6qXr06I0aMYOnSpaxevdp2HGeLROdFa4qqoqB69ep8/PHHjBs3jvLlyzuyztjYWEfWo5w3ZMgQateuzcCBAzl/3u5BCseKRCul/qdBgwY89pirQuN3331HcnK2ddM9tm3bNidiKS+IiIjgueeeY+fOnbzwwgtWszhSJFoplb0///yTdu3a0a5du0LN+2K7BJ7KXadOnWjTpg1jxozB5uFkR4pEi0hVETkIDAFGichBESnjzeBKBYKSJUvyyiuvsHnzZjp37sxff/1VoPXExMQ4nEw5SUSYOXMmKSkpjBs3zloOj65DN8YsN8bEGGPqGGMmuZ+bZ4yZ575/xBhTwxhTxhgT5b7vm2nolPJzHTp0YMGCBfz73/+mV69epKWl5Xsd27dv90Iy5aSGDRvy8MMP8+KLL1qbqkG/+q+UD/Ts2ZNnn32Wt956i1deeSXfr69Ro4YXUimnjR07loiICJ5+Osv3L31CB3SlfCQhIYFFixYRHx+f79f6ey1L5VKlShWGDRvGkiVL+OKLL3zevw7oSvmIiNC9e3eKFSvGkSNH8lVF/rfffvNiMuWkIUOGUL16dRISEnw+Z7oO6EpZMHbsWLp168bixYs9Wl6LRAeOEiVKMGHCBDZs2MDbb7/t0751QFfKgunTp3PDDTfQo0cPli9fnufyOo1AYOnVqxdNmjQhISGB06dP+6xfHdCVsqBEiRJ8+OGHNG7cmK5du/L111/nunxkZKSPkiknhIaGMmvWLA4cOEBSUpLP+tUBXSlLypYty/Lly6lWrRpDhw7N9XhrzZo1fRdMOaJ169Z069aNZ599lgMHDvikTx3QlbKoSpUqrFq1iqVLlyKS3bRJLj/88IMPUymnpO+dJyQk+KQ/HdCVsqx27dpUqFCBc+fOMXLkSE6cOJFlGd1DD0zR0dEMGzaMxYsXs27dOq/3pwO6Un7iu+++IykpiQ4dOmQ5kZaSkmIplSqshIQEoqOjGThwoNdrw+qArpSfuO666/jXv/7F+vXr6dKlyyXzvhR2tkZlT4kSJUhKSmLLli3Mnz/fq305VVNURGSWu32riDR1PqpSwe/uu+9m/vz5rFy5knvuuYezZ88Ceh16oLv77ru56aabGDVqFMePH/daP07VFG0P1HPfHgHsTgqsVADr06cPc+bMYd26dezfvx/Q69ADXfpsjMePH2fMmDFe66eYB8tcrCnqDpZeU3RnhmU6Aa8a13VXG0QkSkSqGWO0bpZSBdCvXz+6d+9OhQoVMMaQnJzMoUOHLlmmcuXKhIWFkZKSku1c61WrViU0NJSTJ09mewy+WrVqhISE8Mcff3Dq1Kks7Zdd5qo0efz48SzH9EWE6tWrA3Ds2DHOnDlzSXtISMjFwtbJyckXP2mkK1asGFWqVAHg999/zzJXTVhYGJUrVwZc0x5krgQUHh5OetWzX3/9lQsXLlzSHhERQYUKFQBX+b7MM1xGRkZerCZ1+PDhLJeMlihRgnLlygFked8BSpUqRdmyZUlLS8u2PGDp0qUpU6YMqampHDlyBICKFSty3333MXfuXB555BEaNWqU5XWFZozJ9QZ0BV7O8Ph+YE6mZZYBN2R4vAZontt6mzVrZpRSeZs5c6bBVfbxktvu3buNMcZMmzYt2/ZDhw4ZY4wZM2ZMtu0nT540xhgzZMiQbNvTPfzww1naSpUqdbH93nvvzdJerVq1i+0dOnTI0l6vXr2L7TfddFOW9qZNm15sb9q0aZb2m2666WJ7vXr1srR36NDhYnu1atWytN97770X20uVKpWl/eGHH77Ynt17M2TIEGOMMSdPnsy2fcyYMcYYYw4dOpSlLTQ01PTv3z//vwj/y7PR5DCuerKH7klNUY/qjorII7gOyRAdHe1B10qpjh07cuDAAerXr3/J8+l7uLfeeisvvfRSlteVLVv24uvT97YzKl68OOA6vtugQYMc+7///vtp0aLFJc+FhYVdvP/QQw9xyy23XNJeokSJi/f79+/PHXfccUl7mTL/q38zdOhQevTocUl7xYoVL94fNWoUR48evaQ9fe8fYMKECVk+oWQcX5KSkrJ8wqhTp87F+7Nnz87yCSDj+5Hde3v11VcDrvcwu/amTV2nEcuWLZul/a+//qJv375ZXuMEMXnMBiYiLYGxxpi27sfDAYwxUzIs8yKw1hizyP14N3CzyeWQS/Pmzc3GjRsLvwVKFQE///wzl19+ed4LqqAnIpuMMc2za3OqpugHwAPuq12uA/7IbTBXSuWPXraoPJHnIRdjzAURSa8pGgq8Ytw1Rd3t84DlQBzwI3Aa6O29yEoVPdl9e1SpzPI85OIteshFKc+lpKRQunRp2zGUHyjsIRellGV6HbryhLU9dBH5HfhvAV9eETia51LeYatv3eai0XdR69dm34G6zVcYYypl12BtQC8MEdmY00eOYO1bt7lo9F3U+rXZdzBusx5yUUqpIKEDulJKBYlAHdCzfjUr+PvWbS4afRe1fm32HXTbHJDH0JVSSmUVqHvoSimlMtEBXSmlgoRfD+i2KiV50G8DEVkvIn+JyFAn+sxH3z3c27pVRL4SkSY+6reTu8/NIrJRRG5wol9P+s6wXAsRSRWRrr7oV0RuFpE/3Nu8WUQSnejXk74z9L9ZRHaIiCMVhj3Y5oQM27vd/X6X91HfZUXkQxHZ4t5mR6YQ8aDfciKyxP37/Y2IODJRuYi8IiK/icj2HNqdH79ymlfX9g3XvDF7gdpAOLAFaJhpmThgBa7pe68DvvZRv5WBFsAkYKiPt7kVUM59v70Pt7kU/zvn0hjY5attzrDcp7jmDerqo22+GVhm6Xc7ClcRmej03zlfvdcZlu8IfOrDbR4BPOu+Xwk4BoT7oN8kYIz7fgNgjUPb3BpoCmzPod3x8cuf99AvVkoyxpwD0islZXSxUpIxZgMQJSLVMq/I6X6NMb8ZY74Fzme3Ai/3/ZUxJr0o4Qagho/6PWXcv4VASbKZ795bfbs9AbwL/Objfr3Bk77vA94zxhwA1++cj/rN6F5gkQP9etq3AUqLiODagTgGXKBwPOm3Ia6iPBhjdgE1RaRKIfvFGPMZrm3IiePjlz8P6JcBP2d4fND9XH6X8Ua/3pLfvh/E9RfeJ/2KyJ0isgv4COjjQL8e9S0ilwF3AvMc6tOjft1aug8BrBCRq3zYdwxQTkTWisgmEXnAR/0CICIlgHa4/og6wZO+5wBXAoeBbcBAY0waheNJv1uALgAici1wBc7sKDmRLV/8eUB3rFKSF/r1Fo/7FpFbcA3ow3zVrzFmiTGmAdAZmOBAv572/TwwzBiT6lCfnvb7Ha55M5oAs4GlPuy7GNAMuB1oC4wWkRgf9JuuI/ClMSa3PUyn+24LbAaqA7HAHBEpQ+F40u8zuP54bsb1SfB7Cv/JwBOOjzWelKCz5SCQsURLDVx/ufO7jDf69RaP+haRxsDLQHtjjBOVD/K1zcaYz0SkjohUNMYUdnIjT/puDrzp+iRORSBORC4YY5Z6s19jzMkM95eLyP/5cJsPAkeNMX8Cf4rIZ0ATYI+X+03XHecOt3jad2/gGfehvR9F5Cdcx7S/8Wa/7p9zb3CdqAR+ct+8zfmxxomD/9644fpjsw+oxf9OZlyVaZnbufSkwje+6DfDsmNx9qSoJ9scjauQSCsf91uX/50UbQocSn/sq/fbvfxCnDkp6sk2V82wzdcCB3y1zbgOPaxxL1sC2A408sV7DZTFdey3pI9/x17AVe4SoIr7d6yiD/qNwn3yFXgY13Ftp7a7JjmfFHV+/HIquDduuM4C78F1lnqk+7nHgMfc9wWY627fBjT3Ub9Vcf11PQmccN8v46O+XwaO4/pouplcKoA73O8wYIe7z/XADb76OWdadiEODOgebnN/9zZvwXUC2sk/onluM5CA60qX7cAgH/YbD7zp1Lbm4/2uDnzi/r+8Hejpo35bAv8BdgHv4b6KzIF+FwG/4Lp44iCuQ6ReHb/0q/9KKRUk/PmkqFJKqXzQAV0ppYKEDuhKKRUkdEBXSqkgoQO6UkoFCR3QVUAQkQoZZgE8IiKH3PdPiMhOL/Q3VvI5k6aInMrh+YVOzRCpVG50QFcBwRiTbIyJNcbE4prTZYb7fiyQ53wfIuLP34pWyhE6oKtgECoi891zaH8iIpEA7omtJrvnEh8oIs1EZJ17squV6TPbicgAEdnpnpP6zQzrbehexz4RGZD+pIgMcc8Vvl1EBmUO457neo57nR/hmm45ve2ZDH1N89Ybooom3WtRwaAecK8x5mERWQzcBbzubosyxtwkImHAOqCTMeZ3EbkH13z2fYCngVrGmL9EJCrDehsAtwClgd0i8gKuueB7A3/D9U2/r0VknTHm+wyvuxOoD1yN6yvsO4FX3IUi7gQaGGNMpr6UKjQd0FUw+MkYs9l9fxOu+TPSveX+tz7QCFjlnuQrFNfXsgG2Av8SkaVcOqPiR8aYv4C/ROQ3XIPzDcAS45o0CxF5D7gR1wx96VoDi4xrdsjDIvKp+/mTwFngZfee+7KCb7JSWekhFxUM/spwP5VLd1T+dP8rwI704/DGmKuNMbe5227HNadGM2BThuPt2a03uylPs5Pd9MMXcE3y9S6uKYg/9nBdSnlEB3RVVOwGKolISwARCRORq0QkBLjcGPNv4ClcM++VymU9nwGdRaSEiJTEdQjl82yW6S4ioe7j9Le4+ywFlDXGLAcG4Tqhq5Rj9JCLKhKMMefclw7OEpGyuH73n8c1C9/r7ucE19UzJ9yHZbJbz3cispD/zdH9cqbj5wBLgL/jmkFvD65j9+A6Fv++iES4+xrs0OYpBaCzLSqlVLDQQy5KKRUkdEBXSqkgoQO6UkoFCR3QlVIqSOiArpRSQUIHdKWUChI6oCulVJD4//bpbtXt7rj/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(df_ideal.threshold, df_ideal.tpr, color='black', linestyle='solid', label='TPR')\n",
    "plt.plot(df_ideal.threshold, df_ideal.fpr, color='black', linestyle='dashed', label='FPR')\n",
    "plt.legend()\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "plt.yticks(np.linspace(0, 1, 11))\n",
    "\n",
    "plt.vlines(1 - y_val.mean(), -1, 2, linewidth=0.5, linestyle='dashed', color='grey')\n",
    "plt.ylim(-0.03, 1.03)\n",
    "\n",
    "plt.xlabel('Thresholds')\n",
    "plt.title('TPR and FPR for the ideal model')\n",
    "\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABC/UlEQVR4nO3dd3wU5fb48c8hlFCEAFGKSBNESiBAaAGB0FuQIkhRpCjqFSsoqFiu5ao/1Gu53q9iARUMSBEJvUuHEIyQgECQFkggBIgBEkh5fn/skhtCCBvIZnY35/165ZWdndmZM4GcPPPMM+cRYwxKKaUcU8TqAJRSyp1o0lRKqTzQpKmUUnmgSVMppfJAk6ZSSuWBJk2llMoDTZpKKZUHmjSV04nIYRFJFpHzIhInItNFpEy2bQJFZI2IJIlIooiEikiDbNuUFZFPROSofV/R9mXfgj0jVZhp0lQFJdgYUwbwB5oCL19ZISJtgBXAr0BVoBbwB7BJRGrbtykOrAYaAj2AskAgkAC0dFbQIlLUWftW7kmTpipQxpg4YDm25HnF/wN+MMZ8aoxJMsacMcZMBrYCb9q3GQFUB/obY/YYYzKMMaeMMW8bY5bkdCwRaSgiK0XkjIicFJFX7O9PF5F3smzXUURisiwfFpGJIrILuCAik0VkbrZ9fyoin9lflxORb0UkVkSOi8g7IuJ1az8p5ao0aaoCJSLVgJ5AtH25FLYW45wcNv8Z6Gp/3QVYZow57+BxbgNWAcuwtV7rYGupOmoo0BvwAX4EeolIWfu+vYDBwE/2bb8H0uzHaAp0Ax7Nw7GUG9GkqQrKAhFJAo4Bp4A37O9XwPb/MDaHz8QCV/orK15nm+vpA8QZYz4yxqTYW7Db8vD5z4wxx4wxycaYI8BOoJ99XSfgojFmq4hUwvZH4DljzAVjzCng38CQPBxLuRFNmqqg9DPG3AZ0BO7lf8nwLJABVMnhM1WA0/bXCdfZ5nruAg7eVKQ2x7It/4St9QkwjP+1MmsAxYBYETknIueAr4A7buHYyoVp0lQFyhjzGzAd+NC+fAHYAgzKYfPB/O+SehXQXURKO3ioY8Dd11l3ASiVZblyTqFmW54DdLR3L/Tnf0nzGHAJ8DXG+Ni/yhpjGjoYp3IzmjSVFT4BuoqIv315EvCIiDwjIreJSHn7jZo2wD/t2/yILUHNE5F7RaSIiFQUkVdEpFcOx1gEVBaR50SkhH2/rezrIrD1UVYQkcrAczcK2BgTD6wDpgGHjDF77e/HYrvz/5F9SFQREblbRDrk8Wei3IQmTVXg7AnoB+A1+/JGoDswAFu/5RFsN1TaGWMO2Le5hO1m0J/ASuBvYDu2y/xr+iqNMUnYbiIFA3HAASDIvvpHbEOaDmNLeLMdDP0neww/ZXt/BFAc2IOtu2EueetKUG5EtAixUko5TluaSimVB5o0lVIqDzRpKqVUHmjSVEqpPNCkqZRSeeB2FVx8fX1NzZo1rQ5DKeVhwsPDTxtjbr/Rdk5LmiLyHbbnf08ZYxrlsF6AT4FewEVgpDFm5432W7NmTXbs2JHf4SqlCjkROeLIds68PJ+Ore7h9fQE6tq/xgL/58RYlFIqXzgtaRpj1gNnctnkfmw1FI0xZivgIyL6FIVSyqVZ2ad5J1dXkomxv5eX8l/XMMaQkZFxK7tQSnmoCxcu8NVXX9GoUSN69ux5U/uwMmlKDu/l+EyniIzFdglP9erVc9xZamoqkZGRxMfH4+3tja3LVCml4OLFiyxYsICQkBASExN59NFH3TJpxmCreXhFNeBEThsaY6YCUwECAgKuSawRERGcOXOGRo0a0bRpU2fEqpRyUzNnzuTZZ58lISGBZs2aMWjQICZNmnTT+7NynOZCYITYtAYS7WW28uzcuXN06tSJO+7Quq9Kqf/ZsmULI0eOpF69eixcuJChQ4fyzDPP3NI+nTnkKARblW5f+6RVb2CrcI0x5ktgCbbhRtHYhhyNclYsSqnCJz4+nsGDB1O9enUWLVpERkYGDRs2pFSpUjf+cC6cljSNMUNvsN4ATznr+Eqpwis9PZ1hw4Zx6tQpvv32W9asWcPAgQOpWLHiLe9bH6NUSnmcf/7zn6xatYpRo0Zx/vx5OnbsmG/7drvHKJVSKjdLlizh7bffpl+/fgQHB9O9e3eKFs2/VKdJUynlMf78808GDx5MvXr1+OmnnyhZsmS+H0Mvz5VSHuGPP/6ga9euGGOYP3++UxImaEtTKeUBjDFMnDiRmJgYFixYQIMGDZx2LE2aSim3lJ6eztmzZ1m7di2JiYksX76cl156ifvvv9+px9WkqZRyOxEREYSGhlK6dGnKly/P008/TYcOHXj33XedfmxNmkopt7Fu3Tp8fHxo1KgRfn5+XLhwgRYtWuDj48OsWbPy9S759eiNIKWUW/j999/5448/qFOnDkWLFqVIkSKMGTOGgwcPMnv2bCpXrlwgcWjSVEq5vFOnTrFq1SqGDx9OmTJlAPj000+ZO3cu7733Hu3bty+wWDRpKqVc3m233cawYcPw9fUFYNOmTbz44ov069ePCRMmFGgsmjSVUi7v7NmzVKlim9jh1KlTDB48mJo1azJ9+vQCr52rSVMp5dIuXrzI999/T0ZGRmYhjjNnzjB37lzKlStX4PHo3XOllEuLioqiVq1aFC1alMmTJ7N69WqmTZtGkyZNLIlHW5pKKZeVmJjI2rVrCQoKYtGiRbz77rs8+uijjBw50rKYNGkqpVxW2bJlefTRR7l48SIPP/wwTZs25fPPP7c0Jk2aSimXc/HiRWbMmEFsbCylSpXigQceAGDu3Ll4e3tbGpv2aSqlXMqJEyf4+eefadSoEZUrV+bJJ59k586dLFy4kNq1a1sdniZNpZTrMMawbNkyunXrRoMGDfjhhx+YOnUqkyZNIjg42OrwAE2aSikXce7cOUqUKMGoUaMQEXbv3s0TTzxBUFAQb7/9ttXhZdI+TaWU5TIyMggNDSUqKgoRITExkYEDB+Lj40NISEiBFOJwlOtEopQqlI4fP55Z5q1p06YYYxg9ejR//fUX69ato1KlSlaHeBVNmkopSyQlJVGyZEkuX75Mu3btaNiwISLCxx9/zPz58/noo49o166d1WFeQ5OmUqpApaWlsXbtWnbu3MnQoUOpVatW5rqNGzfy0ksvMXDgQJ5//nkLo7w+TZpKqQK1dOlSEhMTefLJJylbtmzm+ytWrGDEiBHUrl2b7777rsALcThKbwQppQqMMYY777yTQYMGZSbMc+fOMWbMGLp370758uX59ddfr0qmrkZbmkqpAhETE8OFCxdo1qxZ5nuhoaE88cQTnDx5kldeeYXXXnvN8id+bkRbmkoppzt9+jSzZs3KvOQ+ffo0w4cPp2/fvvj6+rJt2zbeffddl0+YoElTKeVkiYmJ/Pjjj3Tp0oV77rmHOXPm0KBBA+bMmcM///lPwsLCaN68udVhOkwvz5VSTlWkSBGCgoKoXLkyDzzwAPPmzSMgIIDVq1fj5+dndXh5pi1NpZRTZGRksGbNGooVK0ZkZCQNGzZk0aJFfPDBB2zZssUtEyZoS1Mp5SRr165l9+7dfPTRRyxZsoTAwEC+++476tWrZ3Vot0STplIq3+3fv5+vvvqKZcuWkZ6ezqeffspTTz2Fl5eX1aHdMk2aSql8dejQIR5++GG2b99OUFAQ33zzjUvUwcwv2qeplMoXGRkZfP755zRs2JA9e/bw5ZdfsmrVKo9KmKAtTaVUPjhw4ABjxoxhw4YNNGjQgMWLF1OzZk2rw3IKbWkqpW5aeno6H330EX5+fkRERDB48GCWL1/usQkTtKWplLpJUVFRjBw5kh07dnD//ffz4IMPEhgYSLVq1awOzak0aSql8iQ1NZW33nqL999/n+LFizN+/HimTJnislWJ8psmTaWUwyIiIhg1ahQRERF06NCBqVOncs8991gdVoHSPk2l1A1dunSJ1157jYCAAE6cOMH8+fNZt25doUuYoC1NpdQNbNu2jdGjR7Nnzx569OhB//796d+/v9VhWUZbmkqpHCUnJ/Piiy8SGBjI33//zbRp0+jcuTNDhgyxOjRLadJUSl1jw4YNNGnShA8//JDHHnuMXbt2kZSUxIABA1y6qnpBcGrSFJEeIrJPRKJFZFIO68uJSKiI/CEiUSIyypnxKKVyd/78eZ5++mnat29PWloaq1ev5rPPPsPHx4exY8d63NM9N8NpSVNEvIAvgJ5AA2CoiDTIttlTwB5jTBOgI/CRiBR3VkxKqetbtWoVfn5+fPHFFzz77LPs3r2bDh06EBISwu7duylRooTVIboEZ7Y0WwLRxpi/jDGXgVnA/dm2McBtYhvgVQY4A6Q5MSalVDaJiYk89thjdO3aleLFi7NhwwY++eQTjDH8+OOPeHt706hRI6vDdBnOTJp3AseyLMfY38vqP0B94ASwG3jWGJORfUciMlZEdojIjvj4eGfFq1Shs3jxYho2bMh3333HxIkTiYiIoG3btgCEhYVRo0YNBg0aRJEievvjCmcOOcrp8QCTbbk7EAF0Au4GVorIBmPM31d9yJipwFSAgICA7PtQSuXR4cOHmTRpErNnz6ZRo0b88ssvtGjRAoCUlBTOnTtHx44dC81TPnnhzD8fMcBdWZarYWtRZjUKmG9sooFDwL1OjEmpQu3cuXNMnDiRe++9l4ULF/Lmm28SHh6emTAzMjKYP38+v//+uybM63Bm0gwD6opILfvNnSHAwmzbHAU6A4hIJaAe8JcTY1KqUEpNTeU///kPderUYcqUKQwZMoT9+/fzxhtvULy47d5rRkYGoaGhXL58mW7dulkcsety2uW5MSZNRMYBywEv4DtjTJSIPGFf/yXwNjBdRHZju5yfaIw57ayYlCpsjDGEhoby0ksvsW/fPoKCgvjoo49o2rTpNdsmJSWRmprKsGHDPGJaCmdx6mOUxpglwJJs732Z5fUJQP+kKeUE4eHhTJgwgXXr1nHvvfcSGhpK7969r7nsTk9PJywsjJYtW/LAAw9YFK370FtiSnmYY8eOMWLECAICAoiKiuK///0vu3btok+fPtckzOPHjzN16lSOHDlCRsY1A1dUDrRgh1IeIikpiffff5+PP/4YYwyTJk1i0qRJlCtXLsftT5w4QUhICN26dcPPz09v/DhIk6ZSbi4tLY1vv/2W119/nVOnTjF8+HDeffddatSocc22Z86c4ffff8fX15fGjRszbtw4vL29LYjafWnSVMpNGWNYtmwZEyZMYM+ePdx3330sWrQoc/hQVqmpqYSEhBAXF0eTJk2oVq0aIqIJ8yZo0lTKDf3xxx9MmDCBVatWUadOHebPn0+/fv1yvMQ+c+YMFSpUIDAwkJo1a1K0qP7a3wq9EaSUGzlx4gSjR4+madOm7Ny5k08//ZSoqCj69+9/TcI0xrBmzRpCQkJIT0+nTp06mjDzgf4ElXIDFy5cYMqUKUyZMoW0tDReeOEFXn31VcqXL5/j9unp6YSGhhIfH8/IkSN13GU+0qSplAtLT0/n+++/Z/LkycTGxjJ48GDee++9G9a1TEpKokyZMvTq1SvziR+VPzRpKuWiVq5cyYQJE9i1axetW7dm3rx5tGnT5oafM8bg7e1Nly5dCiDKwkf7NJVyMVFRUfTq1Ytu3bqRlJTE7Nmz2bx5s0MJE+Cvv/5i5syZTo6y8NKkqZSLOHnyJI8//jiNGzdm8+bNfPjhh+zdu5fBgwfnaeD5jh07aNKkiRMjLdw0aSplsYsXL/Luu+9Sp04dvvvuO8aNG8fBgwcZP368w1NMGGMrM7t582ZiYmLw8/NzZsiFmvZpKlXAjDEcPnyYDRs2sHHjRhYvXsyJEyfo378/H3zwAXXr1s3T/hISEvj5558ZNmwY/v7+tGzZUocWOZH+ZJVysvT0dCIjIzOT5IYNGzhxwlaP28fHh/vuu4+QkBDat2+f530fOXKEOXPmEBQUdN1nzFX+0qSpVD5LSUkhLCwsM0lu3ryZxMREAKpVq0aHDh1o164d9913Hw0bNryp+XeMMZmPUfbv35+77747v09DXYcmTaVugTGG48ePExYWxrZt29i4cSNhYWFcvnwZgAYNGjBkyJDMJFm9evVbqiaUnJzM+vXrOXXqFA8//DBjx47V6kQFTJOmUnlw+vRpwsLCCAsLY8eOHYSFhREXFwdA0aJFCQgI4JlnnuG+++4jMDAQX1/ffDt2UlIS33//PdWrV6d///4AmjAtoElTqetISkoiPDw8M0mGhYVx+PBhwJas6tWrR9euXWnRogUtWrSgSZMmlCxZ0mnxnD59Gn9/f9q1a+e0Y6gb06SpFLbL7IMHD7J69Wo2bdpEWFgY+/btyxzKU7NmTVq0aME//vEPWrRoQbNmzShbtmyBxHbp0iUiIyNp1qwZtWrVKpBjquvTpKkKrZMnT7J69erMryNHjgBQqVIlWrZsydChQ2nRogUBAQHcfvvtlsSYnp7OnDlzKFeuHM2aNbMkBnU1TZqq0Pj777/57bffMpNkZGQkYBv2ExQUxEsvvUTnzp255557XKKvMCMjg0WLFiEiOU6IpqyhSVN5rEuXLrF161ZWrVrF6tWr2b59O+np6Xh7e9OuXTseeughOnfuTNOmTV2udNrly5cpVqwYpUuXpmfPnjc1LEk5hyZN5THS09P5448/MpPkhg0bSE5OpkiRIrRo0YKJEyfSpUsX2rRp47LTPKSmprJu3Tqio6N54okntFKRC9KkqdzSlUcRw8LC2L59O2FhYezcuZPz588DtvGRjz76KF26dKFDhw5u8bRMSkoK3377LZUqVWLEiBF6Oe6iNGkqt3Dy5MnMYT/bt29nx44dnD59GoASJUrg7+/PyJEjad26NUFBQVStWtXiiPMuKSmJxo0bc99991kdisqFJk3lcowx/PnnnyxZsoQtW7YQFhbG0aNHAShSpAgNGzakb9++meMj/fz83L46+dGjR6lataomTDegSVO5hLS0NDZu3EhoaCgLFy4kOjoagLvvvpvAwECeffbZzPGRpUuXtjja/JWYmMisWbMYM2YMFStWtDocdQOaNJVlEhMTWbZsGaGhoSxZsoSzZ89SvHhxOnfuzAsvvECfPn246667rA7TqYwxLFmyhFatWmnCdBOaNFWBOnToUGZr8rfffiMtLQ1fX1/uv/9+goOD6datG2XKlLE6zAITFxfH2bNnGTx4sNWhKAc5nDRFpLQx5oIzg1Ge6cSJE/z444/MnDmT3bt3A7a72+PHj6dv3760atXK5cZJFpQqVarw2GOPFdrzd0c3TJoiEgh8A5QBqotIE+BxY8w/nB2ccl8pKSmEhoYybdo0li9fTkZGBm3btuXf//43wcHBWv8R2LhxIz4+PjRq1MjqUFQeONLS/DfQHVgIYIz5Q0TyXmJaeTxjDOHh4UybNo2QkBDOnj1LtWrVePnllxk5ciR16tSxOkSXkZyczKZNm3jsscesDkXlkUOX58aYY9kG2qY7JxzljuLi4pg5cybTpk0jKioKb29vBgwYwMiRI+nUqZNeeuZgy5Yt1K9fnwoVKlgdisojR5LmMfsluhGR4sAzwF7nhqXcwY4dO3j33XcJDQ0lPT2d1q1b89VXXzF48GB8fHysDs+lZWRk3NScQMp6jiTNJ4BPgTuBGGAFoP2ZhVhERARvvPEGCxcupEKFCowfP56RI0dSv359q0NzecePHycjI0OfKXdjjpROqWeMGW6MqWSMucMY8xCgvx2FUFRUFA888ABNmzZl/fr1vP322xw6dIgPPvhAE6YDTp48yU8//cTFixetDkXdAkeS5ucOvqc81L59+xg6dCh+fn6sWLGC119/nUOHDjF58uQCq17u7hISEpgxYwY9e/akXr16VoejbsF1L89FpA0QCNwuIi9kWVUW0J59D2aMYd++fWzatImVK1cyZ84cSpYsyaRJk5gwYYLevMijjIwMMjIy6Nq1qw4v8gC59WkWxzY2syhwW5b3/wYecGZQqmClpKSwY8cONm3axKZNm9i8eTMJCQkAVKxYkeeff56JEydaNuWDu0pOTmb58uWUKFGCnj176s/PQ1w3aRpjfgN+E5HpxpgjBRiTcrLU1FTWrl3LypUr2bRpE+Hh4ZnzdN9zzz307duXtm3b0rZtW+rVq6d1HW9CdHQ0CxcupH79+nTq1MnqcFQ+cuTu+UURmQI0BDLLXRtj9H+CG0lLS2Pt2rX8/PPPzJ8/nzNnzlC8eHECAgJ49tlnadu2LYGBgdoaukXp6el4eXmRmJhI//79dfZID+RI0pwJzAb6YBt+9AgQ78ygVP64kijnzJnD/PnzSUhIoEyZMvTt25fBgwfTrVs3p87TXdjEx8cTEhLC4MGDad68udXhKCdxJGlWNMZ8KyLPZrlk/83Zgambk5KSwqpVq1iwYAG//vorp0+fzkyUgwYNonv37poonSA2NpaZM2fSpUsXKleubHU4yokcSZqp9u+xItIbOAFUc2TnItID28B4L+AbY8z7OWzTEfgEKAacNsZ0cGTf6n8SExNZvHgxCxYsYOnSpZw/f56yZcvSu3dvBg0aRI8ePTRROokxBhFh7dq19O7dW8erFgKOJM13RKQcMB7b+MyywHM3+pCIeAFfAF2xPUkUJiILjTF7smzjA/wX6GGMOSoid+T5DAqxlStX8tFHH7FmzRpSU1OpXLkyw4cPp3///gQFBbn9FBCu7Pjx4+zcuZODBw/y9NNPM3ToUL1hVkjcMGkaYxbZXyYCQQAi0taBfbcEoo0xf9k/Mwu4H9iTZZthwHxjzFH7sU45HnrhdejQIV544QUWLFhA9erVee655+jfvz+tWrXS+bGd6Eqrcv369ezcuZPmzZszatQoLUhSyOQ2uN0LGIztmfNlxphIEekDvAKUBJreYN93AseyLMcArbJtcw9QTETWYRsL+qkx5oc8nUEhkZ6eTmRkJPPmzWPKlCkUKVKE9957j+eff54SJUpYHZ5HM8YQFhbGli1bePTRR2nZsiXt2rXTP1CFVG4tzW+Bu4DtwGcicgRoA0wyxixwYN85XauYHI7fHOiMLRFvEZGtxpj9V+1IZCwwFqB69eoOHNr9paens3btWjZs2MDmzZvZtm0bSUlJAAwZMoQpU6ZQrZpDXcvqFly6dIkFCxaQlJTE4MGDPW5SN5V3uSXNAKCxMSZDRLyB00AdY0ycg/uOwZZ0r6iG7SZS9m1O26fRuCAi64EmwFVJ0xgzFZgKEBAQkD3xepRz587x7bff8p///IfDhw9TpEgRmjRpwsMPP0xgYCCBgYE69q8AXZnDaODAgRQtqlNqqdyT5mVjTAaAMSZFRPbnIWEChAF1RaQWcBwYgq0PM6tfgf+ISFFsj222wlYpvtCJjY3l448/5ssvv+T8+fN06NCBKVOm0KNHj0I10ZgrOXz4MGXLlqVz585Wh6JcSG5J814R2WV/LcDd9mUBjDGmcW47Nsakicg4YDm2IUffGWOiROQJ+/ovjTF7RWQZsAvIwDYsKfIWz8ntLF68mMGDB5OSksKDDz7ISy+9hL+/v9VhFWrGGBYvXkzv3r21QIm6Sm5J85YHnBljlgBLsr33ZbblKcCUWz2WO0pISGDq1Km89tpr+Pv7M2vWLJ1Hx0UcPXoUYww1atSwOhTlYnIr2KFFOpzk4MGDvPbaa8yfP59Lly4RHBzMTz/9pJfhLuDKs+P79u2jefPmOvZSXUN7tgtYamoqAwYM4NChQzz22GM89thjNG6ca0+HKgCRkZFEREQQExPDCy+8QPPmzbXAssqRJs0C9vHHH7Nr1y5++eUX+vXrZ3U4ClvLf9WqVXTt2pWBAwdSvHhxKlasaHVYykU5lDRFpCRQ3Rizz8nxeLTo6GjefPNN+vfvrwnTRaSnp1OrVi1Gjx6tLUvlkBs+0iAiwUAEsMy+7C8iC50cl0caN24cxYsX5/PPdYolV5CQkMBXX32FMUYTpnKYI8+BvYntOfJzAMaYCKCmswLyVH/++SfLly/n5Zdf5s4777Q6HAWsWrWKxo0b67PjKk8cSZppxphEp0fiwZKTk3n11Vfx8vJi1KhRVoejsA0pOnHiBK1aZS+HoFTuHOnTjBSRYYCXiNQFngE2OzcszxEbG0twcDDh4eG89957VKpUyeqQCrWUlBQuXbpEqVKl6NevH8WKFbM6JOVmHEmaTwOvApeAn7A94fOOM4PyJO+88w6RkZEsXLiQ4OBgq8MplIwxxMbGsnPnTqKioujQoQOtW7fG19fX6tCUG3IkadYzxryKLXGqPMjIyOCXX36hd+/emjAtkJGRwdmzZylfvjxLly6lbt26PPnkk3rTR90SR5LmxyJSBZgDzDLGRDk5Jo+xfft2YmNjGTBggNWhFDqXLl1i7ty5lC9fnl69ejFmzBirQ1Ie4oY3gowxQUBHbDNQThWR3SIy2dmBeYJPP/2UUqVK0bt3b6tDKVT+/vtvpk2bRtmyZenevbvV4SgP41DpaWNMnDHmM2xT+EYArzszKE8QFhbGrFmzeOGFF/Dx8bE6nELlzJkzNGnShD59+uhwIpXvbnh5LiL1gQeBB4AEYBa2SdbUdRhjmDhxIr6+vrz44otWh1OoREdHU7t2bWrWrGl1KMpDOdLSnAacBboZYzoYY/5PJ0DL3cqVK1m7di2TJ0/Wmw4F6MCBAyxatIjLly9bHYryYI7MRtm6IALxFMYYXnnlFWrUqMETTzxhdTiFxpEjR/j111954IEH8Pb2tjoc5cFym43yZ2PMYBHZzdUTojlUub2wmjdvHuHh4UyfPl1niSwAV6bV/euvvwgODtbLcuV0ubU0n7V/71MQgXiCtLQ0Jk+eTIMGDXjooYesDsejJSYmEh0dzbZt2xg2bBhBQUFWh6QKidwqt8faX/7DGDMx6zoR+QCYeO2nCrfNmzezb98+QkJC9K6tE6SlpVG0aFF27drFsmXLuPvuu+nSpYuOTlAFypHB7V25NkH2zOG9Qm/79u0AOnuhE2RkZDBjxgwCAwNp0KABfn5+OhWFskRufZpPAv8AameZlRLgNmCTswNzR1u2bKFWrVrcfvvtVoficTZu3AhAnTp1KFLEoeHFSjlFbi3Nn4ClwHvApCzvJxljzjg1KjeUkZHBb7/9Rp8+2gWc306cOMG2bdt4/PHHNWEqy+WWNI0x5rCIPJV9hYhU0MR5tcjISBISEvSGhBNUqlSJRx55RMe8KpeQ25/tn+zfw4Ed9u/hWZZVFmvXrgXQpJmPUlNT+emnn7h48SJ33HGH1eEoBeR+97yP/XutggvHfW3cuJGaNWtSvXp1q0PxCMYYFi5ciLe3t84Hr1yKIxOrtRWR0vbXD4nIxyKimSGbbdu20bq1PjyVXzZs2MDp06cJDg7Wu+TKpTjSq/5/wEURaQK8BBwBfnRqVG4mNjaWY8eO0bJlS6tDcXsJCQmkp6dTt25dHnroIZ2OQrkcRydWM8D9wKfGmE+xDTtSdlfGZ7Zo0cLiSNxbXFwc06ZNIy4ujipVqlC6dGmrQ1LqGo4Mbk8SkZeBh4H7RMQL0D//WWzZsoVixYrRvHlzq0NxW3FxccyYMYNevXrpFMfKpTnS0nwQ26Rqo40xccCdwBSnRuVmNm/eTLNmzShZsqTVobitvXv30qtXLxo0aGB1KErlypHpLuKAmUA5EekDpBhjfnB6ZG7i8uXLhIWFERgYaHUobscYw/bt2zl27BhBQUGaMJVbcOTu+WBgOzAIGAxsE5EHnB2Yu4iIiCAlJUWTZh6lpqayYMECwsPDte9SuRVH+jRfBVpcqdYuIrcDq4C5zgzMXWzbtg2ANm3aWByJe1m6dCmpqamMGTOG4sWLWx2OUg5zJGkWyTa9RQIOTshWGMTFxeHl5UXVqlWtDsWtBAUFUbJkSYoWdeS/oFKuw5Hkt0xElovISBEZCSwGljg3LPeRmJhI2bJldQC2g86dO8e8efMoU6aMJkzllhyZI+hFERkAtMM21cVUY8wvTo/MTfz999+UK1fO6jDcQkZGBgsWLKBu3br6R0a5rdzqadYFPgTuBnYDE4wxxwsqMHdx8uRJTZoOWrduHSKi/b/KreV2ef4dsAgYiK2y0ecFEpEb2b9/P6tWraJbt25Wh+IWKlSowMCBA7UmpnJruV2e32aM+dr+ep+I7CyIgNzJv/71L0qUKMH48eOtDsWlnT17lpiYGPz9/a0ORalbllvS9BaRptj6MQFKZl02xhTqJHrw4EFmzJjBM888Q6VKlawOx2UZY/jll1+oX7++1aEolS9yS5qxwMdZluOyLBugk7OCcgdfffUVIsKLL75odSgubffu3aSlpdGqVSurQ1EqX+RWhFhLkF9HamoqP/zwA71796ZKlSpWh+PSoqOj6dmzp/ZjKo+hA+VuwrJlyzh58iSjR4+2OhSXlp6ezoABA6wOQ6l85dQ//yLSQ0T2iUi0iEzKZbsWIpLuLs+0//DDD9xxxx307NnT6lBcVnx8PP/3f/9Henq61aEola+c1tK01938AugKxABhIrLQGLMnh+0+AJY7K5b8tnHjRnr06KFVxXOQmJjIqlWriI6Oplu3bnh5eVkdklL56oZJU2yPbgwHahtj3rLPD1TZGLP9Bh9tCUQbY/6y72cWturve7Jt9zQwD3CLsudxcXHExcXRtGlTq0NxKX/99RelS5emTJkyVK1ald69e+Pt7W11WErlO0cuz/8LtAGG2peTsLUgb+RO4FiW5Rj7e5lE5E6gP/ClA/tzCREREQCaNLOIi4tj7ty5XL58mdKlS9OmTRtNmMpjOZI0WxljngJSAIwxZwFHannl9HCxybb8CTDRGJNrx5eIjBWRHSKyIz4+3oFDO09UVBQAfn5+lsbhKpKTk5k9eza9evXirrvusjocpZzOkT7NVHu/o4HMepoZDnwuBsj6W1QNOJFtmwBglr14gy/QS0TSjDELsm5kjJkKTAUICAjInngL1N9//w1A+fLlrQzDZSQmJuLv70+jRo2sDkWpAuFI0vwM+AW4Q0TeBR4AJjvwuTCgrojUAo4DQ4BhWTcwxtS68lpEpgOLsidMV5OcnIy3t7dW6cH2VFStWrWoXLmy1aEoVWAcmSNoJrb5zt/D9pRQP2PMHAc+lwaMw3ZXfC/wszEmSkSeEJEnbi1s6yQnJ+sEatgmQvv1119JTk62OhSlCpQjd8+rAxeB0KzvGWOO3uizxpglZCtYbIzJ8aaPMWbkjfbnCjRpwokTJwgNDWX48OE6v48qdBy5PF+MrT9TAG+gFrAPaOjEuFzWlcvzwuzPP/+kb9++Oj+5KpQcqdx+1W1iEWkGPO60iFxcYW1pGmNYvXo1DRo0oFOnQl2rRRVyeX6M0l4Szi0GojvD0aNHueOOO6wOo8Bt3bqV/fv3U6FCBatDUcpSjvRpvpBlsQjQDLB2sKRFzp07x++//85rr71mdSgF6sCBA2zevJkxY8YU+q4JpRzp07wty+s0bH2c85wTjmtbv349GRkZBAUVrqp5Pj4+DBs2DB8fH6tDUcpyuSZN+6D2MsYYrbQLrFmzBm9vb1q3bm11KAUmNjaWMmXKcNttt914Y6UKgev2aYpIUfvjjc0KMB6XtnbtWgIDAylRooTVoRQIYwwLFiwgLi7O6lCUchm53Qi6UsUoQkQWisjDIjLgyldBBOdKzpw5w65duwrVpfmBAwcQEerUqWN1KEq5DEf6NCsACdjmBLoyXtMA850Yl8vZvXs3AC1aFI6BA8YYNmzYQPv27fWRUaWyyC1p3mG/cx7J/5LlFZYWzbBCZGQkAA0bev6YfmNs/7yDBg3SvkylssktaXoBZXCsxJvHi4qKoly5coXiKZjw8HASExPp3Lmz1aEo5XJyncLXGPNWgUXi4qKiomjYsKFHX6pmZGSwa9cu1qxZw5gxY6wORymXlNuNIM/NDnlkjMlMmp7IGIMxhsjISMLDwxk2bBgVK1a0OiylXFJuLU29NrPbunUrCQkJBAQEWB1KvouNjWXRokUEBQXh5+eHn5+fR7emlbpV121pGmPOFGQgruztt9/G19eX4cOHWx1Kvrp48SIhISE0b96cu+++GxHRhKnUDTh13nNPsGPHDpYuXcoLL7zgcbUjt27dip+fH82aNdNkqZSDnDbvuad45513KF++PE899ZTVoeSrjIwMOnbsmDm8SCnlGG1p5mL37t38+uuvPPvss5QtW9bqcPJNXFwcX331FQBeXl4WR6OUe9GkmYvFixcDeFQrMzU1lblz5xIYGEiRIvrPr1Re6W9NLvbu3UvVqlXx9fW1OpR8s2LFCqpWrUqTJk2sDkUpt6R9mrn4888/qV+/vtVh5KsWLVpQrlw5q8NQym1pS/M6jDHs3bvXY5Lm6dOnWb58ObfffnuhKW2nlDNo0ryO2NhYkpKSuPfee60O5ZbFx8fzww8/UKlSJR1apNQt0qR5HXv27AFw+5ZmYmIiP/zwA507d8bf39/qcJRye9qneR0hISGULFmSZs3cs3B9eno6p0+f5o477mDIkCGFojqTUgVBW5o5iI2NZcaMGYwePdotJxO7dOkS06dPZ+PGjYiIJkyl8pEmzRxMnz6d1NRUnn/+eatDuSnr16/Hx8eHAQMK3awkSjmdJs0c7Ny5kzp16nD33XdbHUqeZWRkcPz4cbp37643fZRyAu3TzEFkZKRb1s68Uhdz5MiRVoeilMfSlmY2ly5d4sCBA26ZNPfv38/s2bOtDkMpj6ZJM5v9+/eTnp7udkkzLS2NZcuW0apVK6tDUcqjadLM5sqsk40aNbI4krzZtm0blSpVcst+WKXcifZpZnPw4EEA6tata3EkeXPXXXe5/UB8pdyBJs1sTp48iY+PD97e3laH4rADBw5Qs2ZNihUrZnUoSnk8vTzP5uTJk1SqVMnqMBx24MABQkNDSU1NtToUpQoFTZrZxMXFUblyZavDcEhiYiK//vorAwcOpFSpUlaHo1ShoEkzG3dqae7bt482bdpQo0YNq0NRqtDQPs0sTp8+zeHDh7n//vutDiVX58+fJz4+npYtW1odilKFjrY0s/jyyy+5fPkyo0aNsjqUHGVkZLBx40b++9//cuzYMavDUapQ0pam3eXLl/niiy/o0aOHyw7d2bp1K/v27WPMmDFUrFjR6nCUKpQ0adrNnj2buLg4nnvuOatDua6AgAD8/f31po9SFtLLc2yFLj755BPq169Pt27drA7nGikpKcycORNAE6ZSFtOkiW2s486dO3nyySddrpyaMYaFCxdSvnx5ihcvbnU4ShV6mjSxzQUO0Lt3b4sjudbOnTtJSEhwyRawUoWRU5OmiPQQkX0iEi0ik3JYP1xEdtm/NotIE2fGcz0rVqygdu3a1K5d24rD56p8+fIMGjSIokW1+1kpV+C0pCkiXsAXQE+gATBURBpk2+wQ0MEY0xh4G5jqrHiuJzU1lbVr17pcSy41NZVt27ZRq1YtfH19rQ5HKWXnzJZmSyDaGPOXMeYyMAu4atS4MWazMeasfXErUM2J8eRo69atnD9/3qWS5uXLlwkNDSUmJsbqUJRS2Tgzad4JZB2BHWN/73rGAEtzWiEiY0Vkh4jsiI+Pz8cQbUONihUrRlBQUL7u92adOXOGzz77jIyMDPr06eNyN6aUKuyc2VGW02+7yXFDkSBsSbNdTuuNMVOxX7oHBATkuI+bkZCQwLRp03jooYcsn6rXGENCQgIVK1ZkxIgR3HHHHZbGo5TKmTNbmjHAXVmWqwEnsm8kIo2Bb4D7jTEJToznGl9++SUXL15k/PjxBXnYaxhjWLlyJQsXLgTQhKmUC3Nm0gwD6opILREpDgwBFmbdQESqA/OBh40x+50YyzVSUlL4/PPP6dmzp6XzARljWLVqFYcOHWLo0KF6Oa6Ui3Pa5bkxJk1ExgHLAS/gO2NMlIg8YV//JfA6UBH4rz1ZpBljApwVU1YzZ87k5MmTTJgwoSAOd10pKSnEx8fz8MMPU7JkSUtjUUrdmBiTb12EBSIgIMDs2LHjqvfWrVtHx44d87SfwMBALly4QEREhCWtu8TERLZu3UrXrl0pUkSfMVA3LzU1lZiYGFJSUqwOxS14e3tTrVq1a6aHEZFwRxpthXbEdHR0NP379y/whGmMYefOnaxevZrWrVsX6LGVZ4qJieG2226jZs2a2r1zA1duuMbExFCrVq2b2kehTJoXL14kPj6e6tWrF/ixjxw5Qnh4OI888ojbVIhXri0lJUUTpoNEhIoVK3IrQxcLZdK8UsC3IKeJMMbw119/Ubt2bR599FG9JFf5ShOm4271Z1Uof3OPHDkCUKAtzV27drFq1SqMMZowlccRER5++OHM5bS0NG6//Xb69OmTp/3UrFmT06dP3/I2zlQof3uPHj0KFFxL88KFC6xYsYLg4GBNmMojlS5dmsjISJKTkwFYuXIld96Z2wOA7qvQ/Qanpqby008/UbJkSapWrVogxwwLC8Pf37/AjqeUFXr27MnixYsBCAkJYejQoZnrzpw5Q79+/WjcuDGtW7dm165dAJllD5s2bcrjjz9O1tE8M2bMoGXLlvj7+/P444+Tnp5esCd0HYWqT9MYw7hx41i7di3Tpk27ZshBfouMjKRSpUq0a9cOLy8vpx5LKYDnnnuOiIiIfN2nv78/n3zyyQ23GzJkCG+99RZ9+vRh165djB49mg0bNgDwxhtv0LRpUxYsWMCaNWsYMWIEERER/POf/6Rdu3a8/vrrLF68mKlTbYXO9u7dy+zZs9m0aRPFihXjH//4BzNnzmTEiBH5em43o1AlzU8++YSpU6cyadIkRo4c6bTjxMXFsX79ek6ePKm1MFWh0bhxYw4fPkxISAi9evW6at3GjRuZN28eAJ06dSIhIYHExETWr1/P/PnzAVsR8PLlywOwevVqwsPDadGiBQDJycku83hxoflt3r59O+PHj2fgwIG8++67TjnGpUuXKFq0KL/++it+fn7069dPp6hQBcqRFqEz9e3blwkTJrBu3ToSEv5XSiKnh2iu3MXO6W62MYZHHnmE9957z3nB3qRC06f5448/4u3tzbRp0/L9ZkxcXBzfffcdixYtwsvLi7FjxxIYGKgJUxU6o0eP5vXXX8fPz++q99u3b585OeC6devw9fWlbNmyV72/dOlSzp61ldft3Lkzc+fO5dSpU4CtT/TKqBerFYqWpjGGBQsW0L17d2677bZ83XdUVBSLFy+ma9euNGlim61Dx8ypwqpatWo8++yz17z/5ptvMmrUKBo3bkypUqX4/vvvAVtf59ChQ2nWrBkdOnTIHAbYoEED3nnnHbp160ZGRgbFihXjiy++KNCx1ddTKJ49DwsLo2XLlnz//ff53pF85MgRSpYs6TL9Larw2bt3L/Xr17c6DLeS089Mnz3PYv78+Xh5eeV5oG1uTp8+ze+//07Xrl3zbZ9KKddXKPo0N2/eTMuWLalQoUK+7XP58uWULl063/anlHIPhSJpnjhxIl8fmTxw4ABnzpyhVatW+bZPpZR7KBRJMzY2lipVquTb/s6ePUv37t11wLpShZDH92kmJSVx4cKFfEmaxhji4uJo2bJlPkSmlHJHHt/SPHHCNpdbfiTNzZs3s2jRohwH6iqlCgePT5p//vknAHffffct7Sc6OpqtW7cyePBgHYepVDZeXl74+/vTqFEjgoODOXfuXL7sd/r06YwbNy5f9pVfPD5pbt++naJFi9K0adOb3ocxhk2bNvHAAw9Qrly5fIxOKc9QsmRJIiIiiIyMpEKFCnzxxRdWh+Q0hSJpNm7c+KZmekxLS2PDhg1cunSJESNGuMTTCEq5ujZt2nD8+HHA9vsXGBhI06ZNCQwMZN++fYCtBTlgwAB69OhB3bp1eemllzI/P23aNO655x46dOjApk2bMt8/cuQInTt3pnHjxnTu3DmzLu7IkSN58sknCQoKonbt2vz222+MHj2a+vXrO6Uwj8cnzT179lzzHKwjTp48yddff01MTAzp6el6Sa6UA9LT01m9ejV9+/YF4N5772X9+vX8/vvvvPXWW7zyyiuZ20ZERDB79mx2797N7NmzOXbsGLGxsbzxxhts2rSJlStXsmfPnsztx40bx4gRI9i1axfDhw/nmWeeyVx39uxZ1qxZw7///W+Cg4N5/vnniYqKYvfu3fleKs/j755nZGRQokSJPH3m0qVLzJgxg06dOuHv768JU7mVdevWsW7duszlsWPHAmTWqgTo2LEjHTt25KOPPiIpKQmw3Sx9/PHHCQ0NJTw8PHPb8ePH37BmQ3JyMv7+/hw+fJjmzZtnPimXmJjII488woEDBxARUlNTMz/TuXPnzO6uBg0acOTIEU6fPk3Hjh25/fbbAXjwwQfZv38/AFu2bMksI/fwww9f1ToNDg5GRPDz86NSpUqZDaWGDRty+PBh/P39Hf8B3oBHJ83k5GTOnz+fp3qWCQkJVKxYkSeffJJSpUo5MTqlnONKQszuzTffvOa98ePHX/NecHAwwcHBeTrmlT7NxMRE+vTpwxdffMEzzzzDa6+9RlBQEL/88guHDx++Kq6sjRkvLy/S0tIAxwveZN3uyr6KFCly1X6LFCmSud/84tGX59988w3nz59n0KBBuW6XlpbGgQMHWLhwIdOmTePixYuaMJW6CeXKleOzzz7jww8/JDU1lcTExMy5gqZPn37Dz7dq1SqzFmdqaipz5szJXBcYGMisWbMAmDlzJu3atXPKOdyIxybNlJQU3n//fdq3b5/jX11jDHv27CE9PZ29e/eyfv16fH19GTt2rCZMpW5B06ZNadKkCbNmzeKll17i5Zdfpm3btg7N8VOlShXefPNN2rRpQ5cuXWjWrFnmus8++4xp06bRuHFjfvzxRz799FNnnsZ1eWxpuKlTp/L444+zatUqOnfufNU6YwzLly/n0KFDDB8+nLJlyzo7bKWcRkvD5Z2WhsvBxo0bufPOO+nUqdNV7xtjWLFiBUePHmXUqFF4e3tbFKFSyh15bNKMjo7mnnvuuaZTWUTw9fWlffv2mjCVUnnmsX2a0dHR1KlT56r3tm/fnjkk4mYGuyullEcmzZSUFOLj46lZs2bme7Gxsaxbtw4fHx/L4lJKuT+PTJpXZrS7Uqk9NTWVefPm0aNHD02aSqlb4pFJ80qFlSsJMiUlhSZNmtC4cWPrglJKeQSPTJrx8fEAlC9fnqNHj1K0aFHuu+8+i6NSynOVKVMmx/dHjhzJ3Llzb2qfb775Jh9++OGthOUUHpc0jTH861//okyZMtSuXZvZs2dz5swZq8NSSnkIj0uaISEhLF++nHfeeYf169fTpk2bzMe4lFLOZYxh3LhxNGjQgN69e3Pq1KnMdeHh4XTo0IHmzZvTvXt3YmNjAfj6669p0aIFTZo0YeDAgVy8eNGq8B3iUUnTGMPLL79MixYt6Nu3L6VLl6Zt27ZWh6VUofHLL7+wb98+du/ezddff83mzZsB283Yp59+mrlz5xIeHs7o0aN59dVXARgwYABhYWH88ccf1K9fn2+//dbKU7ghjxrcfvToUY4ePcrTTz9NrVq1qFmzppZ1U4VOREREvk03AbYbqo6WVlu/fj1Dhw7Fy8uLqlWrZj6Rt2/fPiIjIzNLxqWnp2fO2xUZGcnkyZM5d+4c58+fp3v37vkWuzN4VNJcuXIlAHFxcZw9e5by5ctbHJFSBS8/a0fejJwaKsYYGjZsyJYtW65ZN3LkSBYsWECTJk2YPn36VbVAXZFHXZ4vXbqUChUq0KdPH02YSlmgffv2zJo1i/T0dGJjY1m7di0A9erVIz4+PjNppqamEhUVBdim2a5SpQqpqanMnDnTstgd5TEtzbS0NFasWIGfn59ldfaUKuz69+/PmjVr8PPzy5znB6B48eLMnTuXZ555hsTERNLS0njuuedo2LAhb7/9Nq1ataJGjRr4+fllVpJ3VR5TGq5cuXI0a9aMr776KrO8v1KFgZaGy7tbKQ3nMZfnCxYsAKBLly7WBqKU8mhOTZoi0kNE9olItIhMymG9iMhn9vW7RKRZTvu5EWMMM2fOpEqVKtSqVevWA1dKqetwWtIUES/gC6An0AAYKiINsm3WE6hr/xoL/N/NHCssLIyDBw/y4osv6hAjpZRTObOl2RKINsb8ZYy5DMwC7s+2zf3AD8ZmK+AjIlXycpDk5GSWLVsGkOcZ9JTyFO52b8JKt/qzcmbSvBM4lmU5xv5eXrfJ1f79+1m7di2VKlW6qn6mUoWFt7c3CQkJmjgdYIwhISHhlmZtcOaQo5yuk7P/qzqyDSIyFtvlO9WrV79qXY0aNZg7dy6tWrXK0/zmSnmKatWqERMTk1ndS+XO29ubatWq3fTnnZllYoC7sixXA07cxDYYY6YCU8E25CjrOh8fHwYOHJgf8SrllooVK6Y3QAuQMy/Pw4C6IlJLRIoDQ4CF2bZZCIyw30VvDSQaY2KdGJNSSt0Sp7U0jTFpIjIOWA54Ad8ZY6JE5An7+i+BJUAvIBq4CIxyVjxKKZUfnNoJaIxZgi0xZn3vyyyvDfCUM2NQSqn85HaPUYpIPHAkh1W+wOkCDsfZPPGcQM/LnXjiOUHO51XDGHP7jT7odknzekRkhyPPjboTTzwn0PNyJ554TnBr5+Uxz54rpVRB0KSplFJ54ElJc6rVATiBJ54T6Hm5E088J7iF8/KYPk2llCoIntTSVEopp3OrpFlQ9TkLmgPnNdx+PrtEZLOINLEizry40Tll2a6FiKSLyAMFGd/NcuS8RKSjiESISJSI/FbQMd4MB/4PlhORUBH5w35eLv8gioh8JyKnRCTyOutvLl8YY9ziC9tTRQeB2kBx4A+gQbZtegFLsRUCaQ1sszrufDqvQKC8/XVPVz8vR84py3ZrsD0A8YDVcefTv5UPsAeobl++w+q48+m8XgE+sL++HTgDFLc69hucV3ugGRB5nfU3lS/cqaVZIPU5LXDD8zLGbDbGnLUvbsVW2MSVOfJvBfA0MA84VZDB3QJHzmsYMN8YcxTAGOMO5+bIeRngNrFV+S6DLWmmFWyYeWOMWY8tzuu5qXzhTkmzQOpzWiCvMY/B9tfRld3wnETkTqA/8CXuw5F/q3uA8iKyTkTCRWREgUV38xw5r/8A9bFVIdsNPGuMySiY8JzmpvKFOxWgzLf6nC7G4ZhFJAhb0nT1OYodOadPgInGmHQ3mqLEkfMqCjQHOgMlgS0istUYs9/Zwd0CR86rOxABdALuBlaKyAZjzN9Ojs2ZbipfuFPSzLf6nC7GoZhFpDHwDdDTGJNQQLHdLEfOKQCYZU+YvkAvEUkzxiwokAhvjqP/B08bYy4AF0RkPdAEcOWk6ch5jQLeN7bOwGgROQTcC2wvmBCd4ubyhdWdtXno1C0K/AXU4n+d1Q2zbdObqzt2t1sddz6dV3Vs5fMCrY43v84p2/bTcY8bQY78W9UHVtu3LQVEAo2sjj0fzuv/gDftrysBxwFfq2N34Nxqcv0bQTeVL9ympWk8tD6ng+f1OlAR+K+9ZZZmXLiIgoPn5HYcOS9jzF4RWQbsAjKAb4wxOQ55cRUO/nu9DUwXkd3YksxEY4xLVz8SkRCgI+ArIjHAG0AxuLV8oU8EKaVUHrjT3XOllLKcJk2llMoDTZpKKZUHmjSVUioPNGkqpVQeaNJUDrFXIorI8lUzl23P58PxpovIIfuxdopIm5vYxzci0sD++pVs6zbfaoz2/Vz5uUTaqwD53GB7fxHplR/HVtbQIUfKISJy3hhTJr+3zWUf04FFxpi5ItIN+NAY0/gW9nfLMd1ovyLyPbDfGPNuLtuPBAKMMePyOxZVMLSlqW6KiJQRkdX2VuBuEbmmipGIVBGR9VlaYvfZ3+8mIlvsn50jIjdKZuuBOvbPvmDfV6SIPGd/r7SILLbXeowUkQft768TkQAReR8oaY9jpn3defv32VlbfvYW7kAR8RKRKSISZq+1+LgDP5Yt2As+iEhLsdU+/d3+vZ6IFAfeAh60x/KgPfbv7Mf5Paefo3IxVj/mpF/u8QWkYyvYEAH8gu3Ru7L2db7Ynqq4cuVy3v59PPCq/bUXcJt92/VAafv7E4HXczjedOyPVgKDgG3YCmHsBkpjK08WBTQFBgJfZ/lsOfv3ddhadZkxZdnmSoz9ge/tr4tjq3pTEhgLTLa/XwLYAdTKIc7zWc5vDtDDvlwWKGp/3QWYZ389EvhPls//C3jI/toH2zPqpa3+99av63+5zWOUynLJxhj/KwsiUgz4l4i0x/a44J3YnkmOy/KZMOA7+7YLjDERItIBaABssj8SWhxbCy0nU0RkMhCPrbpTZ+AXYyuGgYjMB+4DlgEfisgH2C7pN+ThvJYCn4lICaAHsN4Yk2zvEmgs/6soXw6oCxzK9vmSIhKB7RnncGBllu2/F5G62CrnFLvO8bsBfUVkgn3ZG1utgb15OAdVgDRpqps1HFsF7+bGmFQROYztFz6TMWa9Pan2Bn4UkSnAWWClMWaoA8d40Rgz98qCiHTJaSNjzH4RaY7tOeL3RGSFMeYtR07CGJMiIuuwlT57EAi5cjjgaWPM8hvsItkY4y8i5YBFwFPAZ9ie1V5rjOlvv2m27jqfF2CgMWafI/Eq62mfprpZ5YBT9oQZBNTIvoGI1LBv8zXwLbapB7YCbUXkSh9lKRG5x8Fjrgf62T9TGtul9QYRqQpcNMbMAD60Hye7VHuLNyezsBVruA9b0Qrs35+88hkRucd+zBwZYxKBZ4AJ9s+Uw1YJCGyX5FckYeumuGI58LTYm90i0vR6x1CuQZOmulkzgQAR2YGt1flnDtt0BCJE5Hds/Y6fGmPisSWREBHZhS2J3uvIAY0xO7H1dW7H1sf5jTHmd8AP2G6/TH4VeCeHj08Fdl25EZTNCmzzyawytukewFa7dA+wU2wTc33FDa7M7LH8AQwB/h+2Vu8mbP2dV6wFGly5EYStRVrMHlukfVm5MB1ypJRSeaAtTaWUygNNmkoplQeaNJVSKg80aSqlVB5o0lRKqTzQpKmUUnmgSVMppfJAk6ZSSuXB/wcsmuZDamSfMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(df_scores.fpr, df_scores.tpr, color='black', label='Model')\n",
    "plt.plot(df_rand.fpr, df_rand.tpr, color='black', lw=1,\n",
    "         linestyle='dashed', alpha=0.5, label='Random')\n",
    "plt.plot(df_ideal.fpr, df_ideal.tpr, color='black', lw=0.5,\n",
    "         linestyle='solid', alpha=0.5, label='Ideal')\n",
    "\n",
    "plt.legend()\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7GklEQVR4nO3dd3xUZdr/8c+VQAw9SJMiTWlBaQYUEBBRQGmGZtv1h/XZXbu4C7uuLvtYV9a+uurjKquLDQJIR0CkhZIEIiQgSBOQUEIPJCSTXL8/ZmBjTJlAZs6U6/165ZUp95zzPYRcuc8597mPqCrGGGO8E+F0AGOMCSZWNI0xphysaBpjTDlY0TTGmHKwommMMeVgRdMYY8rBiqYxxpSDFU3jcyKyS0SyRSRLRPaLyCQRqV6kTQ8R+UZETorIcRGZJSKxRdrUFJHXRWS3Z1nbPM/r+neLTDizomn8ZYiqVgc6AZ2BP559Q0S6A18DXwGNgBbAd8BKEWnpaRMFLAbaAwOBmkAP4DDQzVehRaSSr5ZtgpMVTeNXqrofWIC7eJ71MvCxqr6hqidV9Yiq/hlYDUzwtLkLaArEq+omVS1Q1YOq+qyqzi1uXSLSXkQWisgRETkgIn/yvD5JRJ4r1O46Edlb6PkuERknIhuAUyLyZxGZWmTZb4jIm57HtUTkXyKSISI/ichzIhJ5Yf9SJlBZ0TR+JSJNgJuAbZ7nVXH3GKcU0/xL4EbP4xuA+aqa5eV6agCLgPm4e6+X4+6peut2YBAQA3wC3CwiNT3LjgRGA5962v4bcHnW0RnoD9xXjnWZIGJF0/jLDBE5CewBDgJ/8bx+Me7/hxnFfCYDOHu8sk4JbUoyGNivqq+oao6nB7umHJ9/U1X3qGq2qv4IrANu8bx3PXBaVVeLSAPcfwQeU9VTqnoQeA24rRzrMkHEiqbxl1tUtQZwHdCW/xbDo0AB0LCYzzQEMj2PD5fQpiSXAtvPK6nbniLPP8Xd+wS4g//2MpsBlYEMETkmIseA94D6F7BuE8CsaBq/UtWlwCTg757np4BVwKhimo/mv7vUi4ABIlLNy1XtAS4r4b1TQNVCzy8pLmqR51OA6zyHF+L5b9HcA5wB6qpqjOerpqq29zKnCTJWNI0TXgduFJFOnufjgf8nIo+ISA0Rqe05UdMd+KunzSe4C1SCiLQVkQgRqSMifxKRm4tZx2zgEhF5TEQu8iz3as97qbiPUV4sIpcAj5UVWFUPAd8CHwE7VXWz5/UM3Gf+X/EMiYoQkctEpE85/01MkLCiafzOU4A+Bp72PF8BDACG4z5u+SPuEyrXquoPnjZncJ8M+h5YCJwA1uLezf/FsUpVPYn7JNIQYD/wA9DX8/YnuIc07cJd8L7wMvqnngyfFnn9LiAK2IT7cMNUyncowQQRsUmIjTHGe9bTNMaYcrCiaYwx5WBF0xhjysGKpjHGlIMVTWOMKYegm8Glbt262rx5c6djGGNCTEpKSqaq1iurnc+Kpoh8iPv634OqekUx7wvwBnAzcBoYo6rrylpu8+bNSU5Orui4xpgwJyI/etPOl7vnk3DPe1iSm4BWnq8HgH/6MIsxxlQInxVNVV0GHCmlyTDccyiqqq4GYkTErqIwxgQ0J08ENebnM8ns9bxmjDE+cerUKV599VXmz59/3stwsmhKMa8Ve02niDwgIskiknzo0CEfxzLGhJqsrCxefvllmjVrxtixY5kzZ855L8vJorkX95yHZzUB9hXXUFXfV9U4VY2rV6/Mk1vGGHPO5MmTad68OePGjaNhw4bMnTuXt95667yX5+SQo5nAQyLyOXA1cNwzzZYxxlSIVatWMWbMGLp168bo0aO56667qF279gUt05dDjj7DPUt3Xc9Nq/6Ce4ZrVPVdYC7u4UbbcA85uttXWYwx4efQoUOMHj2axo0b88ILL9CnT8VMceqzoqmqt5fxvgIP+mr9xpjwlZ+fzx133MHBgwd56qmn6Ny5c4UtO+iuCDLGmLL89a9/ZdGiRbzyyis88MADVK9evcKWbUXTGBNS5s6dy7PPPkvv3r157LHHiIio2PPdNmGHMSZk7Nq1izvvvJOmTZsybdq0Ci+YYEXTGBMizpw5w7Bhw8jPz2fRokXUqVPHJ+ux3XNjTEi499572bBhA1OmTKFVq1Y+W4/1NI0xQe+TTz5h8uTJjB07lpEjR/p0XVY0jTFBbe7cudx333306dOHl156yefrs6JpjAlaGzZs4L777qN27dp8/vnnVKrk+yOOdkzTGBOUMjIymDBhAgcOHGDJkiVccsklflmvFU1jTNBJS0vjlVdeYfr06bz88sv07t3bb+u23XNjTFA5c+YM8+fP55NPPuGWW27hySef9Ov6radpjAka69evJyIigtdee40WLVowadIk3Lcb8x8rmsaYoJCcnMxPP/3Em2++yZEjR5g7dy61atXyew7bPTfGBLycnBwaNGjA2rVr+eabb/jnP/9Jx44dHcliPU1jTEBbuXIlLpeLkydP8sILL3DfffcxZswYx/JY0TTGBKwffviBnJwcWrRowVVXXUXnzp0v6FYVFcGKpjEm4Kgq69evp0OHDjRp0oRrr70WgKlTpxIdHe1oNiuaxpiAoqosWrSI6OhoKlWqxIMPPsi6deuYOXMmLVu2dDqeFU1jTOBQVfLy8qhfvz4dO3bk448/5v3332f8+PEMGTLE6XiAnT03xgQIVWXevHlkZGTQsWNHNm7cyG9+8xv69u3Ls88+63S8c6xoGmMCwqJFi2jQoAHNmjXj+PHjjBgxgpiYGD777DO/TMThrcBJYowJSwUFBfz000/06tWL6OhoVJV77rmHHTt28O2339KgQQOnI/6M9TSNMY7Jz89nxowZHD169NxZ8ddee41p06bx8ssvnztrHkisp2mMccyePXto06YN7du3B2DFihX84Q9/YMSIETz++OMOpyue9TSNMX7ncrmYPXs2TZo0OVcwv/76a0aOHEnLli358MMP/T4Rh7esaBpj/MrlcpGQkEDr1q2pVKkSx44d495772XAgAHUrl2br776ipo1azods0RWNI0xfpObm4vL5aJv3760bt2aWbNm0b59e/7973/zpz/9ifXr19OuXTunY5bKiqYxxi/OnDnDlClTyMzMJCIigjvvvJOhQ4dSt25d1qxZw/PPP+/4JZLesKJpjPGLFStWcO2117Jq1SpiY2OZMmUKf/3rX0lKSuKqq65yOp7X7Oy5McansrOzSU9PJzY2locffpiEhATi4uJYvHgxV155pdPxys2KpjHGZ06fPs2UKVM4ceIEEyZM4NSpU/ztb3/jiSeeCKirfMojOFMbYwJeQUEBP/zwA5988gmLFy+mR48efPjhh7Rp08bpaBfEiqYxpsKdOHGCxx9/nClTppCfn88bb7zBgw8+SGRkpNPRLpgVTWNMhdq2bRvDhg1j06ZN9O3blw8++CAg5sGsKHb23BhTIQoKCnj55Zfp2LEju3fv5t1332XRokUhVTDBeprGmArwww8/cNddd7F69Wr69evHhx9+SNOmTZ2O5RPW0zTGnLf8/HxeeeUVOnToQFpaGu+88w4LFy4M2YIJ1tM0xpyn9PR07rnnHtauXUv37t1JSEigYcOGTsfyOetpGmPKJS8vj+eff54uXbqwbds2fvvb37JgwYKwKJhgPU1jTDmkpqZy9913k5qayvDhw3n99depU6cOVatWdTqa31jRNMaU6cyZMzz33HO89NJL1K1blw8++ICoqCjq16/PRRdd5HQ8v7Ldc2NMqdasWUOXLl147rnnuPPOO9m4cSMXX3wxo0aNCruCCdbTNMaUIDs7m2eeeYZXX32VRo0aMXfuXNq1a8exY8eIj493Op5jrKdpjPmF5cuX07FjR/7+979z//33k56eTps2bUhMTAzp4UTe8GnRFJGBIrJFRLaJyPhi3q8lIrNE5DsRSReRu32ZxxhTuqysLB5++GF69+6Ny+Vi8eLFvPvuu1SpUgURYdSoUURFRTkd01E+K5oiEgm8DdwExAK3i0hskWYPAptUtSNwHfCKiIT3T8QYhyxatIgrr7ySt99+m0cffZSNGzdy/fXXs2XLFubNm0eLFi2oXLmy0zEd58ueZjdgm6ruUNVc4HNgWJE2CtQQ923nqgNHAJcPMxljijh+/Dj3338/N954I1FRUSxfvpzXX3+datWqsX//fjZs2MCgQYOcjhkwfFk0GwN7Cj3f63mtsH8A7YB9wEbgUVUtKLogEXlARJJFJPnQoUO+ymtM2JkzZw7t27fnww8/ZNy4caSmptKzZ08Atm7dSkxMDCNGjAiJKd0qii+LZnE3LdYizwcAqUAjoBPwDxH5xb07VfV9VY1T1bh69epVdE5jws6uXbu47bbbGDx4MLVr12b16tW89NJLVKlSBXAPYt+0aRNRUVFERNj54sJ8+a+xF7i00PMmuHuUhd0NTFO3bcBOoK0PMxkT1o4dO8a4ceNo27YtM2fOZMKECaSkpNC1a9dzbVSV06dPM3ToUCuYxfDlOM0koJWItAB+Am4D7ijSZjfQD1guIg2ANsAOH2YyJizl5eXx3nvvMWHCBI4cOcJdd93Fc889R5MmTX7WLikpiejoaHr06OFQ0sDnsz8jquoCHgIWAJuBL1U1XUR+IyK/8TR7FughIhuBxcA4Vc30VSZjwo2qMnPmTK688koefvhhOnToQEpKCpMmTfpFwUxJSSEzM5MrrrjCobTBQVSLHmYMbHFxcZqcnOx0DGMCXkpKCk8++STffvstbdu2ZeLEiQwaNAj3YJWfO3jwIFWrVqVatWrFvh8ORCRFVePKamcHLIwJMXv27OGuu+4iLi6O9PR03nnnHTZs2MDgwYOLLYhLly4lLS2N6tWrh23BLA+79tyYEHHy5EleeuklXn31VVSV8ePHM378eGrVqlXiZw4dOoSIcP311/sxaXCznqYxQc7lcvHee+9x+eWX88ILLzBixAi2bNnCiy++WGLBVFW+/fZbqlatSu/evf2cOLhZT9OYIKWqzJ8/nyeffJJNmzbRq1cvZs+e/bPhQyV9bsGCBdSqVYtq1ar5KW3osJ6mMUHou+++o3///tx8883k5uYybdo0li5d6lXBzM3NJTY2lu7du/spbWixomlMENm3bx/33HMPnTt3Zt26dbzxxhukp6cTHx9f5kmcgoICZs2axfbt28N+ercLYbvnxgSBU6dOMXHiRCZOnIjL5eKJJ57gqaeeonbt2l4vY/Xq1TRt2pTY2KKTjZnysKJpTADLz8/n3//+N3/+85/JyMhg9OjRvPjii7Rs2bJcy/juu+/o3r27DSmqALZ7bkyAWrhwIV26dOHee++lWbNmJCYm8sUXX5S7YE6fPp3o6GgrmBXEiqYxASY9PZ2bb76Z/v37c/LkSb744gsSExPLfeJGVcnKyqJDhw62S16BrGgaEyAOHDjA//zP/9ChQwcSExP5+9//zubNmxk9enS5e4l5eXl8+eWXRERE0Lp1ax8lDk92TNMYh50+fZrXXnuNl156iZycHB566CGeeeYZ6tSpc17LU1WmT59OXFwcNWrUqOC0xoqmMX6mquzatYvly5ezYsUK5syZw759+4iPj+dvf/sbrVq1Ou9lnzlzhmPHjjF06FCio6MrMLU5y4qmMT6Wn59PWlrauSK5fPly9u1zz8cdExNDr169+Oyzzy74csacnBymTp1K7969rWD6kBVNYypYTk4OSUlJ54pkYmIix48fB6BJkyb06dOHa6+9ll69etG+ffsKmx198+bNXHfddb+YJ9NULCuaxlwAVeWnn34iKSmJNWvWsGLFCpKSksjNzQUgNjaW22677VyRbNq0aYUP/Tl9+jTLli1j4MCBFbpcUzwrmsaUQ2ZmJklJSSQlJZGcnExSUhL79+8HoFKlSsTFxfHII4/Qq1cvevToQd26dX2a59SpUyQkJDBgwACfrsf8lxVNY0pw8uRJUlJSzhXJpKQkdu3aBYCI0KZNG2688Ua6du1K165d6dix47m7OfrD6dOnqVSpEoMGDTrvM+2m/KxoGoN7N3v79u0sXryYlStXkpSUxJYtWzh7O5jmzZvTtWtXfve739G1a1e6dOlCzZq/uNu035w4cYLp06cTHx9vBdPPrGiasHXgwAEWL1587uvHH38EoEGDBnTr1o3bb7+drl27EhcXR7169RxO+3OLFy9myJAhjhbucGVF04SNEydOsHTp0nNFMi0tDXAP++nbty9/+MMf6NevH61btw7Y67SPHDnCgQMHuOWWWwI2Y6izomlC1pkzZ1i9ejWLFi1i8eLFrF27lvz8fKKjo7n22mv51a9+Rb9+/ejcuTORkZFOxy1TZmYms2fPZvjw4VYwHWRF04SMs1OgnS2Sy5cvJzs7m4iICLp27cq4ceO44YYb6N69e9AN/i4oKODkyZOMGDHCLo10mBVNE5TOXoqYlJTE2rVrSUpKYt26dWRlZQHu8ZH33XcfN9xwA3369Cn1joyBbv/+/SxduvS8Ju4wFc+KpgkKBw4cODfsZ+3atSQnJ5OZmQnARRddRKdOnRgzZgzXXHMNffv2pVGjRg4nrhgnT57k66+/ZuTIkVYwA4QVTRNwVJXvv/+euXPnsmrVKpKSkti9ezcAERERtG/fnqFDh54bH3nllVcSFRXlcOqKt2/fPmrVqsXtt99O5cqVnY5jPKxomoDgcrlYsWIFs2bNYubMmWzbtg2Ayy67jB49evDoo4+eGx8ZDred3b17N8uWLWPkyJFWMAOMFU3jmOPHjzN//nxmzZrF3LlzOXr0KFFRUfTr148nnniCwYMHc+mllzod0xG7du1i1KhRXHTRRU5HMUVY0TR+tXPnznO9yaVLl+Jyuahbty7Dhg1jyJAh9O/fn+rVqzsd0zHbt28nMzPzgqeJM77jddEUkWqqesqXYUxo2rdvH5988gmTJ09m48aNgPvs9tixYxk6dChXX311UIyT9LXt27eTlJTEiBEjnI5iSlFm0RSRHsAHQHWgqYh0BP5HVX/n63AmeOXk5DBr1iw++ugjFixYQEFBAT179uS1115jyJAhXHbZZU5HDCjHjh2jbt26jBw5kkqVbAcwkHnz03kNGADMBFDV70TE9h3ML6gqKSkpfPTRR3z22WccPXqUJk2a8Mc//pExY8Zw+eWXOx0xIKWnp7N582ZGjBhhw4qCgFd/0lR1T5EfZr5v4phgtH//fiZPnsxHH31Eeno60dHRDB8+nDFjxnD99dfbrncpTp06xbZt24iPj7eCGSS8KZp7PLvoKiJRwCPAZt/GMsEgOTmZ559/nlmzZpGfn88111zDe++9x+jRo4mJiXE6XsD77rvvaNy4McOGDXM6iikHb4rmb4A3gMbAXuBrwI5nhrHU1FT+8pe/MHPmTC6++GLGjh3LmDFjaNeundPRgkZKSgr79u3jyiuvdDqKKSdvimYbVb2z8Asi0hNY6ZtIJlClp6fzl7/8hYSEBGJiYnj22Wd55JFHbE7HcnK5XNSoUYPBgwfbLnkQ8qZovgV08eI1E6K2bNnChAkT+OKLL6hevTrPPPMMjz/+uO2Cn4dVq1adG0lgglOJRVNEugM9gHoi8kSht2oCdmQ/hKkqW7ZsYeXKlSxcuJApU6ZQpUoVxo8fz5NPPsnFF1/sdMSglJ6eTlZWFjfccIPTUcwFKK2nGYV7bGYloPAEfieAkb4MZfwrJyeH5ORkVq5cycqVK0lMTOTw4cMA1KlTh8cff5xx48YF3C0fgoWqsnnzZtq0aUNsbKztkge5Eoumqi4FlorIJFX90Y+ZjI/l5eWxZMkSFi5cyMqVK0lJSTl3n+7WrVszdOhQevbsSc+ePWnTpo39kl8AVeWbb76hcuXKxMbGOh3HVABvjmmeFpGJQHvg3HTXqnq9z1KZCudyuViyZAlffvkl06ZN48iRI0RFRREXF8ejjz5Kz5496dGjh/UmK1hOTg41a9aka9euTkcxFcSbojkZ+AIYjHv40f8DDvkylKkYZwvllClTmDZtGocPH6Z69eoMHTqU0aNH079/f7/epzucqCoLFiygU6dOVjBDjDdFs46q/ktEHi20y77U18HM+cnJyWHRokXMmDGDr776iszMzHOFctSoUQwYMMAKpR8sWLCAOnXqcMkllzgdxVQwb4pmnud7hogMAvYBTbxZuIgMxD0wPhL4QFVfKqbNdcDrQGUgU1X7eLNs81/Hjx9nzpw5zJgxg3nz5pGVlUXNmjUZNGgQo0aNYuDAgVYo/aSgoIAjR47Qs2dPuwFaiPKmaD4nIrWAsbjHZ9YEHivrQyISCbwN3Ij7SqIkEZmpqpsKtYkB3gEGqupuEalf7i0IYwsXLuSVV17hm2++IS8vj0suuYQ777yT+Ph4+vbtG5K3gAhkBQUFzJw5k5YtW9KhQwen4xgfKbNoqupsz8PjQF84d0VQWboB21R1h+cznwPDgE2F2twBTFPV3Z51HfQ+evjauXMnTzzxBDNmzKBp06Y89thjxMfHc/XVVxMREeF0vLC1detWLr/8cq644gqnoxgfKm1weyQwGvc15/NVNU1EBgN/AqoAnctYdmNgT6Hne4Gri7RpDVQWkW9xjwV9Q1U/LtcWhIn8/HzS0tJISEhg4sSJRERE8OKLL/L444/bLREcdvaEW79+/eyPVhgoraf5L+BSYC3wpoj8CHQHxqvqDC+WXdzgPi1m/VcB/XAX4lUislpVt/5sQSIPAA8ANG3a1ItVB7/8/HyWLFnC8uXLSUxMZM2aNZw8eRKA2267jYkTJ9KkiVeHlo0PuVwuEhIS6NSpkxXMMFFa0YwDOqhqgYhEA5nA5aq638tl78VddM9qgvskUtE2mZ7baJwSkWVAR+BnRVNV3wfeB4iLiytaeEPKsWPH+Ne//sU//vEPdu3aRUREBB07duTXv/41PXr0oEePHrRo0cLpmAb3RQKqSs+ePe0PWBgprWjmqmoBgKrmiMjWchRMgCSglYi0AH4CbsN9DLOwr4B/iEgl3JdtXo17pviwk5GRwauvvsq7775LVlYWffr0YeLEiQwcODCsbzQWqHJzc5k6dSrXX3+9FcwwU1rRbCsiGzyPBbjM81wAVdVSTw+qqktEHgIW4B5y9KGqpovIbzzvv6uqm0VkPrABKMA9LCntArcp6MyZM4fRo0eTk5PDrbfeyh/+8Ac6derkdCxTioULF9KjRw8bhxmGRLX4vV0RaVbaB526Hj0uLk6Tk5OdWHWFO3z4MO+//z5PP/00nTp14vPPP7f76AS4nJwctm/fbhNvhCARSVHVuLLalTZhh03S4SPbt2/n6aefZtq0aZw5c4YhQ4bw6aef2m54gMvOzj63S24FM3zZvUL9LC8vj+HDh7Nz507uv/9+7r//fhsIHQRUlUOHDnHDDTfQsGFDp+MYB1nR9LNXX32VDRs2MH36dG655Ran4xgvZGVlMWvWLEaPHm131jR4NbBMRKqISBtfhwl127ZtY8KECcTHx1vBDBK5ubkkJCRwww03WME0gBdFU0SGAKnAfM/zTiIy08e5QtJDDz1EVFQUb731ltNRjBeOHz/OmTNnGD16tM0zas7xpqc5Afd15McAVDUVaO6rQKHq+++/Z8GCBfzxj3+kcePGTscxZTh69CjTp0/H5XLZDFHmZ7wpmi5VPe7zJCEsOzubp556isjISO6++26n4xgvbNy4kVtuuYXatWs7HcUEGG9OBKWJyB1ApIi0Ah4BEn0bK3RkZGQwZMgQUlJSePHFF2nQoIHTkUwpMjMz2bx5M71793Y6iglQ3vQ0H8Z9f6AzwKe4p4h7zIeZQspzzz1HWloaM2fOZPz48U7HMaU4ePAgc+bMsauxTKlKvCLoXAORzqq63k95yhRMVwQVFBTQpEkTunfvTkJCgtNxTClOnTpFfn4+kZGRVKtWzek4xgHeXhHkTU/zVRH5XkSeFZH2FZAtbKxdu5aMjAyGDx/udBRTin379pGQkEDVqlWtYJoylVk0VbUvcB3uO1C+LyIbReTPvg4WCt544w2qVq3KoEGDnI5iSuByuVi2bBmjRo2iUiW71sOUzavB7aq6X1XfxH0L31TgGV+GCgVJSUl8/vnnPPHEE8TExDgdxxTjxx9/5MCBA9x66602rMh4zZvB7e1EZIKIpAH/wH3m3CYQLIWqMm7cOOrWrcvvf/97p+OYYuzcuZOVK1dSr149m3zDlIs3+yMfAZ8B/VW16MzrphgLFy5kyZIlvP7669SsWdPpOKaIgoICsrOzGTVqFJUrV3Y6jgkyZZ49DzSBfvZcVenatSuZmZls2bLFbnoWYLZu3cqOHTsYOHCg01FMgLng+TRF5EtVHS0iG/n5DdG8mrk9XCUkJJCSksKkSZOsYAaYPXv28N133xEfH+90FBPESpu5vaGqZpQ0g7vN3P5LLpeLK664gsjISDZs2GCz4gSQXbt20ahRIyIjI+3nYop1weM0VTXD8/B3qvpj4S/gdxUVNJQkJiayZcsWnn76afvFDCAbNmxg/fr1VKpUyX4u5oJ5M+ToxmJeu6mig4SCtWvXAtCvXz+Hk5iz8vPzyczMZNiwYXZfclMhSjum+VvcPcqWhe5KCVADWOnrYMFo1apVtGjRwuZeDBDJycnExMRw/fXXOx3FhJDShhx9CswDXgQKzzRxUlWP+DRVECooKGDp0qUMHjzY6SgG98UFhw4d4qqrrnI6igkxpRVNVdVdIvJg0TdE5GIrnD+XlpbG4cOH6du3r9NRwt7x48dp2bIlcXFxNnDdVLiyepqDgRTcQ44K/+9ToKUPcwWdJUuWAFjRdNjy5ctxuVz2czA+U9p9zwd7vrfwX5zgtWLFCpo3b07Tpk2djhK29u7di8vl4rrrrnM6iglh3lx73lNEqnke/0pEXhURqwxFrFmzhmuuucbpGGFJVUlMTKRevXr07dvXdsmNT3kzBuOfwGkR6Qj8AfgR+MSnqYJMRkYGe/bsoVu3bk5HCTuqysKFCwHsCizjF97eWE2BYcAbqvoG7mFHxuPs+MyuXbs6nCS8qCoul4vmzZvTo0cPp+OYMOFN0TwpIn8Efg3MEZFIwKaGKWTVqlVUrlzZhrf4kaoye/Zsdu3aRevWrZ2OY8KIN0XzVtw3VbtHVfcDjYGJPk0VZBITE+nSpYtNZOtHS5cupUmTJrRq1crpKCbMeHO7i/3AZKCWiAwGclT1Y58nCxK5ubkkJSXZ7qGfFBQU8P3339OrVy86d+7sdBwThrw5ez4aWAuMAkYDa0RkpK+DBYvU1FRycnKsaPpBfn4+06dPx+Vy2cQbxjHezNz+FNBVVQ8CiEg9YBEw1ZfBgsWaNWsA6N69u8NJQl9mZiaxsbG0a9fO6SgmjHlzTDPibMH0OOzl58LC/v37iYyMpFGjRk5HCVkul4uEhARq1aplBdM4zpue5nwRWYD7PkHgPjE013eRgsvx48epWbOmDaj2kYKCAqZOnUpcXBzR0dFOxzGm7KKpqr8XkeHAtbivP39fVaf7PFmQOHHiBLVq1XI6RkjKzc0lOzubm2++2W5QZwJGibvZItJKRL7y3Lp3FPCKqj5uBfPnDhw4YEXTB86cOcOXX37JiRMnrGCagFLasckPgdnACNwzHb3ll0RBZOvWrSxatIj+/fs7HSXkrFu3jj59+nDppZc6HcWYnylt97yGqv6f5/EWEVnnj0DB5IUXXuCiiy5i7NixTkcJGadPnyYpKYk+ffo4HcWYYpVWNKNFpDP/nUezSuHnqhrWRXT79u385z//4ZFHHqFBgwZOxwkJp06dYurUqQwYMMDpKMaUqLSimQG8Wuj5/kLPFQjrG6+89957iAi///3vnY4SEnJzc1FVbrrpJurXr+90HGNKVNokxDb1dQny8vL4+OOPGTRoEA0bNnQ6TtA7ceIE06ZNY/To0VYwTcCzQernYf78+Rw4cIB77rnH6ShBT1WZM2cOQ4YMoWrVqk7HMaZMPi2aIjJQRLaIyDYRGV9Ku64ikh8s17R//PHH1K9fn5tustu/X4ijR4+ya9cubr31VurUqeN0HGO84rOi6Zl3823gJiAWuF1EYkto9zdgga+yVLQVK1YwcOBAKle2aUXP1+HDh/nqq6+oXbs2ERG2w2OChzezHInn3kDPeJ43FRFv7uvQDdimqjtUNRf4HPfs70U9DCQAB4t5L+Ds37+f/fv327RkF0BVOXDgAMOHDycmJsbpOMaUizd/4t8BugO3e56fxN2DLEtjYE+h53s9r50jIo2BeOBdL5YXEFJTUwGsaJ6nAwcO8NVXXxEbG2tX+pig5E3RvFpVHwRyAFT1KBDlxeeKm8FCizx/HRinqvmlLkjkARFJFpHkQ4cOebFq30lPTwfgyiuvdDRHMDpy5AgLFizgxhtvdDqKMefNm1mO8jzHHRXOzadZ4MXn9gKFr4FrAuwr0iYO+NwzQ1Bd4GYRcanqjMKNVPV94H2AuLi4ooXXr06cOAFA7dq1nYwRdA4ePEhMTAyjR4+22YpMUPOmp/kmMB2oLyLPAyuAF7z4XBLQSkRaiEgUcBsws3ADVW2hqs1VtTnuSY1/V7RgBprs7Gyio6NtKrhy2LNnDwsWLKCgoMAKpgl63kwNN1lEUoB+uHe5b1HVzV58ziUiD+E+Kx4JfKiq6SLyG8/7QXMcs7Ds7Gy7gVo5qCrp6emMHj3a7ktuQkKZRVNEmgKngVmFX1PV3WV9VlXnUmTC4pKKpaqOKWt5gcCKpvd27NhBVlYWAwcOdDqKMRXGm2Oac3AfzxQgGmgBbAHa+zBXwDq7e25Kt337dtauXcvIkUFxvYIxXvNm9/xnp4lFpAvwPz5LFOCsp1m206dPU6NGDUaNGkWlSt78XTYmeJT7UgzPlHBdfZAlKOzevdsmlSjF5s2bmT9/PvXr17eCaUKSN8c0nyj0NALoAjg7WNIhx44dY/369Tz99NNORwlIR48eZfPmzQwbVtyFX8aEBm96mjUKfV2E+xhnWP5WLFu2jIKCAvr2tVnzikpPT6dSpUrEx8cTGRnpdBxjfKbUnqZnUHt1VbWZdoFvvvmG6OhorrnmGqejBJT169ezZ88e2rVrZ+NXTcgr7W6UlTyXN3bxY56AtmTJEnr06GHjDQspKChARBgyZIjNVmTCQmk9zbW4C2aqiMwEpgCnzr6pqtN8nC2gHDlyhA0bNvDss886HSVgrFmzhoiICLp2DdvzgiYMeXN682LgMO57Ap0dr6lAWBXNjRs3AliB8EhNTeXYsWN2+2ITdkormvU9Z87T+G+xPMvRSTOckJaWBkD79mE5pv9nduzYQWxsLB07drRjmCbslFY0I4HqeDfFW8hLT0+nVq1aNG7cuOzGIUpVWbJkCREREbRo0cIKpglLpd7CV1X/129JAlx6ejrt27cP60Jx8uRJoqOj6dGjh9NRjHFMaac7w7c6FHF2pp5w3TVXVb7++mvy8/OtYJqwV1rR7Oe3FAFu9erVHD58mLi4OKejOGL+/PnExMTYxMvGUMruuaoe8WeQQPbss89St25d7rzzTqej+JWqkpWVRdeuXalbt67TcYwJCDYauQzJycnMmzePJ554gmrVqjkdx28KCgqYOXMmu3btsoJpTCFWNMvw3HPPUbt2bR588EGno/jVd999R4sWLewGcsYUYXN3lWLjxo189dVXTJgwIWxuN5ufn09iYiLXXnttWI8UMKYk1tMsxZw5cwDCppfpcrlISEigXr16VjCNKYH1NEuxefNmGjVqFBbH9PLz88nLy6Nr1660aNHC6TjGBCzraZbi+++/p127dk7H8Lnc3Fy++OILTp8+bQXTmDJY0SyBqrJ58+awKJqzZ8+me/fu1KlTx+koxgQ82z0vQUZGBidPnqRt27ZOR/GZnJwcMjIyGDp0qN3PxxgvWU+zBJs2bQII2Z5mdnY2U6ZMoXLlylYwjSkH+20pwWeffUaVKlXo0iU0J67fvXs3/fr1o1GjRk5HMSaoWE+zGBkZGfznP//hnnvuISYmxuk4FerUqVNMmzaN1q1bW8E05jxY0SzGpEmTyMvL4/HHH3c6SoXKzs5m6tSpNnDdmAtgu+fFWLduHZdffjmXXXaZ01EqzMmTJ4mKiiI+Pj5srm4yxhesp1mMtLS0kJo789ixYyQkJJCdnW0F05gLZEWziDNnzvDDDz+EVNFcs2YNt9xyS8gdnzXGCbZ7XsTWrVvJz88PiaJ5+PBhdu7cyYABA5yOYkzIsJ5mEWfvOnnFFVc4nOTCHDp0iNmzZ9OmTRunoxgTUqynWcT27dsBaNWqlcNJzl9ubi4FBQWMGDGC6tWrOx3HmJBiPc0iDhw4QExMDNHR0U5HOS8ZGRl8+eWX1KtXzwqmMT5gRbOIAwcO0KBBA6djnJecnBwWL17MiBEjiIiwH60xvmC750Xs37+fSy65xOkY5bZnzx6qVq3KHXfcYQXTGB+y364igrGnuWvXLpYtW0b16tWtYBrjY/YbVkhmZia7du2iWbNmTkfxmqqSmZnJqFGjuOiii5yOY0zIs93zQt59911yc3O5++67nY7ilR9++IGffvqJ6667zukoxoQN62l65Obm8vbbbzNw4MCgmENz586drFu3jmuvvdbpKMaEFetpenzxxRfs37+fxx57zOkoZcrIyKBhw4aMHDmSyMhIp+MYE1asp4n7uODrr79Ou3bt6N+/v9NxSrVx40YSExOJioqygmmMA6xo4j42uG7dOn77298G9DyTubm5/PTTT8THx9tZcmMcYrvnwNdffw3AoEGDHE5SsnXr1lG/fn0GDhzodBRjwppPuysiMlBEtojINhEZX8z7d4rIBs9Xooh09GWeknz99de0bNmSli1bOrH6MiUlJbFv3z4aN27sdBRjwp7PiqaIRAJvAzcBscDtIhJbpNlOoI+qdgCeBd73VZ6S5OXlsWTJkoA9lpmdnU2jRo0YNGhQQB86MCZc+HL3vBuwTVV3AIjI58AwYNPZBqqaWKj9aqCJD/MUa/Xq1WRlZQVk0VyxYgUFBQX07t3b6SjGGA9f7p43BvYUer7X81pJ7gXmFfeGiDwgIskiknzo0KEKjOgealS5cmX69u1bocu9UNu2bSM3N5devXo5HcUYU4gve5rF7UtqsQ1F+uIumsWO1FbV9/HsusfFxRW7jPNx+PBhPvroI371q18FzK0gVJWUlBQ6derEZZddZrvkxgQYXxbNvcClhZ43AfYVbSQiHYAPgJtU9bAP8/zCu+++y+nTpxk7dqw/V1siVWXRokVUqVKFSpVsYIMxgciXv5lJQCsRaQH8BNwG3FG4gYg0BaYBv1bVrT7M8gs5OTm89dZb3HTTTQFxPyBVxeVy0aBBAzp06OB0HGNMCXxWNFXVJSIPAQuASOBDVU0Xkd943n8XeAaoA7zj2Q11qWqcrzIVNnnyZA4cOMCTTz7pj9WVSlWZO3cu7du3t4JpTIAT1Qo7ROgXcXFxmpycfMHL6dGjB6dOnSI1NdXx44YLFy7k4osv5qqrrnI0hzHhTERSvOm0he2Bs23bthEfH+9owSwoKGDv3r307t3b5sI0JkiE5QXMp0+f5tChQzRt2tSxDPn5+cyYMYPjx49bwTQmiIRlT3PPHvfwUSdnaN+7dy9t27YlNrboRVLGmEAWlj3NH3/8EcCRnqbL5WLWrFk0atTICqYxQSgsi+bu3bsB//c0XS4XU6dOpW3btlSuXNmv6zbGVIyw2z3Py8vj008/pUqVKjRq1Mhv683NzSU/P59+/fpRr149v63XGFOxwqqnqao89NBDLFmyhHfeecdvvb0zZ84wZcoUDh8+bAXTmCAXVkXz9ddf5/3332f8+PGMGTPGb+tduXIlvXr1okkTv0/iZIypYGGze7527VrGjh3LiBEjeP755/2yzuzsbDZu3Mj111/vl/UZY3wvbHqan3zyCdHR0Xz00Ud+ub/O6dOnmTJlCpdeemnZjY0xQSMsepqqyowZMxgwYAA1atTw+fry8/M5ffo0AwYMoEGDBj5fnzHGf8Kip5mcnMzevXuJj4/3+bpOnjzJ5MmTqVmzphVMY0JQWBTNadOmERkZyeDBg326nrOXRt58881ERUX5dF3GGGeExe55YmIi3bp14+KLL/bZOo4dO0ZOTg633367TSBsTAgLi57mvn37fHrJ5JEjR5gxYwbR0dFWMI0JcWHxG56RkUHDhg19tvydO3cSHx9PrVq1fLYOY0xgCPmiefLkSU6dOuWTonnw4EHWr1/PgAEDKnzZxpjAFPK75/v2ue/lVtFFMzMzk3nz5tGzZ88KXa4xJrCFfE/z+++/B+Cyyy6rsGUeOXKEGjVqMGrUKKpWrVphyzXGBL6Q72muXbuWSpUq0blz5wpZ3t69e5kzZw6AFUxjwlBYFM0OHTpQpUqVC15WQUEBKSkpjBo1ym5RYUyYCvnd802bNnHjjTde8HJ27dpFXl4ew4YNq4BUxphgFfI9zYKCggvuFW7fvp1Vq1Y5ek8hY0xgCOmeZnZ2NllZWRc04DwvL49KlSoxcuRIu0WFMSa0e5offPABWVlZjBo16rw+//333zNv3jyaNWtmBdMYA4Rw0czJyeGll16id+/eXHfddeX+/IEDB0hLS2PQoEEVH84YE7RCdvf8448/Zt++fXz88cfl/uyWLVto1qwZw4cP98uExcaY4BGyFWHFihU0bty43LeaSE1NZfPmzURFRVnBNMb8QshWhW3bttG6dWtExOvPqCrZ2dkMHTrUCqYxplghWxm2bdvG5Zdf7nX7tWvXsnHjRrp3724F0xhTopCsDjk5ORw6dIjmzZt71T4lJYUjR45w5ZVX+jaYMSboheSJoKNHjwJ4NVP7/v37adu2LV26dCnXrrwxJjyFZE/z2LFjAMTExJTYRlVZunQpmzdvplq1alYwjTFeCcme5qFDhwCoXbt2iW0OHz5MREQEffr08VcsY0wICLmepqrywgsvUL16dTp16lTs+0uWLKFKlSr06tXL/wGNMUEt5Hqan332GQsWLODNN9/8xX3HVZX58+dTu3ZtqlWr5lBCY0wwE1V1OkO5xMXFaXJycrHvqSrNmzenQYMGrFq1isjIyJ+9d+bMGQ4dOsSll17qr7jGmCAhIimqGldWu5DaPd+9eze7d+/m17/+9c8KZkFBAbNmzWLHjh1WMI0xFySkiubChQsBfnHp5Jo1a2jWrBmxsbFOxDLGhJCQOqY5b948mjRpcq445ufnk5qayjXXXGNDiowxFSJkepoul4tFixZx0003ISK4XC6mTZtGlSpVrGAaYypMyPQ0N27cyIkTJ+jbty+qyqlTp+jUqROtWrVyOpoxJoSETE9z7dq1AHTp0oUvvvgCEbGCaYypcD4tmiIyUES2iMg2ERlfzPsiIm963t8gIl3Od12ff/45jRs3JjU1lW7dulGzZs0LC2+MMcXwWdEUkUjgbeAmIBa4XUSKnr6+CWjl+XoA+Of5rGvDhg18++233H///QwbNoyWLVteQHJjjCmZL3ua3YBtqrpDVXOBz4GiNw0fBnysbquBGBFpWJ6VZGdnM2nSJMA91Cg6OvrCkxtjTAl8WTQbA3sKPd/rea28bUq1detWXnvtNerVq0f37t3PK6gxxnjLl0WzuHE+Ra/Z9KYNIvKAiCSLSPLZGYzOatasGVOnTmXdunUXdH9zY4zxhi+L5l6g8DWLTYB959EGVX1fVeNUNa5evXo/ey8mJoYRI0bQpEmTikltjDGl8GXRTAJaiUgLEYkCbgNmFmkzE7jLcxb9GuC4qmb4MJMxxlwQn+3PqqpLRB4CFgCRwIeqmi4iv/G8/y4wF7gZ2AacBu72VR5jjKkIPj0IqKpzcRfGwq+9W+ixAg/6MoMxxlSkoJtPU0QOAT8W81ZdINPPcXwtFLcJbLuCSShuExS/Xc1UtV5xjQsLuqJZEhFJ9mYC0WASitsEtl3BJBS3CS5su0Lm2nNjjPEHK5rGGFMOoVQ033c6gA+E4jaBbVcwCcVtggvYrpA5pmmMMf4QSj1NY4zxuaAqmv6cn9OfvNiuOz3bs0FEEkWkoxM5y6OsbSrUrquI5IvISH/mO1/ebJeIXCciqSKSLiJL/Z3xfHjxf7CWiMwSke882xXwF6KIyIciclBE0kp4//zqhaoGxRfuq4q2Ay2BKOA7ILZIm5uBebgnArkGWON07grarh5Abc/jmwJ9u7zZpkLtvsF9AcRIp3NX0M8qBtgENPU8r+907grarj8Bf/M8rgccAaKczl7GdvUGugBpJbx/XvUimHqafpmf0wFlbpeqJqrqUc/T1bgnNglk3vysAB4GEoCD/gx3AbzZrjuAaaq6G0BVg2HbvNkuBWqI+y6F1XEXTZd/Y5aPqi7DnbMk51Uvgqlo+mV+TgeUN/O9uP86BrIyt0lEGgPxwLsED29+Vq2B2iLyrYikiMhdfkt3/rzZrn8A7XDPQrYReFRVC/wTz2fOq14E0wSUFTY/Z4DxOrOI9MVdNK/1aaIL5802vQ6MU9X8ILrFsjfbVQm4CugHVAFWichqVd3q63AXwJvtGgCkAtcDlwELRWS5qp7wcTZfOq96EUxFs8Lm5wwwXmUWkQ7AB8BNqnrYT9nOlzfbFAd87imYdYGbRcSlqjP8kvD8ePt/MFNVTwGnRGQZ0BEI5KLpzXbdDbyk7oOB20RkJ9AWWOufiD5xfvXC6YO15TioWwnYAbTgvwer2xdpM4ifH9hd63TuCtquprinz+vhdN6K2qYi7ScRHCeCvPlZtQMWe9pWBdKAK5zOXgHb9U9ggudxA+AnoK7T2b3YtuaUfCLovOpF0PQ0NUTn5/Ryu54B6gDveHpmLg3gSRS83Kag4812qepmEZkPbAAKgA9UtdghL4HCy5/Xs8AkEdmIu8iMU9WAnv1IRD4DrgPqishe4C9AZbiwemFXBBljTDkE09lzY4xxnBVNY4wpByuaxhhTDlY0jTGmHKxoGmNMOVjRNF7xzESUWuireSltsypgfZNEZKdnXetEpPt5LOMDEYn1PP5TkfcSLzSjZzln/13SPLMAxZTRvpOI3FwR6zbOsCFHxisikqWq1Su6bSnLmATMVtWpItIf+LuqdriA5V1wprKWKyL/Braq6vOltB8DxKnqQxWdxfiH9TTNeRGR6iKy2NML3Cgiv5jFSEQaisiyQj2xXp7X+4vIKs9np4hIWcVsGXC557NPeJaVJiKPeV6rJiJzPHM9ponIrZ7XvxWROBF5CajiyTHZ816W5/sXhXt+nh7uCBGJFJGJIpLkmWvxf7z4Z1mFZ8IHEekm7rlP13u+txGRKOB/gVs9WW71ZP/Qs571xf07mgDj9GVO9hUcX0A+7gkbUoHpuC+9q+l5ry7uqyrO7rlkeb6PBZ7yPI4EanjaLgOqeV4fBzxTzPom4bm0EhgFrME9EcZGoBru6cnSgc7ACOD/Cn22luf7t7h7decyFWpzNmM88G/P4yjcs95UAR4A/ux5/SIgGWhRTM6sQts3BRjoeV4TqOR5fAOQ4Hk8BvhHoc+/APzK8zgG9zXq1Zz+edtXyV9BcxmlcVy2qnY6+0REKgMviEhv3JcLNsZ9TfL+Qp9JAj70tJ2hqqki0geIBVZ6LgmNwt1DK85EEfkzcAj37E79gOnqngwDEZkG9ALmA38Xkb/h3qVfXo7tmge8KSIXAQOBZaqa7Tkk0EH+O6N8LaAVsLPI56uISCrua5xTgIWF2v9bRFrhnjmncgnr7w8MFZEnPc+jcc81sLkc22D8yIqmOV934p7B+ypVzRORXbh/4c9R1WWeojoI+EREJgJHgYWqersX6/i9qk49+0REbiiukapuFZGrcF9H/KKIfK2q/+vNRqhqjoh8i3vqs1uBz86uDnhYVReUsYhsVe0kIrWA2cCDwJu4r9VeoqrxnpNm35bweQFGqOoWb/Ia59kxTXO+agEHPQWzL9CsaAMRaeZp83/Av3DfemA10FNEzh6jrCoirb1c5zLgFs9nquHetV4uIo2A06r6H+DvnvUUlefp8Rbnc9yTNfTCPWkFnu+/PfsZEWntWWexVPU48AjwpOcztXDPBATuXfKzTuI+THHWAuBh8XS7RaRzSeswgcGKpjlfk4E4EUnG3ev8vpg21wGpIrIe93HHN1T1EO4i8pmIbMBdRNt6s0JVXYf7WOda3Mc4P1DV9cCVwFrPbvJTwHPFfPx9YMPZE0FFfI37fjKL1H27B3DPXboJWCfuG3O9Rxl7Zp4s3wG3AS/j7vWuxH2886wlQOzZE0G4e6SVPdnSPM9NALMhR8YYUw7W0zTGmHKwommMMeVgRdMYY8rBiqYxxpSDFU1jjCkHK5rGGFMOVjSNMaYcrGgaY0w5/H/PfM+o7idMTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(df_scores.fpr, df_scores.tpr, color='black')\n",
    "plt.plot([0, 1], [0, 1], color='black', lw=0.7, linestyle='dashed', alpha=0.5)\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Scikit-Learn for plotting the ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuKUlEQVR4nO3de5xV5X3v8c+Pm8MdK6ACIqgIjhFGHY2gwIgaEBVELkbapua0LzXVtLY5ba4nzWltTprYJqa56CS1NokkRm4iKniJgIrKdVAQJYgodxhuAjMwzMzv/LHW4GbcM7NnmLXXvnzfr9e8Zl/W3uu3QL4+z3qe9Sxzd0REJDVt4i5ARCSbKDRFRJpBoSki0gwKTRGRZlBoiog0g0JTRKQZFJoiIs2g0JTImdlmM6s0s8NmttPMHjOzLvW2GWFmfzCzQ2Z20MyeNrPCett0M7MfmdlH4XdtDJ/3TO8RST5TaEq63OLuXYAi4FLg63VvmNlw4HngKaAPMBBYA7xmZueF23QAXgIuBsYB3YARwF7gyqiKNrN2UX23ZCeFpqSVu+8EFhKEZ53vA79y94fc/ZC773P3bwFvAN8Jt/kC0B+Y5O7vuHutu+92939x92eT7cvMLjazF8xsn5ntMrNvhK8/ZmYPJGxXYmZbE55vNrOvmtlbwBEz+5aZzaz33Q+Z2Y/Dx93N7L/MbIeZbTOzB8ys7an9SUmmUmhKWplZP+BGYGP4vBNBi/HJJJv/HrghfHw9sMDdD6e4n67Ai8ACgtbrBQQt1VTdAdwE9AB+DYw3s27hd7cFpgEzwm3/B6gO93Ep8Dngr5qxL8kiCk1Jl7lmdgjYAuwG/il8/U8I/jvckeQzO4C685VnNLBNQ24Gdrr7v7v70bAF+2YzPv9jd9/i7pXu/iGwCrg1fG8MUOHub5jZmQT/E7jf3Y+4+27gh8Dnm7EvySIKTUmXW929K1ACDOGTMNwP1AJnJ/nM2UB5+HhvA9s05Bzg/RZVGthS7/kMgtYnwHQ+aWWeC7QHdpjZATM7ADwC9D6FfUsGU2hKWrn7YuAx4MHw+RHgdWBqks2n8UmX+kVgrJl1TnFXW4DzG3jvCNAp4flZyUqt9/xJoCQ8vTCJT0JzC3AM6OnuPcKfbu5+cYp1SpZRaEocfgTcYGZF4fOvAX9hZn9jZl3N7PRwoGY48H/DbX5NEFCzzGyImbUxszPM7BtmNj7JPuYDZ5nZ/WZ2Wvi9nw3fKyM4R/knZnYWcH9TBbv7HmAR8N/AB+6+Pnx9B8HI/7+HU6LamNn5Zja6mX8mkiUUmpJ2YQD9Cvg/4fNXgbHAbQTnLT8kGFC5xt3/GG5zjGAw6F3gBeBjYBlBN/9T5yrd/RDBINItwE7gj8C14du/JpjStJkg8J5IsfQZYQ0z6r3+BaAD8A7B6YaZNO9UgmQR0yLEIiKpU0tTRKQZFJoiIs2g0BQRaQaFpohIMyg0RUSaIetWcOnZs6cPGDAg7jJEJMesXLmy3N17NbVdZKFpZo8SXP+7290/k+R9Ax4CxgMVwJ3uvqqp7x0wYAArVqxo7XJFJM+Z2YepbBdl9/wxgnUPG3IjMCj8uQv4eYS1iIi0ishC092XAPsa2WQiwRqK7u5vAD3MTFdRiEhGi3MgqC8nrySzNXxNRCQSpaWllJSUcP/997f4O+IcCLIkryW9ptPM7iLowtO/f/8oaxKRHFRaWsqMGTNYvHgxAEVFRS3+rjhDcyvBmod1+gHbk23o7qVAKUBxcbEulheRlJWWlnL33XcDMHjwYO6++27+7u/+rsXfF2f3fB7wBQtcBRwMl9kSEWkViYH55S9/mddff/2UAhOinXL0W4JVunuGN636J4IVrnH3h4FnCaYbbSSYcvTFqGoRkfyTGJjf/OY3eeCBB5r4RGoiC013v6OJ9x24N6r9i0j+SgzMO++8k3/8x39ste/OuiuCRESaMmNGsE7097//fb70pS/RpUuXVvtuhaaI5JTS0lIWL17MkCFD+MpXvkKbNq07dKMFO0QkZyR2y++9995WD0xQS1NEckD9eZg///nPueeeeyLZl0JTRLJaYutyyJAh3HfffZEFJig0RSTL1Q36fP3rX+c73/kOHTp0iHR/OqcpIlmp7jrylStXMmzYML773e9GHpig0BSRLDVjxgxWrVrFOeecc6J7ng7qnotIVqqqqmLYsGG8/PLLtGuXvihTS1NEss7atWs5ePAgbdq0SWtggkJTRLLMsWPH2LJlCz179iS4a056KTRFJGusXr2a8vJytmzZwpIlS2KpQaEpIllhxYoVbNu2jT59+pyYZjR9+vS016GBIBHJeEePHuXMM8/k8ssvP9ElHz16NHfddVfaa1FoikhGe+2116iurmb06NFxlwKoey4iGeyPf/wjR48eZdSoUSdeq1vFKC5qaYpIxnF3Vq9ezdChQ7ngggtOGiWP83wmKDRFJMO4Oy+++CIFBQWfmoNZ18qM63wmKDRFJIO4O8ePH6d3794MGzbspPcSVzOKq5UJOqcpIhnC3XnuuefYsWNHo4H5yCOPxNbKBIWmiGSIF198kTPPPJNzzz33pNczKTBB3XMRiVltbS3btm1j5MiRFBQUnPRepgUmqKUpIjGqqalh7ty57N+/PysCE9TSFJEYbdmyhcGDB3PxxRcDn9zrBzgxFzOTAhMUmiISg+rqahYsWMC4ceNo167dp26MNnr0aEaPHs306dMzKjBBoSkiaVZdXc2sWbO49NJLTwRmXTc8U4MykUJTRNKmqqqK2tparr32Wnr37g18coVPpnXDG6KBIBFJi2PHjvHkk09SXl5+IjDrxHmFT3MpNEUkLV599VWuueYa+vXrd+JOkiUlJZSVlcVdWrOoey4ikaqsrGTdunWMGTMGM/vUOcyioqJYL4tsLoWmiESmoqKCmTNncv311/OLX/zipBHybDmHWZ9CU0QiUVtby5EjR9i1axfTp08/aTpRpo+QN0ahKSKt7tChQ8ydO5fbb7+dZ555hrKysqwPyzoKTRFpVTU1NcyZM4fx48fz2GOPnVj/ctGiRXGX1io0ei4irebAgQPs3buXO+64g9mzZ2fE+petTaEpIq1i3759zJ07lw4dOtC+ffusm7SeKnXPRaRVPPjggyxZsoTHHnsM4MR5zFwKTFBLU0RO0Z49e3j++edZunQpa9euPfF6ts2/TJVamiLSYuXl5Tz77LMcPHgw5wZ8GqLQFJEW+eEPf8icOXNwd1599VUgtwZ8GqLuuYg027Zt2ygtLWXNmjW0bduW0aNH59yAT0PU0hSRZqmtreWBBx7g3XffzYvueH0KTRFJSWlpKY8++ijuzrJly4D86I7Xp9AUkQYlu2fPyJEjc+aSyJaINDTNbBzwENAW+KW7f6/e+92B3wD9w1oedPf/jrImEWlcQ0F55ZVXcuedd/KlL30pzvJiF1lomllb4KfADcBWYLmZzXP3dxI2uxd4x91vMbNewHtm9ri7V0VVl4gk19DNzcaMGUNRURETJkyIucLMEGVL80pgo7tvAjCz3wETgcTQdKCrmRnQBdgHVEdYk4jUkyws67reO3fu5JVXXuGmm26KucrMEWVo9gW2JDzfCny23jY/AeYB24GuwO3uXlv/i8zsLuAugP79+0dSrEi+mjFjRtKl2zZs2ED//v2ZPHkybdpodmKdKEPTkrzm9Z6PBcqAMcD5wAtm9oq7f3zSh9xLgVKA4uLi+t8hIi1Q18IsKyujqKjopKlDZWVlbN68mQsuuECBWU+UobkVOCfheT+CFmWiLwLfc3cHNprZB8AQYFmEdYnkpcQBHuBT3fE67k5FRQUTJkxQYCYRZWguBwaZ2UBgG/B5oP6kro+A64BXzOxMYDCwKcKaRPJS/ZuZ1f2uP21o+fLlFBQUMGLEiFjqzAaRhaa7V5vZfcBCgilHj7r7OjO7J3z/YeBfgMfM7G2C7vxX3b08qppE8lUqa1uuXLmS8vJyxo0bl87Sso4FPePsUVxc7CtWrIi7DJGMVr8rnuy8ZaLdu3fTqVMnOnfuTDCZJf+Y2Up3L25qO52wEMkxdV3xunOW0PjalosXL2bt2rV06dIlbwOzOXQZpUgOSTx3mcqqQ3v27MHMGDNmTDrKywlqaYrkkFTvy+PuLFq0iE6dOjFq1Kh0lZcTFJoiOaap+/K4OwsXLuS0006jc+fOaawsNyg0RXJEaWnpSecxk3F3qqqqKCwsZPjw4WmqLLfonKZIlqt/7XhDAz61tbXMnz+fCy64gMLCwnSWmFMUmiJZrqFrx+t744036N+/vwLzFCk0RbJYXZe8sdtO1NTUsGbNGoYPH64pRa1A5zRFslBpaSklJSUnphc11CWvqalhzpw5FBQUKDBbiVqaIlmksbUv63N3Dh8+zNChQ7nwwgvTXWrOUmiKZIHmhCXA8ePHmT17NuPHj1dgtjKFpkiGq79CUVM3NHN35syZQ3FxMV27dk1XmXlDoSmSJvUX0UhVXesylcsijx07xoEDB5gwYQIFBQUtqlMap9AUiUCygEzsWjdHqrfLPXr0KDNnzmTUqFEKzAgpNEVaWbIFf+seR3mv8PXr11NSUkK/fv0i+X4JKDRFTlFDt5FIpTvdGioqKliyZIkWD04ThaZIM6TS7Y66RZnoyJEjzJo1i7Fjx0a+LwkoNEWSaGjQJtl5yXSGZKKKigratWvHTTfdxBlnnJHWfeczhaZIKDEoGxq0iSsg6/v444+ZM2cOkyZNUmCmmUJT8k4qrchMCceGvPTSS9xyyy1069Yt7lLyjkJT8kpDI9t1zzM5KAH27dvHrl27uPXWW3UteUwUmpLzknW70zWy3ZrKy8uZP38+t912mwIzRgpNyVnJrtfOhtZkMrW1tRw6dIjJkyfr0siYKTQlqzV2aWKqi1tkup07d7J48WKmTZumFmYGUGhKVkrWiqwv28MS4NChQzz//PNMmTJFgZkhFJqSkZpa3CJXWpGN2b59O927d+eOO+6gffv2cZcjIYWmxCbVrnUyuRyWAB999BFLlixhypQpCswMo9CUtMuXrvWp2Lx5M1OnTuW0006LuxSpR6EpadXcBXXzzfvvv095eTmjRo2KuxRpQMqhaWad3f1IlMVIbmlscYtsnCcZtffff5/ly5czefLkuEuRRjR5N0ozG2Fm7wDrw+fDzOxnkVcmWa2uRVkXknVGjx6twEziwIED9OzZU+cws0AqLc0fAmOBeQDuvsbM1HeQT8mVK2/Sbd26daxfv57JkydrWlEWSKl77u5b6v1l1kRTjmSjXLryJt2OHDnCxo0bmTRpkgIzS6QSmlvMbATgZtYB+BvCrrrkt+beVlZOtmbNGvr27cvEiRPjLkWaIZXQvAd4COgLbAWeB/46yqIk82kU/NSsXLmS7du3c8kll8RdijRTKqE52N3/NPEFM7saeC2akiQb1J271DnL5quurqZr167cfPPN6pJnoVRC8z+By1J4TXJQQ1ftlJWVMXr0aAVmM73++uvU1tZy9dVXx12KtFCDoWlmw4ERQC8z+/uEt7oBbaMuTOLV1FU7RUVFTJ8+PY7Ssta6des4fPgw119/fdylyClorKXZAegSbpO4gN/HwJQoi5J4NHSPHJ2vPDXuzvr16xk8eDCFhYXqkme5BkPT3RcDi83sMXf/MI01ScRy4R452cLd+cMf/kD79u0pLCyMuxxpBamc06wwsx8AFwMFdS+6+5jIqpJWVT8kM/1Oi7nk6NGjdOvWjSuuuCLuUqSVpBKajwNPADcTTD/6C2BPlEVJ60l2IzGFY/TcnYULF1JUVKTAzDGphOYZ7v5fZva3CV32xU1+SmLRUKtSU4PSa+HChZxxxhmcddZZcZcirSyV0Dwe/t5hZjcB24F+qXy5mY0jmBjfFvilu38vyTYlwI+A9kC5uydfdVYa1NAATt1vtSrTp7a2ln379nH11VfrBmg5KpXQfMDMugNfIZif2Q24v6kPmVlb4KfADQRXEi03s3nu/k7CNj2AnwHj3P0jM+vd7CPIY7rmO7PU1tYyb948zjvvPIYOHRp3ORKRJkPT3eeHDw8C18KJK4KaciWw0d03hZ/5HTAReCdhm+nAbHf/KNzX7tRLlxkzZpyYZK6gjN+GDRu44IIL+MxnPhN3KRKhxia3twWmEVxzvsDd15rZzcA3gI7ApU18d19gS8LzrcBn621zIdDezBYRzAV9yN1/1awjyCP1z1eWlZVRVFTEokWL4itKqK6u5uWXX+a6666jTZsml6iVLNdYS/O/gHOAZcCPzexDYDjwNXefm8J3J5vB60n2fzlwHUEQv25mb7j7hpO+yOwu4C6A/v37p7Dr7JbKPErQVTmZoLq6mlmzZlFUVKTAzBONhWYxMNTda82sACgHLnD3nSl+91aC0K3Tj2AQqf425eFtNI6Y2RJgGHBSaLp7KVAKUFxcXD94c0qyKUJ11A3PLMePH8fdufrqq+nXL6WxUckBjYVmlbvXArj7UTPb0IzABFgODDKzgcA24PME5zATPQX8xMzaEVy2+VmCleLzUmJgaopQZquqqmLmzJmMGTNGgZlnGgvNIWb2VvjYgPPD5wa4uzc6POju1WZ2H7CQYMrRo+6+zszuCd9/2N3Xm9kC4C2glmBa0tpTPKasoltEZKcXXniBESNGaB5mHjL35L1dMzu3sQ/GdT16cXGxr1ixIo5dR6KkpOTEgA6g7neGO3r0KO+//74W3shBZrbS3Yub2q6xBTu0SEdEEluXGgHPHpWVlSe65ArM/JXyfc+lddQf6NEIeHZwd/bs2cP111/P2WefHXc5EiOFZprpNhHZ5/Dhwzz99NNMmzaNtm21/na+S2limZl1NLPBUReT60pLS1m8eLFuE5FFqqqqmDVrFtdff70CU4AUQtPMbgHKgAXh8yIzmxdxXTmprpWp7nh2OHjwIMeOHWPatGn06tUr7nIkQ6TS0vwOwXXkBwDcvQwYEFVBuUqtzOyyf/9+5syZQ3V1NR07doy7HMkgqYRmtbsfjLySHKdWZnZ5++23ufXWWzn99NPjLkUyTCoDQWvNbDrQ1swGAX8DLI22rNxRN71It7zNDuXl5axfv55Ro0bFXYpkqFRaml8muD/QMWAGwRJx90dYU06pC0xNLcp8u3fv5plnnjlxoYFIMqm0NAe7+zeBb0ZdTK7S5PXMd+TIEQoKCpgyZQqdO3eOuxzJYKm0NP/DzN41s38xs4sjryiH1A3+SGbbvn07s2bNolOnTgpMaVKToenu1wIlBHegLDWzt83sW1EXlu0Sr/xRtzxzVVdXs2TJEqZOnUq7drrWQ5qW0uR2d9/p7j8muIVvGfDtKIvKBbryJ/N9+OGH7Nq1i9tvv13TiiRlqUxuv8jMvmNma4GfEIycawHBRmhOZub74IMPeO211+jVq5cW35BmSaU/8t/Ab4HPuXv9ldelHnXLM19tbS2VlZVMnTqV9u3bx12OZJlU7kZ5VToKyRXqlme2DRs2sGnTJsaNGxd3KZKlGrsb5e/dfZqZvc3JN0RLaeX2fKZueWbasmULa9asYdKkSXGXIlmssZbm34a/b05HISJR2rx5M3369OG2227TakVyShocCHL3HeHDv3b3DxN/gL9OT3nZRfMyM9Nbb73F6tWradeunQJTTlkqU45uSPLaja1dSC7QohyZp6amhvLyciZOnKj7kkuraOyc5pcIWpTnJdyVEqAr8FrUhWUrnc/MHCtWrKBHjx6MGTMm7lIkhzT2v94ZwC3AvPB33c/l7v5naagtq6hrnlmWL1/O7t27Of/88+MuRXJMYwNB7u6bzeze+m+Y2Z+4+74I68o66ppnjoMHD3LeeedRXFysievS6hoLzRkEI+crCaYcJf7X58B5EdaVldQ1j98rr7xCdXU11157bdylSI5q7L7nN4e/B6avHJGW27p1K9XV1ZSUlMRdiuSwVK49v9rMOoeP/8zM/sPM+kdfmkhq3J2lS5fSq1cvrr32WnXJJVKpzMH4OVBhZsOAfwQ+BH4daVVZRoNA8XF3XnjhBQBOO+20mKuRfJDqjdUcmAg85O4PEUw7kpAGgeLh7lRXVzNgwABGjBgRdzmSJ1IJzUNm9nXgz4FnzKwtoKVh6tEgUHq5O/Pnz2fz5s1ceOGFcZcjeSSV0Lyd4KZq/8vddwJ9gR9EWpVIExYvXky/fv0YNGhQ3KVInkllabidZvY4cIWZ3Qwsc/dfRV+ayKfV1tayYcMGRo4cqevIJRapjJ5PA5YBU4FpwJtmNiXqwrKFBoHSp6amhjlz5lBdXa3AlNiksnL7N4Er3H03gJn1Al4EZkZZWLbQIFD6lJeXU1hYyEUXXRR3KZLHUjmn2aYuMEN7U/xc3tAgULSqq6uZNWsW3bt3V2BK7FJpaS4ws4UE9wmCYGDo2ehKEvlEbW0tM2fOpLi4mIKCgrjLEUlpIOgfzOw24BqC689L3X1O5JVJ3quqqqKyspLx48fTrVu3uMsRARrpZpvZIDN7Krx171Tg39397xSYkg7Hjh3j97//PR9//LECUzJKY+cmHwXmA5MJVjr6z7RUJAKsWrWK0aNHc84558RdishJGgvNru7+C3d/z90fBAakqaasUFpaSklJCWVlZXGXklMqKipYvHgxw4cPV2BKRmrsnGaBmV3KJ+todkx87u6roi4uk82YMYOysjKKioo03aiVHDlyhJkzZzJ27Ni4SxFpUGOhuQP4j4TnOxOeO5D3N14pKipi0aJFcZeRE6qqqnB3brzxRnr37h13OSINamwRYi19LWnx8ccfM3v2bKZNm6bAlIynSeotoEsnW4+788wzz3DLLbfQqVOnuMsRaVKkoWlm48zsPTPbaGZfa2S7K8ysJluuadelk61j//79bN68mdtvv50zzjgj7nJEUhJZaIbrbv4UuBEoBO4ws8IGtvs3YGFUtURBl06emr179/LUU09x+umn06aNOjySPVJZ5cjCewN9O3ze38yuTOG7rwQ2uvsmd68Cfkew+nt9XwZmAbuTvCc5yN3ZtWsXt912Gz169Ii7HJFmSeV/8T8DhgN3hM8PEbQgm9IX2JLwfGv42glm1heYBDycwvdlBJ3PPDW7du3iqaeeorCwUFf6SFZKJTQ/6+73AkcB3H0/0CGFzyW7JaDXe/4j4KvuXtPoF5ndZWYrzGzFnj17Uth1dHQ+s+X27dvHwoULueGGG+IuRaTFUlnl6Hh43tHhxHqatSl8biuQeElHP2B7vW2Kgd+Ft1ztCYw3s2p3n5u4kbuXAqUAxcXF9YM37XQ+s/l2795Njx49mDZtmlYrkqyWSkvzx8AcoLeZ/SvwKvDdFD63HBhkZgPNrAPweWBe4gbuPtDdB7j7AIJFjf+6fmBK9tuyZQsLFy6ktrZWgSlZL5Wl4R43s5XAdQRd7lvdfX0Kn6s2s/sIRsXbAo+6+zozuyd8P2vOY0rLuTvr1q1j2rRpui+55IQmQ9PM+gMVwNOJr7n7R0191t2fpd6CxQ2Fpbvf2dT3SXbZtGkThw8fZty4cXGXItJqUjmn+QzB+UwDCoCBwHvAxRHWJVnu/fffZ9myZUyZkhXXK4ikLJXu+SWJz83sMuDuyCqSrFdRUUHXrl2ZOnUq7dql8v9lkezR7EsxwiXhroigFskB69evZ8GCBfTu3VuBKTkplXOaf5/wtA1wGRDvZEnJSPv372f9+vVMnJjswi+R3JBKS7Nrws9pBOc49a9CTrJu3TratWvHpEmTaNu2bdzliESm0ZZmOKm9i7v/Q5rqyWh1l1COHj067lIyyurVq9myZQsXXXQR4YUKIjmrsbtRtgsvb7wsjfVkNF1C+Wm1tbWYGbfccotWK5K80FhLcxlBYJaZ2TzgSeBI3ZvuPjvi2jJKYitTl1AG3nzzTdq0acMVV2hcUPJHKsObfwLsJbgnUN18TQfyKjTVyjxZWVkZBw4c4HOf+1zcpYikVWOh2TscOV/LJ2FZJ/ZFM+KgVmZg06ZNFBYWMmzYMJ3DlLzTWGi2BbqQ2hJvkgfcnZdffpk2bdowcOBABabkpUZv4evu/5y2SiTjHTp0iIKCAkaMGBF3KSKxaWy4U82IUL6v1u7uPP/889TU1CgwJe81FprXpa2KDJfvg0ALFiygR48enH766XGXIhK7Brvn7r4vnYVkunwcBHJ3Dh8+zBVXXEHPnj3jLkckI2g2siRVW1vLvHnz2Lx5swJTJIFCswn5ej5zzZo1DBw4kEsuuaTpjUXyiNbuakRpaSl33x0sHZov5zNrampYunQp11xzjaYUiSShlmYj6gaAHnnkkbw4n1ldXc2sWbPo1auXAlOkAWppNiFfBoBqamo4fvw4V1xxBQMHDoy7HJGMpZamUFVVxRNPPEFFRYUCU6QJCs0G5NMA0Pz58xk+fDhnnHFG3KWIZDx1zxuQDxPajx49yo4dO5gwYYLu5yOSIrU0G5HL5zMrKyt58sknad++vQJTpBn0ryVPffTRR1x33XX06dMn7lJEsopamnnmyJEjzJ49mwsvvFCBKdICCs0kcnUQqLKykpkzZ2riusgpUPc8iVwcBDp06BAdOnRg0qRJdOvWLe5yRLKWWpoNyKVBoAMHDjBr1iwqKysVmCKnSKGZB958801uvfVWevToEXcpIllPoVlPLp3P3Lt3LytWrGDs2LEKTJFWotCsJ1fOZ+7Zs4f58+czePDguEsRySkaCEoi289nVlVVUVtby+TJk+nSpUvc5YjkFLU0c8yOHTv4/e9/T69evRSYIhFQaOaQo0eP8tJLLzF58mTatNFfrUgU1D3PEVu2bKFTp05Mnz5dgSkSIf3rygGbN29myZIldOnSRYEpEjH9C0uQjdON3J3y8nKmTp3KaaedFnc5IjlP3fME2Tbd6I9//CPbtm2jpKQk7lJE8oZamqG6Vma2TDf64IMPWLVqFddcc03cpYjkFbU0Q9nUytyxYwdnn302U6ZMoW3btnGXI5JX1NIku1qZb7/9NkuXLqVDhw4KTJEYKDTJnlZmVVUV27ZtY9KkSRolF4mJuuehTG9lrlq1it69ezNu3Li4SxHJa5E2V8xsnJm9Z2YbzexrSd7/UzN7K/xZambDoqwnWy1fvpzt27fTt2/fuEsRyXuRtTTNrC3wU+AGYCuw3Mzmufs7CZt9AIx29/1mdiNQCnw2qpqyUWVlJX369KG4uFi3qBDJAFG2NK8ENrr7JnevAn4HTEzcwN2Xuvv+8OkbQL8I60kqkye0v/rqqyxfvpy+ffsqMEUyRJSh2RfYkvB8a/haQ/4SeC7ZG2Z2l5mtMLMVe/bsabUCS0tLufvuu4HMGwTauHEjVVVVjBw5Mu5SRCRBlANByZpGnnRDs2sJQjPpTG13LyXoulNcXJz0O1qibtT8kUceyZhBIHdn5cqVFBUVcf7556uFKZJhogzNrcA5Cc/7Advrb2RmQ4FfAje6+94I6zlJJs7NdHdefPFFOnbsSLt2mtggkomi/Je5HBhkZgOBbcDngZP6wGbWH5gN/Lm7b4iwlk/JtLmZ7k51dTVnnnkmQ4cOjbscEWlAZKHp7tVmdh+wEGgLPOru68zsnvD9h4FvA2cAPwu7odXuXhxVTfVlSivT3Xn22We5+OKLFZgiGS7SPqC7Pws8W++1hxMe/xXwV1HWkA1efPFFzjrrLAYMGBB3KSLShLy8Fi9TphnV1tby0UcfMWrUKC6//PK4yxGRFORlaGbC+cyamhrmzp3LwYMHtXiwSBbJ2yHauM9nbt26lSFDhlBYWBhbDSLSfHnZ0oxTdXU1Tz/9NH369FFgimQhhWYaVVdXM3PmTIYMGUL79u3jLkdEWiBvu+fpVlVVRU1NDddddx29evWKuxwRaaG8ammWlpZSUlJCWVlZWvd77NgxnnzySfbu3avAFMlyeRWaM2bMoKysjKKiorSOnL/22muMHDmSfv3SvoiTiLSyvOmeJ15rvmjRorTss7KykrfffpsxY8akZX8iEr28aWmme25mRUUFTz75JOecc07TG4tI1siLlma6VzSqqamhoqKCsWPHcuaZZ0a+PxFJn7xoaaazlXno0CEef/xxunXrpsAUyUF5EZqQniuA6i6NHD9+PB06dIh0XyISj7zonqfDgQMHOHr0KHfccYcWEBbJYXnT0ozSvn37mDt3LgUFBQpMkRynf+Gt4IMPPmDSpEl079497lJEJGIKzVOwe/duVq9ezdixY+MuRUTSJOe751EtOFxeXs5zzz3H1Vdf3erfLSKZK+dbmlFMN9q3bx9du3Zl6tSpdOrUqdW+V0QyX863NKF1pxtt3bqVZ555BkCBKZKH8iI0W0ttbS0rV65k6tSpukWFSJ7K+e55a9m8eTPHjx9n4sSJcZciIjFSSzMF77//Pq+//jrnnntu3KWISMxyOjRbY+T8+PHjtGvXjilTpujSSBHJ3dAsLS3l7rvvBlo+cv7uu+/y3HPPce655+qePiIC5HBo1k01euSRR1o0cr5r1y7Wrl3LTTfd1NqliUgWy9nQhJZPNXrvvffo3r07t912G23bto2gMhHJVjkdmi1RVlbG+vXr6dChA23a6I9HRE6mVEjg7lRWVjJhwgQFpogkpXmaoWXLllFQUMDw4cPjLkVEMlhONqeaO9Vo5cqV7Nu3j0suuSTCqkQkF+RkS7M5i3Ts3LmTIUOGcNlll2FmUZcmIlkuJ1ua0PTIubuzePFi1q9fT+fOnRWYIpKSnGxppmLv3r20adOG0aNHx12KiGSRnG1pNsTdefnll+nYsSMjR46MuxwRyTI5F5qNDQK5OwsWLKBjx4507tw5zZWJSC7Iue55Q4NA7s6xY8f4zGc+wznnnBNHaSKSA3KqpVnXyqw/CFRbW8vTTz/Npk2bFJgickpyKjQbamW++eabnHvuuRQWFsZRlojkkJzrnie2MmtqaigrK+Oqq67SlCIRaRU509KsPwBUXV3N7Nmz6dixowJTRFpNzrQ0E7vm7s6RI0coKipi0KBBMVcmIrkkZ1qaEHTNv/jFL/LEE09gZgpMEWl1kYammY0zs/fMbKOZfS3J+2ZmPw7ff8vMLjuV/bk7c+bM4corr6Rbt26n8lUiIklFFppm1hb4KXAjUAjcYWb1h69vBAaFP3cBP2/JvurOZx4/fpwJEyZw3nnnnULlIiINi7KleSWw0d03uXsV8Dug/k3DJwK/8sAbQA8zO7u5O/rNb34DwIQJEygoKDjFskVEGhZlaPYFtiQ83xq+1txtmlRRUcFVV13F1772qTMAIiKtKsrR82TzfLwF22BmdxF03+nfv/+nPnDNNde0oDwRkeaLMjS3AonXLPYDtrdgG9y9FCgFKC4u/lSo/uhHPzrFUkVEUhNl93w5MMjMBppZB+DzwLx628wDvhCOol8FHHT3HRHWJCJySiJrabp7tZndBywE2gKPuvs6M7snfP9h4FlgPLARqAC+GFU9IiKtIdIrgtz9WYJgTHzt4YTHDtwbZQ0iIq3JgtzKHma2B/gwyVs9gfI0lxO1XDwm0HFlk1w8Jkh+XOe6e6+mPph1odkQM1vh7sVx19GacvGYQMeVTXLxmODUjiunrj0XEYmaQlNEpBlyKTRL4y4gArl4TKDjyia5eExwCseVM+c0RUTSIZdamiIikcuq0Ez3+pzpksJx/Wl4PG+Z2VIzGxZHnc3R1DElbHeFmdWY2ZR01tdSqRyXmZWYWZmZrTOzxcm2yTQp/DfY3cyeNrM14XFl/IUoZvaome02s7UNvN+yvHD3rPghuKrofeA8oAOwBiist8144DmChUCuAt6Mu+5WOq4RwOnh4xsz/bhSOaaE7f5AcAHElLjrbqW/qx7AO0D/8HnvuOtupeP6BvBv4eNewD6gQ9y1N3Fco4DLgLUNvN+ivMimlmba1udMsyaPy92Xuvv+8OkbBAubZLJU/q4AvgzMAnans7hTkMpxTQdmu/tHAO6eDceWynE50NWCuxR2IQjN6vSW2TzuvoSgzoa0KC+yKTTTtj5nmjW35r8k+L9jJmvymMysLzAJeJjskcrf1YXA6Wa2yMxWmtkX0lZdy6VyXD8BLiJYhext4G/dvTY95UWmRXmRTXejbLX1OTNMyjWb2bUEoZnpC4imckw/Ar7q7jVZdIvlVI6rHXA5cB3QEXjdzN5w9w1RF3cKUjmusUAZMAY4H3jBzF5x948jri1KLcqLbArNVlufM8OkVLOZDQV+Cdzo7nvTVFtLpXJMxcDvwsDsCYw3s2p3n5uWClsm1f8Gy939CHDEzJYAw4BMDs1UjuuLwPc8OBm40cw+AIYAy9JTYiRalhdxn6xtxknddsAmYCCfnKy+uN42N3Hyid1lcdfdSsfVn2D5vBFx19tax1Rv+8fIjoGgVP6uLgJeCrftBKwFPhN37a1wXD8HvhM+PhPYBvSMu/YUjm0ADQ8EtSgvsqal6Tm6PmeKx/Vt4AzgZ2HLrNozeBGFFI8p66RyXO6+3swWAG8BtcAv3T3plJdMkeLf178Aj5nZ2wQh81V3z+jVj8zst0AJ0NPMtgL/BLSHU8sLXREkItIM2TR6LiISO4WmiEgzKDRFRJpBoSki0gwKTRGRZlBoSkrClYjKEn4GNLLt4VbY32Nm9kG4r1VmNrwF3/FLMysMH3+j3ntLT7XG8Hvq/lzWhqsA9Whi+yIzG98a+5Z4aMqRpMTMDrt7l9betpHveAyY7+4zzexzwIPuPvQUvu+Ua2rqe83sf4AN7v6vjWx/J1Ds7ve1di2SHmppSouYWRczeylsBb5tZp9axcjMzjazJQktsZHh658zs9fDzz5pZk2F2RLggvCzfx9+11ozuz98rbOZPROu9bjWzG4PX19kZsVm9j2gY1jH4+F7h8PfTyS2/MIW7mQza2tmPzCz5eFai3en8MfyOuGCD2Z2pQVrn64Ofw82sw7APwO3h7XcHtb+aLif1cn+HCXDxH2Zk36y4weoIViwoQyYQ3DpXbfwvZ4EV1XU9VwOh7+/AnwzfNwW6BpuuwToHL7+VeDbSfb3GOGllcBU4E2ChTDeBjoTLE+2DrgUmAz8IuGz3cPfiwhadSdqStimrsZJwP+EjzsQrHrTEbgL+Fb4+mnACmBgkjoPJxzfk8C48Hk3oF34+HpgVvj4TuAnCZ//LvBn4eMeBNeod47771s/Df9kzWWUErtKdy+qe2Jm7YHvmtkogssF+xJck7wz4TPLgUfDbee6e5mZjQYKgdfCS0I7ELTQkvmBmX0L2EOwutN1wBwPFsPAzGYDI4EFwINm9m8EXfpXmnFczwE/NrPTgHHAEnevDE8JDLVPVpTvDgwCPqj3+Y5mVkZwjfNK4IWE7f/HzAYRrJzTvoH9fw6YYGb/O3xeQLDWwPpmHIOkkUJTWupPCVbwvtzdj5vZZoJ/8Ce4+5IwVG8Cfm1mPwD2Ay+4+x0p7OMf3H1m3RMzuz7ZRu6+wcwuJ7iO+P+Z2fPu/s+pHIS7HzWzRQRLn90O/LZud8CX3X1hE19R6e5FZtYdmA/cC/yY4Frtl919UjhotqiBzxsw2d3fS6VeiZ/OaUpLdQd2h4F5LXBu/Q3M7Nxwm18A/0Vw64E3gKvNrO4cZSczuzDFfS4Bbg0/05mga/2KmfUBKtz9N8CD4X7qOx62eJP5HcFiDSMJFq0g/P2lus+Y2YXhPpNy94PA3wD/O/xMd4KVgCDoktc5RHCaos5C4MsWNrvN7NKG9iGZQaEpLfU4UGxmKwhane8m2aYEKDOz1QTnHR9y9z0EIfJbM3uLIESHpLJDd19FcK5zGcE5zl+6+2rgEmBZ2E3+JvBAko+XAm/VDQTV8zzB/WRe9OB2DxCsXfoOsMqCG3M9QhM9s7CWNcDnge8TtHpfIzjfWedloLBuIIigRdo+rG1t+FwymKYciYg0g1qaIiLNoNAUEWkGhaaISDMoNEVEmkGhKSLSDApNEZFmUGiKiDSDQlNEpBn+P+QW2+K/fTxsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(fpr, tpr, color='black')\n",
    "plt.plot([0, 1], [0, 1], color='black', lw=0.7, linestyle='dashed', alpha=0.5)\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AUC: Area under the ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_small = tpr_fpr_dataframe(y_val, y_pred_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.665108427638009"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc(df_scores.fpr, df_scores.tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47769928012581675"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc(df_scores_small.fpr, df_scores_small.tpr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing multiple models with ROC curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7nUlEQVR4nO3deXyU9bX48c8hAcKOElBkV1kEhSBBBZVEQDbZZVG0Lb29BYu15arXRmp/5aqtXGurtrXKaL3YChUFsUBVXAEVFQgEZSmIgCQSlrDJakhyfn88M3EIWWaSeWY979crr2RmnuU8LCff/SuqijHGmMDUinQAxhgTSyxpGmNMECxpGmNMECxpGmNMECxpGmNMECxpGmNMECxpGmNMECxpGteJyC4ROSUix0Vkr4jMEZGGZY7pKyLvicgxETkqIktEpGuZYxqLyBMistt7re3e16nhfSKTyCxpmnAZoaoNgTSgJ3C/7wMR6QO8BfwTuAjoAGwAPhKRi73H1AHeBboBQ4DGQF/gIHCVW0GLSLJb1zaxyZKmCStV3Qssw0mePo8Cf1PVJ1X1mKoeUtUHgE+Amd5jvg+0Bcao6mZVLVHV/ar6kKq+Xt69RKSbiLwtIodEZJ+IzPC+P0dEHvY7LlNE8vxe7xKRX4jIZ8AJEXlARBaUufaTIvJH789NROSvIpIvIl+LyMMiklSzPykTrSxpmrASkdbAUGC793V9nBLjK+Uc/jJwo/fngcCbqno8wPs0At4B3sQpvV6KU1IN1K3ATUBT4O/AMBFp7L12EjABmOc99gWgyHuPnsAg4D+DuJeJIZY0Tbi8JiLHgFxgP/Br7/vn4/w7zC/nnHzA117ZrIJjKjIc2Kuqv1fV094S7KdBnP9HVc1V1VOq+hWwDhjt/aw/cFJVPxGRC3B+CUxX1ROquh94HLgliHuZGGJJ04TLaFVtBGQCXfguGR4GSoCW5ZzTEijw/nywgmMq0gb4slqROnLLvJ6HU/oEmMR3pcx2QG0gX0SOiMgRYDbQogb3NlHMkqYJK1VdAcwBHvO+PgF8DIwv5/AJfFelfgcYLCINArxVLnBJBZ+dAOr7vb6wvFDLvH4FyPQ2L4zhu6SZC3wLpKpqU+9XY1XtFmCcJsZY0jSR8ARwo4ikeV9nAT8QkZ+JSCMROc/bUdMH+B/vMX/HSVALRaSLiNQSkWYiMkNEhpVzj6XAhSIyXUTqeq97tfezHJw2yvNF5EJgelUBq+oBYDnwf8BOVd3ifT8fp+f/994hUbVE5BIRyQjyz8TECEuaJuy8CehvwK+8rz8EBgNjcdotv8LpULlOVb/wHvMtTmfQv4G3gW+A1TjV/HPaKlX1GE4n0ghgL/AFcIP347/jDGnahZPw5gcY+jxvDPPKvP99oA6wGae5YQHBNSWYGCK2CLExxgTOSprGGBMES5rGGBMES5rGGBMES5rGGBMES5rGGBOEmFvBJTU1Vdu3bx/pMIwxcSY7O7tAVZtXdZxrSVNEnseZ/7tfVS8v53MBngSGASeByaq6rqrrtm/fnrVr14Y6XGNMghORrwI5zs3q+RycdQ8rMhTo6P2aAjztYizGGBMSriVNVV0JHKrkkFE4ayiqqn4CNBURm0VhjIlqkewIasXZK8nked8zxhhXeDweMjMzmT59erWvEcmkKeW8V+6cThGZIiJrRWTtgQMHXA7LGBNPPB4PnTt3pmnTpkydOpUVK1awYMGCqk+sQCR7z/Nw1jz0aQ3sKe9AVfUAHoD09HSbLG+MqZLH4+Hyyy/nyy+/ZNu2bQDUq1ePiy66iPvuu6/a141k0lwM/FREXgKuBo56l9kyxpgamzdvHocPH+azzz4D4K677uJ//ud/OO+882p0XTeHHP0DZ5XuVO+mVb/GWeEaVX0GeB1nuNF2nCFHP3QrFmNM4snPzy8tYf7yl7/k4YcfruKMwLiWNFX11io+V+BOt+5vjElct99+e2nCnDx5co2q42XF3IwgY4ypytKlSwEYO3Ysf/rTn2jYsGHIrm1J0xgT8zweD/PmOQvq9+zZk6NHj9K+fXteeeUVatUK7SAhS5rGmJjm8XiYOnUqAJ06dWLFihUA3HPPPSFPmGBJ0xgT4z7++GMAOnbsWNqO+fTTT3PHHXe4cj9bGs4YE3M8Hg/XXnstS5YsoUOHDgB88cUXdOnShT//+c+uJUywkqYxJkb4t1v6quD5+fm89957ANx///3MnDmTOnXquBpHzO1GmZ6errY0nDGJJzc3l1mzZrFp0yYAWrduTV5eHtnZ2VxyySXk5OTU6Poikq2q6VUdZyVNY0xMaNOmDU899VTp68zMTNatW0ebNm1KO4LCwZKmMSYmzJ8/H4CJEycCUFhYSI8ePXj//fdJTg5fKrOkaYyJWv7tmDk5OaSlpTFx4kQ2btzI0aNHSU1NDWvCBOs9N8ZEsW3btpV2+qSlpTFp0iS+/fZbcnNzSU1Nxdk1J7yspGmMiTpz5swBICsri06dOjFlyhQA1q9fT0FBAbm5uaxcuZKMjIywx2ZJ0xgTdXxJc/LkyaUJc+3atezdu5e0tLTSKvukSZPCHptVz40xUcXj8ZRWyX1Onz7NBRdcwE033VRaJc/IyChNqOFkJU1jTFQpW4r86KOPKCoqikhVvDxW0jTGRJV+/fqVliK/+OILTp8+Tb9+/Uo/L68kGk5W0jTGRAWPx0NRURFZWVm0atWKdevW0b17dy699NKzeskj2Z4JVtI0xkSJefPmMWvWLOrVq8fFF1/MiRMnSE5OPith+kqZkWrPBEuaxpgI8+1FnpOTQ4cOHThz5gwtWrTg+uuvP+c433TJSJUywZKmMSbC5s2bx4oVK+jRowfdu3cnPz+fHj16nHWMf8KcPXt2xEqZYG2axpgIW758OQBvv/02559/Pu3atTvr82hKmGAlTWNMBPiq5KmpqTz66KPk5uZy/fXX06tXr3OOi6aECZY0jTERMG/ePHJycujWrRs7duzg8OHDpKSknHVMNCZMsOq5MSYCfD3gc+bM4cSJE3Tr1g0of3X2aEqYYEnTGBMBgwYNomPHjrRu3Zrk5OTSZOlLlBkZGWRkZDBp0qSoSphgSdMYE0YPPfQQxcXF/Md//Ac9e/YsTZi+ani0Jkp/ljSNMa7zlSTXr19Pjx49mDZtGi1atAC+m+ETbdXwiljSNMa4zleS7NKlCzfddFNpwvSJ5AyfYFnSNMa4buzYsbRt25bp06fTrl27crexiBWWNI0xrvF4PGzbto3777+fXr16ISLntGH6trGIFZY0jTGu8E+Od999N88+++xZPeSx0oZZliVNY4wr5s6dCzhV80mTJp01nCjae8grY0nTGBNyx44dY+XKlVx//fUcPHiQnJycmE+WPpY0jTEhVVxczKJFi7j55ptJSUlh7ty5ZGRklC7MEessaRpjQubIkSPcd9991K1bl0GDBkXF+pehZknTGBMShw4dYvHixWzevJnk5GQ+//xzIHY7fCpiSdMYExKPPfYY8+fPZ8eOHTRp0gSIrUHrgbKkaYypkQMHDrB+/XpWrVrFV199BUCLFi246KKL4qpa7mNJ0xhTbQUFBbz++uscPXqUFStW0LJlS4YPH47H44l0aK6xpGmMqZbHH3+cRYsWoap8+OGHAMycOTPuquNlWdI0xgTt66+/xuPxsGfPHi644AIAOnXqFPcJE2y7C2NMkEpKSnj44Yf597//Tc+ePbnooosAuOeeeyIcWXhYSdMYExCPx8Pzzz+PqrJ69WrAGX85derUuOwlr4glTWNMhcrbs+f666+nY8eOnHfeeUyZMoXNmzfTtWvXSIYZVqKq7l1cZAjwJJAEPKeqs8p83gR4EWiLk8AfU9X/q+ya6enpunbtWpciNsZUlCi//fZbJk+eTFJSUulMHzfzR7iJSLaqpld1nGslTRFJAp4CbgTygDUislhVN/sddiewWVVHiEhzYKuIzFXVQrfiMsaUr6LNzfr3709aWhojR448Z1vdRORm9fwqYLuq7gAQkZeAUYB/0lSgkYgI0BA4BBS5GJMxpozykqVvNaK9e/fywQcfcNNNNwGwc+dOIP6mRgbDteq5iIwDhqjqf3pffw+4WlV/6ndMI2Ax0AVoBExU1X+Vc60pwBSAtm3b9vLNOjDG1FxmZmbplhP+S7dt27aNtm3bUqdOHcaPHw/AwoULIxmqqyJePQeknPfKZujBQA7QH7gEeFtEPlDVb846SdUDeMBp0wx9qMYkHl8J05cw/Zduy8nJYdeuXbz33nu89NJLMbePj5vcTJp5QBu/162BPWWO+SEwS53i7nYR2YlT6lztYlzGJCT/Dh7gnOq4j6py8uRJRo4cSf/+/c8qhRp3k+YaoKOIdAC+Bm4Byv6p7wYGAB+IyAVAZ2CHizEZk5DKbmbm+152JfU1a9aQkpJC3759ARgwYAADBgzgV7/6VfiDjlKuJU1VLRKRnwLLcIYcPa+qm0TkDu/nzwAPAXNE5HOc6vwvVLXArZiMSVS+EmZlHTjZ2dkUFBQwZMiQ0vcsWZ7L1XGabrBxmsZUrWxVvLx2S3/79++nfv36NGjQAGcwi3ONRYsW8cYbb4Qj5IgLtCPI5p4bE2d8VXFfmyVQaZvkihUr2LhxIw0bNixNmOCUTt98803X4401No3SmDhSdvB5VWMpDxw4gIjQv3//c66zYsWK0vZP8x0raRoTRwJpuwSnh3z58uXUr1+ffv36nfP5jBkzgPjaEC1ULGkaE2eqWnFIVVm2bBl169alQYMGZ33mW3E9KysroWf9VMaq58bEiUCq1KpKYWEhXbt2pW3btuecP3XqVA4dOkRWVpbb4cYsS5rGxLiyc8crqlKXlJSwdOlSLr300nKXcvNV7c8//3z3go0DljSNiXG+qZDlDVb398knn9C2bdtK175MpMWEq8uSpjExzL9KXtEYzOLiYjZs2ECfPn3OGlJkqseSpjExKNAqeXFxMYsWLaJr165VJswJEyaEPM54ZEnTmBhS2dqXZakqx48fp3v37nTq1KnS6548eZLJkydTv359V+KOJzbkyJgY4PF4yMzMLJ3pk5GRwezZs1m+fHm5CfPMmTO8/PLL1KpVq8qE6fF4aNCgAcOGDXMr/LhiJU1jolzZFYoq6+wBp4S5aNEi0tPTadSoUZXX9/Wa20D2wFjSNCZMyi6iEShfVTyQwebffvstR44cYeTIkaSkpAQUk6/kar3mgbGkaYwLykuQ/u2QwQikdAlw+vRpFixYQL9+/QJKmAB16tShc+fOVsoMgi0NZ0yIlbfgr08gya+61q9fT/PmzWndunVAMc6bN48FCxaQmprqSjyxJhr2CDImIVS0jUS45m6fPHmSlStXnrV4cGX8k7oJniVNY4IQSLU70Op0KJw4cYKFCxcyePDggM/xXwnJSpnBs+q5MeWoqNOmonbJcCVJfydPniQpKYnjx4/TrFmzgM/r2bMnTZo0qXAGUaKy6rkxQfJPlBUlx3CWIivzzTffsGjRIsaMGRNwwszNzQWcZd+OHj3qZnhxzZKmSTiBlCKjJTlW5N1332XEiBE0btw4oON97ZiVzVE3gbGkaRJKZT3b0Z4oAQ4dOsS+ffsYPXp0UItv2AD20LGkaeJeedXuWFyVvKCggKVLlzJ27NigEqYNYA8tS5ombpW3uEUslCbLU1JSwrFjx7j55psDmhrpr2XLlvTt29dKmSFiSdPEtMqmJgayElAs2Lt3LytWrGDChAlBr4e5detWOnXqxEcffeRSdInHkqaJSeWVIsuK9WQJcOzYMd566y3GjRtXrQWEfe231vkTOpY0TVSqanGLeClFVmbPnj00adKEW2+9ldq1a0c6HONlSdNETKBV6/LEc7IE2L17NytXrmTcuHHVTpiB7E5pgmdJ04RdolSta2LXrl2MHz+eunXrVvsaNszIHTaN0oRVsAvqJpovv/ySgoICrr766hpf65133gFg4MCBNb5WIgj5NEoRaaCqJ2oWlkkklS1uEYvjJN325ZdfsmbNGm6++eaQXM+SpTuq3CNIRPqKyGZgi/d1DxH5i+uRmZjmK1H6kqSPb28bS5hnO3LkCKmpqTVqw/Tx7Sd0yy23kJOTE5oATalASpqPA4OBxQCqukFE+rkalYlJ8TLzJtw2bdrEli1buPnmm0OyL/m8efPIyckhLS2N1atXk5aWVvMgTamAqueqmlvmL7PYnXBMLIqnmTfhduLECbZv386YMWNCkjD9e8xtbKY7AkmauSLSF1ARqQP8DG9V3SS2YPbgNufasGEDrVq1YtSoUSG75uWXX86IESMYPnx4yK5pzhZI0rwDeBJoBeQBbwHT3AzKRD/rBa+Z7Oxs9uzZwxVXXBGya3o8Hi6//HIWL14csmuacwWSNDur6m3+b4jItYBNZk1g/lsmWLIMTlFREY0aNWL48OEhqZLDd7/ERowYYUnTZYEkzT8BVwbwnolDFc3aycnJsaXGquHjjz+mpKSEa6+9NiTXK9tEYtVy91WYNEWkD9AXaC4id/t91BhIcjswE1lVzdpJS0uzmSZB2rRpE8ePHw/p+MmrrrqK9957z5pIwqiykmYdoKH3GP8F/L4BxrkZlImMivbIsf+MNaOqbNmyhc6dO9O1a9eQVcl9M35eeumlkFzPBKbCpKmqK4AVIjJHVb8KY0zGZfGwR06sUFXee+89ateuTdeuXUN2Xf89f2zmT3gF0qZ5UkR+B3QDUnxvqmp/16IyIVU2SUb7Tovx5PTp0zRu3JjevXuH7Jr+IxesiST8Akmac4H5wHCc4Uc/AA64GZQJnfI2ErPk6D5VZdmyZaSlpYU0YYKNXIi0QJJmM1X9q4j83K/KvqLKs0xEVFSqtP9g4bVs2TKaNWvGhRdeGPJrz549G4DOnTuH/NqmalUu2AGc8X7PF5GbRKQn0DqQi4vIEBHZKiLbRSSrgmMyRSRHRDZZMq4e3wINmZmZ5yySYQtkhFdJSQkFBQVce+21Ia+SZ2Zmcu2117Jt2zZLmBEUSEnzYRFpAtyDMz6zMTC9qpNEJAl4CrgRZybRGhFZrKqb/Y5pCvwFGKKqu0WkRdBPkMBsznd0KSkpYfHixVx88cV07949ZNct28SSn58fsmub4FWZNFV1qffHo8ANUDojqCpXAdtVdYf3nJeAUcBmv2MmAa+q6m7vvfYHHrrxrWZjiTI6bNu2jUsvvZTLL788pNe1NszoUtng9iRgAs6c8zdVdaOIDAdmAPWAnlVcuxWQ6/c6Dyi7HHUnoLaILMcZC/qkqv4tqCdIIGXbK33Lf9lqNpFVVFTE+++/z4ABA6hVK5AWr+D8/e9/B6BNmzYhv7YJXmUlzb8CbYDVwB9F5CugD5Clqq8FcO3yRvCW3VsjGegFDMBJxB+LyCequu2sC4lMAaYAtG3bNoBbx7ZAxlGCzcqJBkVFRSxcuJC0tDRXEiZYsow2lSXNdKC7qpaISApQAFyqqnsDvHYeTtL1aQ3sKeeYAu82GidEZCXQAzgraaqqB/CAs0dQgPePSeUNEfKxanh0OXPmDKrKtddeS+vWAfWNBs3j8fD000+TlZXFxIkTXbmHCU5lSbNQVUsAVPW0iGwLImECrAE6ikgH4GvgFpw2TH//BP4sIsk40zavxlkpPiH5J0xrv4puhYWFLFiwgP79+7uaMH3/Ho4ePerKPUzwKkuaXUTkM+/PAlzifS2Aqmql3YOqWiQiPwWW4Szw8byqbhKRO7yfP6OqW0TkTeAzoAR4TlU31vCZYoptERGb3n77bfr27evKOEwf6wCKThVu4Ssi7So7MVLz0eNtC9/MzMzSDh3Aqt9R7vTp03z55ZchXXjDn8fjoU6dOkyePJl7772XtWvXWkdfmNR4C19bpMM9/qVL6wGPHadOnSqtkruVMKdOnUrnzp2ZPHkyWVlZvPrqqyG/j6mZgPc9N6FRtqPHesBjg6py4MABBg4cSMuWLV25h+8X6d13O8vXpqamWq0jClnSDDNrp4o9x48fZ8mSJUyYMIGkJHfW3/bfRdL+XUS3gJKmiNQD2qrqVpfjiWv2HyP2FBYWsnDhQoYNG+ZawgS4/fbbycvLc60n3oROlUlTREYAj+EMCeogImnAg6o60uXY4o6vlGnV8dhw9OhRatWqxYQJE6hXr55r9/F4PBQVFfHggw+6dg8TOoFMYZiJM4/8CICq5gDt3QooXlkpM7YcPnyYRYsWUVRU5FrC9K1cNHXqVGbNmuXKPUzoBVI9L1LVo270FiYSK2XGls8//5zRo0fTtGlT1+5Rdt94ExsCSZobRWQSkCQiHYGfAavcDSt++IYX2Za3saGgoIAtW7bQr1+/kFxv6NChnDp16qz3hg8fzr333mvTYmNUINXzu3D2B/oWmIezRNx0F2OKK76EaUOLot/+/fv517/+VTrRoLo8Hg8PPfRQlcctX77cEmYMqnBGUOkBIj1VdX2Y4qlSrM0IyszMBLDB61HuxIkTFBcXk5SURIMGDap9Hd843Pr163PixIkQRmjcFuiMoEBKmn8QkX+LyEMi0i0EsSUMX+ePiW579uxh4cKF1K9fPyQJE+DxxxN23Zm4V2XSVNUbgEycHSg9IvK5iDzgdmCxzrZZjQ1FRUWsXLmS8ePHk5xcs7key5YtA2ziQtxT1YC/gCuAv+MsGxfUuaH66tWrl8aCjIwMBXT27NmRDsVUYNeuXZqXl6clJSXVvsbs2bMV0KysrBBGZiIBWKsB5KAqS5oicpmIzBSRjcCfcXrObdpCJWxMZvTbuXMnH330Ec2bN6/W4hv+YywBOnToEOoQTZQKpD7yf8A/gEGqWnbldVOGVcujX0lJCadOnWL8+PHUrl27WtdYuXJl6S9GGzaUWKrsPY820d57npmZyYoVK6xdK0pt27aNHTt2MGTIkEiHYqJMjXvPReRl7/fPReQzv6/P/VZ0N+Wwanl0ys3NZcOGDQwcOLBG1/F4PEyfPj00QZmYU1n1/Ofe78PDEYgxbtq1axcXXXQRY8eOrdFqRf7NL0888USIojOxpMKSpqrme3+cpqpf+X8B08ITXmyxcZnR6bPPPmP9+vUkJyfXKGFOmTLlrI3vTGIKZHD7jeW8NzTUgcQDW5Qj+hQXF1NQUMCoUaNCsi95RkaGtVcnuAqr5yLyE5wS5cVl2jAbAR+5HVissvbM6LF27VqaNm1K//79q30NX3X8xz/+MR6PJ4TRmVhV2a/eecAIYLH3u++rl6reHobYYopVzaPLmjVr2L9/P5dcckmNruOrPaSnV9mpahJEZR1Bqqq7ROTOsh+IyPmqesjFuGKOVc2jx9GjR7n44otJT0+v0a6R06dPtyX9zDkqS5rzcHrOswEF/P/1KXCxi3HFJPvPFXkffPABRUVF3HDDDSG5ni3pZ8qqbN/z4d7vNj/MxIS8vDyKiopKl+OrDl8b5m233caLL74YuuBM3Ahk7vm1ItLA+/PtIvIHEWnrfmjGBEZVWbVqFc2bN+eGG26odpXcfwxmqFZuN/EnkDEYTwMnRaQHcB/wFc5KR8bLOoEiR1V5++23Aahbt26NruU/BtOaWUxFAt1YTUVkFPCkqv5VRH7gdmCxxDqBIkNVKSoqon379nTq1KnG18vKyqJDhw6WME2lAkmax0TkfuB7wPUikgRUb2mYOGadQOGlqixdupQuXbpUO2H6Nr0DaNasGQsXLgxliCZOBVI9n4izqdp/qOpeoBXwO1ejMqYKK1asoHXr1nTs2LFa5/vaL61ZxQSrypKmqu4VkblAbxEZDqxW1b+5H5ox5yopKWHbtm1cf/31NZpHvm/fPurXr8/jjz9uNQQTlEB6zycAq4HxwATgUxEZ53ZgscI6gcKnuLiYRYsWUVRUVO2E6fF4GDp0KL/61a84ceKEJUwTtEDaNH8J9FbV/QAi0hx4B1jgZmCxwjqBwqegoICuXbty2WWXBX2ur/3SfsGZmgqkTbOWL2F6HQzwvIRhnUDuKioqYuHChTRp0qRaCROcX26+KZG2rJupiUBKmm+KyDKcfYLA6Rh63b2QjPlOSUkJCxYsID09nZSUlGpfZ/jw4QwfPpx77703hNGZRBRIR9B/i8hY4Dqc+eceVV3kemQm4RUWFnLq1CmGDRtG48aNq3UNj8fDoUOHyMrKCnF0JlFVtp5mR+Ax4BLgc+BeVf06XIGZxPbtt9/yyiuvkJGRQZs2bap9HV875vnnn29NKCYkKmubfB5YCtyMs9LRn8ISkTHAunXrapQwffuS29JuJtQqS5qNVPVZVd2qqo8B7cMUU0zw/09pQufkyZOsWLGCPn361KiEWVRUxI4dO2xpNxNylbVppohIT75bR7Oe/2tVXed2cNHM1xtr/ylD58SJEyxYsIDBgwfX+FrTpk1j2jTb/8+EXmVJMx/4g9/rvX6vFaj+xitxIi0tjeXLl0c6jLhQWFiIqjJ06FBatGhRo2t5PB7y8vLIysqifv36IYrQGEdlixCHZulrY6rwzTff8OqrrzJhwoSQJEzfEm+tW7e2tkwTcjZIvRps6mToqCr/+te/GDFiREhKhb4ZWrYmpnGLq0lTRIaIyFYR2S4iFQ6UE5HeIlIcK3PabepkaBw+fJhdu3YxceJEmjVrFpJr7t2713rLjasCmRFULd51N58CbgTygDUislhVN5dz3P8Cy9yKxQ32H7NmDh48yJIlSxg9ejS1atX8d3dBQQHgLCRcWFhY4+sZU5Eqk6Y4G67cBlysqg969we6UFVXV3HqVcB2Vd3hvc5LwChgc5nj7gIWAr2DDd7EJlVl3759jB07ttozffz52jEzMjKsY864LpBf8X8B+gC3el8fwylBVqUVkOv3Os/7XikRaQWMAZ4J4HpRwdoza2bfvn3885//pGvXriFJmGDNJSa8AkmaV6vqncBpAFU9DNQJ4LzytgTUMq+fAH6hqsWVXkhkioisFZG1Bw4cCODW7rH/oNV36NAhli1bxo033lij63g8HnJznd/H8+fPt1k/JqwCSZpnvO2OCqXraZYEcF4e4D+lozWwp8wx6cBLIrILGAf8RURGl72QqnpUNV1V05s3bx7Ard1l/0GDt3//fho2bMiECRNo0KBBta/jq4rPmjWr9D2bYGDCKZCOoD8Ci4AWIvIbnOT2QADnrQE6ikgH4GvgFuCsf9mq2sH3s4jMAZaq6msBRW5iRm5uLsuXL2f8+PE1Wt7Nfwxmjx49AJg4cSITJ04MSZzGBCKQpeHmikg2MACnyj1aVbcEcF6RiPwUp1c8CXheVTeJyB3ez2OmHdNUn6qyadMmJkyYUON9yV944QXAxmCayAqk97wtcBJY4v+equ6u6lxVfZ0yCxZXlCxVdXJV1zOxZceOHRw/fpwhQ4bU6Dpbt24FnOFE+fn5ljBNRAVSPf8XTnumAClAB2Ar0M3FuEyM+/LLL1m9ejXjxlV/voJvXx/fwig2nMhEg0Cq51f4vxaRK4GprkVkYt7Jkydp1KgR48ePJzm5+vMnbCUpE42C/hetqutExAaim3Jt2bKFLVu2MHbs2Gqd7/F4uOqqq0hLSyM5OdlKmCbqBNKmebffy1rAlUBkB0uaqHT48GG2bNnCqFGjqnW+r3d84sSJvPTSS2RlZbFjx44QR2lMzQRS0mzk93MRThvnQnfCMbFq06ZNtG3bljFjxuDMvA2O/3Ci/v2dpVoHDhwY0hiNCYVKk6Z3UHtDVf3vMMUT1XxTKDMyMiIdSlRZv349ubm5XHbZZdVKmGBLupnYUdlulMnesZZXhjOgaGZTKM9VUlKCiDBixIhqJ0yA3/72twD07ds3VKEZ44rKSpqrcdovc0RkMfAKcML3oaq+6nJsUcW/lGklIcenn35KrVq16N27Zv2Cq1atAixhmtgQSJvm+cBBnD2BfOM1FUiopGmlzLPl5ORw5MgRBg0aVO1r2DhME4sqS5otvD3nG/kuWfqUXa0oIVgp07Fjxw66du1Kjx49ql0lz8nJKe34ycjIsF9GJmZUtspREtDQ+9XI72ffl0kwqsp7773H7t27qV27dtAJ07dX/DvvvAM4yXL27NksX77cfhmZmFHpFr6q+mDYIjFR79ixY6SkpFSr7dF/SNEDDzxg1XETsyoraVa/KzTOJPpq7arKW2+9RXFxcbU7a/yHFNn4SxPLKkuaA8IWRZRL9E6gN998k6ZNm3LeeefV6DrWJmziQYXVc1U9FM5Aol0i/odXVY4fP07v3r1JTU0N6lxfzzjAPffcw+zZs90I0Ziwc20LXxPbSkpKWLJkCRdffDFXXHFF1Sd45ebm8r3vfa+0OcM3e6pz586uxGlMuNV8w+k4l6jtmRs2bKBDhw4BJUxfr/j8+fNL3/PvGR8xYoSboRoTVlbSrIR/j2+itGcWFxezatUqrrvuuiqHFPmq4L5fKj/5yU9o06aN9YqbuGYlzUok2iISRUVFLFy4kObNmwc0BnPbtm2lU0tnz55tG5yZhGAlzSokSgdQcXExZ86coXfv3nTo0KHC4zweD3/4wx/IysoiKyuLTp06JcSfjzE+ljQNhYWFLFiwgMGDB5ebMP17wn1V8cLCQlJTUy1hmoRj1fMKJFIH0NKlS+nTpw/NmjUr9/Pbb7+dfv36Ad918FiyNInKSpoVSIQB7adPnyY/P5+RI0dWuAGax+OhqKiIBx+0GbXGgCXNSsVze+apU6dYsGABN9xwwzkJs2x1vE2bNkybNi0SYRoTdax6nqB2797NgAEDaN269Tmf+Q8jysjI4IEHHgh3eMZELStpJpgTJ06wbNmyczZA83g8zJgxg6ysLBtnaUwlLGmWI143UPNVyYcOHVqaMIcOHcqpU6dKS5aNGzeOZIjGRD2rnpcjHjuBjh07Rq1atRgzZgyvvfYaDz300FmfW6+4MYER1djauSI9PV3Xrl3r6j0yMzMB4qaaeuTIEV577TVGjx7Nyy+/zNSpU6lfvz4nTpyo+mRjEoSIZKtqelXHWfU8AXz66aeMHj2aH/3oR7z6qrMf3uOPPx7hqIyJTVY9LyOeBrUfPHiQGTNmsHz5cpo2bQpYNdyYmrKSZhnx0p554MAB7rvvPubMmQPAI488wsKFCyMblDFxwEqa5Yj1Qe2FhYWUlJSwfft2AFs13ZgQsqQZZ/Lz83n55Zdp3rw5SUlJMf8LwJhoY9XzOHL69Gneffddbr75ZmrVqlXubB9jTM1Y0owTubm51K9fn0mTJvHcc8+xefNmXnzxxUiHZUzcsep5HNi1axcrV66kYcOGPPfcc0ydOpUnn3wy0mEZE5csafqJxeFGqkpBQQHjx4/nrrvuKt3TyDp/jHGHVc/9xNpwoy+++IKvv/66dAYTOD3/kyZNss4fY1xiSdPLf5GOWEg4O3fuZN26dRw6dIiZM2fSqVMnPB5PpMMyJu5Z9dwrlkqZ+fn5tGzZknHjxjF//nxycnIiHZIxCcOSJrFVyvz8889ZtWoVderUISkpCYC0tDQrZRoTJpY0iZ1SZmFhIV9//TVjxoyhVi3nry7WOq6MiXXWpukV7aXMdevW0aJFC4YMGXLW+z//+c/p2rVrhKIyJvG4mjRFZAjwJJAEPKeqs8p8fhvwC+/L48BPVHWDmzHFojVr1rBv3z569uxZ+p7H42HlypU2gN2YMHMtaYpIEvAUcCOQB6wRkcWqutnvsJ1AhqoeFpGhgAe42q2YYtGpU6e46KKLSE9P59lnnz1rl0jAkqYxYeZmm+ZVwHZV3aGqhcBLwCj/A1R1laoe9r78BAj7ZOloHtD+4YcfsmbNGlq1aoWIsHPnzrN2ibQB7MaEn5vV81ZArt/rPCovRf4IeKO8D0RkCjAFoG3btqGKD4/HUzqDJto6gbZv305hYSFffPEFGRkZjB07loULF/LII49EOjRjEpqbJU0p571yNyQSkRtwkuYvyvtcVT2qmq6q6c2bNw9ZgL6qbjStZK6qrF27lvbt2/PFF19wxx13ADB48OAIR2aMAXdLmnlAG7/XrYE9ZQ8Ske7Ac8BQVT3oYjxnicaxmarKO++8Q7169UhOTuYf//gHEF1J3ZhE52bSXAN0FJEOwNfALcBZdWARaQu8CnxPVbe5GMs5om1spqpSVFTEBRdcQPfu3QEYMGAAAwYMsIRpTBRxdQtfERkGPIEz5Oh5Vf2NiNwBoKrPiMhzwM3AV95TiqraQjNUW/hG0za9qsrrr79Ot27daN++faTDMSYhBbqFr6szglT1dVXtpKqXqOpvvO89o6rPeH/+T1U9T1XTvF9VBhyP3nnnHS688MLShOnxeBARhg4dGtnAjDHnSMhplNEyzKikpITdu3fTr18/evXqBZzdoz9mzJhIhmeMKUdCJs1oaM8sLi7mtdde4+jRo9StW7f0/RkzZgDW+WNMtErYueeR7jXPy8ujS5cufPjhh9x1112A076alZVF48aNLWEaE6USNmlGSlFREW+88Qa7du1i4cKFZ83wAbj33nsjGZ4xpgoJWT2PlKeffpoJEybQrl270rUwfdMho6EX3xhTNStphklhYSHz5s3jww8/ZMiQIUybNo1p06ZFOixjTJASqqTp8XjIzMwM+/YQTz31FD169GDDhg0Rb0s1xtRMQiXNefPmkZOTQ1paWlh7zr/66isKCwu58soro2YGkjGmehKmeu4/1zxc7YenTp3igw8+4L777uPRRx8Nyz2NMe5KmJJmuMdmnjx5kmnTpjF48GDGjRsXlnsaY9yXEEkz3CsaFRcX88wzzzBnzhwgehYFMcbUXEJUz8NZyjx27BiLFi3itddeA2xmjzHxJiGSJoRnBpBvauSwYcOoW7cud955JxMnTnT1nsaY8EqYpOm2I0eOcPr0aW699Vby8/Pp27cvbdq0qfpEY0xMsaQZAocOHWLx4sWMHj2a5ORkvve97wHRsVanMeU5c+YMeXl5nD59OtKhhF1KSgqtW7emdu3a1TrfkmYI7Ny5kzFjxtCkSZNIh2JMQPLy8mjUqBHt27dHpLztvOKTqnLw4EHy8vLo0KFDta6REL3nbtm/fz/Lli2jV69eNGnSJGIzjowJ1unTp2nWrFlCJUwAEaFZs2Y1KmHHfdJ0a8HhgoIC3njjDbZu3UpmZiZbt26lZcuWnDlzJuwzjoypjkRLmD41fe64r567Mdzo0KFDNGrUiPHjxzNs2LDSkuWIESMYMWJEyO5jTDxr2LAhx48fj3QYQYv7pAmhHW6Ul5fH+++/zzfffMMrr7xSOpe9c+fOIbm+MaZyRUVFJCdHLnXFffU8lEpKSsjOzmb8+PF07tyZ5ORkq4obE0JLlizh6quvpmfPngwcOJB9+/YBMHPmTKZMmcKgQYP4/ve/z4EDB7jxxhu58sormTp1Ku3ataOgoACAF198kauuuoq0tDSmTp1KcXFxSGNMiJJmKOzatYszZ84watQocnJySE1N5Z133ol0WMbU2PTp00PeeZmWlsYTTzwR9HnXXXcdn3zyCSLCc889x6OPPsrvf/97ALKzs/nwww+pV68eP/3pT+nfvz/3338/b775Jh6PB4AtW7Ywf/58PvroI2rXrs20adOYO3cu3//+90P2bJY0A/Dll1+yevVqDh8+zI9//OPSKrmNwzQmtPLy8pg4cSL5+fkUFhaeNSxo5MiR1KtXD4APP/yQRYsWATBkyBDOO+88AN59912ys7Pp3bs34Kw01qJFi5DGGNdJ03+hjuo6c+YMycnJjBs3jhtvvDEi63Ea46bqlAjdctddd3H33XczcuRIli9fzsyZM0s/a9CgQenPqlru+arKD37wAx555BHXYozbNk3//cOrm+D+/e9/88Ybb9CuXTtq165N48aNS0uYtgiHMaF39OhRWrVqBcALL7xQ4XHXXXcdL7/8MgBvvfUWhw8fBmDAgAEsWLCA/fv3A85Il6+++iqkMcZtSdM31Ki6qwzt27ePjRs3MmbMmNL3srKy2LhxY8hiNCaRnTx5ktatW5e+vvvuu5k5cybjx4+nVatWXHPNNezcubPcc3/9619z6623Mn/+fDIyMmjZsiWNGjUiNTWVhx9+mEGDBlFSUkLt2rV56qmnaNeuXcjiloqKudEqPT1d165dW+VxmZmZQPXmf2/dupV27dpRp04datVyCuOrVq0CoG/fvkFfz5hos2XLFi677LJIh1Ft3377LUlJSSQnJ/Pxxx/zk5/8JKjOrPKeX0SyVTW9qnPjtqRZXTk5OezatYuOHTuWJkxfVT+cW2UYYyq2e/duJkyYQElJCXXq1OHZZ58N270tafpRVU6dOsXIkSPPSZhgK7AbEy06duzI+vXrI3JvS5peq1evJiUlhT59+pz1/nvvvQfYCuzGGEdcJs1ghxplZ2eX9rL97Gc/AyA5OZmsrCyysrLo37+/JUxjDBCnSTOYRTr27t1Lly5duPLKK7nhhhtKx2EWFRWxY8cOBg4cSFpamssRG2NiRVwmTah6kQ5VZeXKlSxevJjMzExGjBhROvvAOnuMMRWJ28HtVTl48CC1atUiOzubWbNmAc44TOvsMSZ8fvOb39CtWze6d+9OWloan376aY2v2bBhQ8BZL+Lyyy+v8fXKituSZkVUlXvvvZfVq1eTlJRUWh0H6Ny5sy3xZkyYfPzxxyxdupR169ZRt25dCgoKKCwsjHRYVYq7kmZlK7XPnj2b+++/n379+tG9e3cAm0duTITk5+eTmppK3bp1AUhNTeWiiy6iffv2zJgxgz59+pCens66desYPHgwl1xyCc888wwAx48fZ8CAAVx55ZVcccUV/POf/wxb3HFX0qyoE0hVmTt3LgcPHmTWrFmMGjUqEuEZE5V8M+j8TZgwgWnTpnHy5EmGDRt2zueTJ09m8uTJFBQUMG7cuLM+C6RfYNCgQTz44IN06tSJgQMHMnHixNIRL23atOHjjz/mv/7rv5g8eTIfffQRp0+fplu3btxxxx2kpKSwaNEiGjduTEFBAddccw0jR44MyxYecZU0/Yca+XcClZSUsHTpUk6dOkXz5s0jGKExxqdhw4ZkZ2fzwQcf8P777zNx4sTS/oWRI0cCcMUVV3D8+HEaNWpEo0aNSElJ4ciRIzRo0IAZM2awcuVKatWqxddff82+ffu48MILXY87rpJmRaXMTz/9lDVr1rB27doaLRNnTLyqrGRYv379Sj9PTU2t9oiTpKQkMjMzyczM5Iorrihd2chXZa9Vq1bpz77XRUVFzJ07lwMHDpCdnU3t2rVp37592PZwj7s2Tf9SZnFxMdnZ2VxzzTWcOnUKsKmQxkSLrVu38sUXX5S+zsnJCXg1oqNHj9KiRQtq167N+++/H/Ll3yoTNyXNsrOAioqKuPPOO3nrrbf49a9/TVZWFp06dbKZPcZEiePHj3PXXXdx5MgRkpOTufTSS/F4PCxdurTKc2+77TZGjBhBeno6aWlpdOnSJQwRe6lqTH316tVLy5ORkaGAzp49W0tKSvTIkSPau3fv0veMMd/ZvHlzpEOIqPKeH1irAeSguClpglM1/+EPf8iUKVPYsmUL27ZtC+n2vcYY42qbpogMEZGtIrJdRLLK+VxE5I/ezz8TkSure689e/awfv167rjjDqZPn07//v1tDKYxJuRcK2mKSBLwFHAjkAesEZHFqrrZ77ChQEfv19XA097vQbn99ttLG5S7d+9Ot27dePDBB2v4BMYYcy43S5pXAdtVdYeqFgIvAWVHlI8C/uZtUvgEaCoiLYO90ZIlSwAYPXo0P//5z2sYtjGJQWNsq5tQqelzu5k0WwG5fq/zvO8Fe0yVSkpKaNSoUek+yMaYyqWkpHDw4MGES5yqysGDB0lJSan2NdzsCCpvPlPZv6FAjkFEpgBTANq2bXvOCT/60Y+qEZ4xiat169bk5eVx4MCBSIcSdikpKWftghksN5NmHtDG73VrYE81jkFVPYAHnN0oy34eTZvdGxMLateuXbp+rAmOm9XzNUBHEekgInWAW4DFZY5ZDHzf24t+DXBUVfNdjMkYY2rEtZKmqhaJyE+BZUAS8LyqbhKRO7yfPwO8DgwDtgMngR+6FY8xxoSCq4PbVfV1nMTo/94zfj8rcKebMRhjTChJrPWeicgBoLzZ+alAQZjDcVs8PhPYc8WSeHwmKP+52qlqlWtHxlzSrIiIrFXV9EjHEUrx+ExgzxVL4vGZoGbPFXdLwxljjJssaRpjTBDiKWl6Ih2AC+LxmcCeK5bE4zNBDZ4rbto0jTEmHOKppGmMMa6LqaQZzvU5wymA57rN+zyficgqEekRiTiDUdUz+R3XW0SKRWRcRcdEk0CeS0QyRSRHRDaJyIpwx1gdAfwbbCIiS0Rkg/e5on4iiog8LyL7RWRjBZ9XL18Esrx7NHzhzCr6ErgYqANsALqWOWYY8AbOQiDXAJ9GOu4QPVdf4Dzvz0Oj/bkCeSa/497DmQAxLtJxh+jvqimwGWjrfd0i0nGH6LlmAP/r/bk5cAioE+nYq3iufsCVwMYKPq9WvoilkmbY1ucMsyqfS1VXqeph78tPcBY2iWaB/F0B3AUsBPaHM7gaCOS5JgGvqupuAFWNhWcL5LkUaCQiAjTESZpF4Q0zOKq6EifOilQrX8RS0gzb+pxhFmzMP8L57RjNqnwmEWkFjAGeIXYE8nfVCThPRJaLSLaIfD9s0VVfIM/1Z+AynFXIPgd+rqol4QnPNdXKF7G0sVrI1ueMMgHHLCI34CTN61yNqOYCeaYngF+oarFTeIkJgTxXMtALGADUAz4WkU9UdZvbwdVAIM81GMgB+gOXAG+LyAeq+o3LsbmpWvkilpJmyNbnjDIBxSwi3YHngKGqejBMsVVXIM+UDrzkTZipwDARKVLV18ISYfUE+m+wQFVPACdEZCXQA4jmpBnIc/0QmKVOY+B2EdkJdAFWhydEV1QvX0S6sTaIRt1kYAfQge8aq7uVOeYmzm7YXR3puEP0XG1xls/rG+l4Q/VMZY6fQ2x0BAXyd3UZ8K732PrARuDySMcegud6Gpjp/fkC4GsgNdKxB/Bs7am4I6ha+SJmSpoap+tzBvhc/w9oBvzFWzIr0iheRCHAZ4o5gTyXqm4RkTeBz4AS4DlVLXfIS7QI8O/rIWCOiHyOk2R+oapRvfqRiPwDyARSRSQP+DVQG2qWL2xGkDHGBCGWes+NMSbiLGkaY0wQLGkaY0wQLGkaY0wQLGkaY0wQLGmagHhXIsrx+2pfybHHQ3C/OSKy03uvdSLSpxrXeE5Eunp/nlHms1U1jdF7Hd+fy0bvKkBNqzg+TUSGheLeJjJsyJEJiIgcV9WGoT62kmvMAZaq6gIRGQQ8pqrda3C9GsdU1XVF5AVgm6r+ppLjJwPpqvrTUMdiwsNKmqZaRKShiLzrLQV+LiLnrGIkIi1FZKVfSex67/uDRORj77mviEhVyWwlcKn33Lu919ooItO97zUQkX9513rcKCITve8vF5F0EZkF1PPGMdf72XHv9/n+JT9vCfdmEUkSkd+JyBrvWotTA/hj+Rjvgg8icpU4a5+u937vLCJ1gAeBid5YJnpjf957n/Xl/TmaKBPpaU72FRtfQDHOgg05wCKcqXeNvZ+l4syq8NVcjnu/3wP80vtzEtDIe+xKoIH3/V8A/6+c+83BO7USGA98irMQxudAA5zlyTYBPYGbgWf9zm3i/b4cp1RXGpPfMb4YxwAveH+ug7PqTT1gCvCA9/26wFqgQzlxHvd7vleAId7XjYFk788DgYXenycDf/Y7/7fA7d6fm+LMUW8Q6b9v+6r4K2amUZqIO6Wqab4XIlIb+K2I9MOZLtgKZ07yXr9z1gDPe499TVVzRCQD6Ap85J0SWgenhFae34nIA8ABnNWdBgCL1FkMAxF5FbgeeBN4TET+F6dK/0EQz/UG8EcRqQsMAVaq6ilvk0B3+W5F+SZAR2BnmfPriUgOzhznbOBtv+NfEJGOOCvn1K7g/oOAkSJyr/d1Cs5aA1uCeAYTRpY0TXXdhrOCdy9VPSMiu3D+w5dS1ZXepHoT8HcR+R1wGHhbVW8N4B7/raoLfC9EZGB5B6nqNhHphTOP+BEReUtVHwzkIVT1tIgsx1n6bCLwD9/tgLtUdVkVlzilqmki0gRYCtwJ/BFnrvb7qjrG22m2vILzBbhZVbcGEq+JPGvTNNXVBNjvTZg3AO3KHiAi7bzHPAv8FWfrgU+Aa0XE10ZZX0Q6BXjPlcBo7zkNcKrWH4jIRcBJVX0ReMx7n7LOeEu85XkJZ7GG63EWrcD7/Se+c0Skk/ee5VLVo8DPgHu95zTBWQkInCq5zzGcZgqfZcBd4i12i0jPiu5hooMlTVNdc4F0EVmLU+r8dznHZAI5IrIep93xSVU9gJNE/iEin+Ek0S6B3FBV1+G0da7GaeN8TlXXA1cAq73V5F8CD5dzugf4zNcRVMZbOPvJvKPOdg/grF26GVgnzsZcs6miZuaNZQNwC/AoTqn3I5z2Tp/3ga6+jiCcEmltb2wbva9NFLMhR8YYEwQraRpjTBAsaRpjTBAsaRpjTBAsaRpjTBAsaRpjTBAsaRpjTBAsaRpjTBAsaRpjTBD+P0mU+1vVkWoDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr_large, tpr_large, _ = roc_curve(y_val, y_pred)\n",
    "fpr_small, tpr_small, _ = roc_curve(y_val, y_pred_small)\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(fpr_large, tpr_large, color='black', linestyle='solid', label='Large')\n",
    "plt.plot(fpr_small, tpr_small, color='black', linestyle='dashed', label='Small')\n",
    "plt.plot([0, 1], [0, 1], color='black', lw=0.7, linestyle='dashed', alpha=0.5)\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6633335483198801"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.46511760590981477"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_val, y_pred_small)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interpretation of AUC: the probability that a randomly chosen positive example\n",
    "ranks higher than a randomly chosen negative example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6691"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg = y_pred[y_val == 0]\n",
    "pos = y_pred[y_val == 1]\n",
    "\n",
    "np.random.seed(1)\n",
    "neg_choice = np.random.randint(low=0, high=len(neg), size=10000)\n",
    "pos_choice = np.random.randint(low=0, high=len(pos), size=10000)\n",
    "(pos[pos_choice] > neg[neg_choice]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-fold cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(df, y):\n",
    "    cat = df[categorical + numerical].to_dict(orient='records')\n",
    "    \n",
    "    dv = DictVectorizer(sparse=False)\n",
    "    dv.fit(cat)\n",
    "\n",
    "    X = dv.transform(cat)\n",
    "\n",
    "    model = LogisticRegression(solver='liblinear')\n",
    "    model.fit(X, y)\n",
    "\n",
    "    return dv, model\n",
    "\n",
    "\n",
    "def predict(df, dv, model):\n",
    "    cat = df[categorical + numerical].to_dict(orient='records')\n",
    "    \n",
    "    X = dv.transform(cat)\n",
    "\n",
    "    y_pred = model.predict_proba(X)[:, 1]\n",
    "\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = KFold(n_splits=10, shuffle=True, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "aucs = []\n",
    "\n",
    "for train_idx, val_idx in kfold.split(df_train_full):\n",
    "    df_train = df_train_full.iloc[train_idx]\n",
    "    y_train = df_train.converted.values\n",
    "\n",
    "    df_val = df_train_full.iloc[val_idx]\n",
    "    y_val = df_val.converted.values\n",
    "\n",
    "    dv, model = train(df_train, y_train)\n",
    "    y_pred = predict(df_val, dv, model)\n",
    "\n",
    "    rocauc = roc_auc_score(y_val, y_pred)\n",
    "    aucs.append(rocauc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.677, 0.684, 0.655, 0.648, 0.627, 0.622, 0.662, 0.647, 0.673,\n",
       "       0.668])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(aucs).round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc = 0.656 ± 0.020\n"
     ]
    }
   ],
   "source": [
    "print('auc = %0.3f ± %0.3f' % (np.mean(aucs), np.std(aucs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tuning the parameter `C`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(df, y, C=1.0):\n",
    "    cat = df[categorical + numerical].to_dict(orient='records')\n",
    "    \n",
    "    dv = DictVectorizer(sparse=False)\n",
    "    dv.fit(cat)\n",
    "\n",
    "    X = dv.transform(cat)\n",
    "\n",
    "    model = LogisticRegression(solver='liblinear', C=C)\n",
    "    model.fit(X, y)\n",
    "\n",
    "    return dv, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.001, auc = 0.656 ± 0.020\n",
      "C=0.01, auc = 0.656 ± 0.020\n",
      "C=0.1, auc = 0.656 ± 0.020\n",
      "C=0.5, auc = 0.656 ± 0.020\n",
      "C=1.0, auc = 0.656 ± 0.020\n"
     ]
    }
   ],
   "source": [
    "nfolds = 10\n",
    "kfold = KFold(n_splits=nfolds, shuffle=True, random_state=1)\n",
    "\n",
    "for C in [0.001, 0.01, 0.1, 0.5, 1.0]:\n",
    "    aucs = []\n",
    "\n",
    "    for train_idx, val_idx in kfold.split(df_train_full):\n",
    "        df_train = df_train_full.iloc[train_idx]\n",
    "        df_val = df_train_full.iloc[val_idx]\n",
    "\n",
    "        y_train = df_train.converted.values\n",
    "        y_val = df_val.converted.values\n",
    "\n",
    "        dv, model = train(df_train, y_train, C=C)\n",
    "        y_pred = predict(df_val, dv, model)\n",
    "        \n",
    "        auc = roc_auc_score(y_val, y_pred)\n",
    "        aucs.append(auc)\n",
    "\n",
    "    print('C=%s, auc = %0.3f ± %0.3f' % (C, np.mean(aucs), np.std(aucs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'model_C=1.0.bin'"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_file = f'model_C={C}.bin'\n",
    "output_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_out = open(output_file, 'wb')\n",
    "pickle.dump((dv, model), f_out)\n",
    "f_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(output_file, 'wb') as f_out:\n",
    "    pickle.dump((dv, model), f_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_file = 'model_C=1.0.bin'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_file, 'rb') as f_in:\n",
    "    dv, model = pickle.load(f_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(DictVectorizer(sparse=False), LogisticRegression(solver='liblinear'))"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dv, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "prospect_id= {\n",
    " 'lead_origin': 'lead_add_form',\n",
    " 'lead_source': 'reference',\n",
    " 'do_not_email': 'no',\n",
    " 'do_not_call': 'no',\n",
    " 'last_activity': 'sms_sent',\n",
    " 'country': 'india',\n",
    " 'specialization': 'select',\n",
    " 'how_did_you_hear_about_x_education': 'select',\n",
    " 'what_is_your_current_occupation': 'unemployed',\n",
    " 'what_matters_most_to_you_in_choosing_a_course': 'better_career_prospects',\n",
    " 'search': 'no',\n",
    " 'magazine': 'no',\n",
    " 'newspaper_article': 'no',\n",
    " 'digital_advertisement': 'no',\n",
    " 'through_recommendations': 'no',\n",
    " 'receive_more_updates_about_our_courses': 'no',\n",
    " 'tags': 'will_revert_after_reading_the_email',\n",
    " 'lead_quality': 'might_be',\n",
    " 'update_me_on_supply_chain_content': 'no',\n",
    " 'lead_profile': 'select',\n",
    " 'city': 'select',\n",
    " 'asymmetrique_activity_index': '02.medium',\n",
    " 'asymmetrique_profile_index': '02.medium',\n",
    " 'i_agree_to_pay_the_amount_through_cheque': 'no',\n",
    " 'a_free_copy_of_mastering_the_interview': 'no',\n",
    " 'last_notable_activity': 'sms_sent',\n",
    " 'newspaper': 'no',\n",
    " 'x_education_forums': 'no',\n",
    " 'get_updates_on_dm_content': 'no',\n",
    " 'lead_number': 627173,\n",
    " 'totalvisits': 0.0,\n",
    " 'total_time_spent_on_website': 0,\n",
    " 'page_views_per_visit': 0.0,\n",
    " 'asymmetrique_activity_score': 14.306252489048187,\n",
    " 'asymmetrique_profile_score': 16.344882516925527\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "X =dv.transform([prospect_id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22915116268136698"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_proba(X)[0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}