project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "train_data = pd.read_excel(\"data/cases_2021_train_processed_2.xlsx\")\n",
    "test_data = pd.read_excel(\"data/cases_2021_test_processed_unlabelled_2.xlsx\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.1 Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train_data.copy()\n",
    "train = train[['age', 'country', 'chronic_disease_binary', 'Case_Fatality_Ratio','outcome_group']]\n",
    "test = test_data.copy()\n",
    "test = test[['age', 'country', 'chronic_disease_binary', 'Case_Fatality_Ratio']]\n",
    "train_data, test_data = train,test "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.2 Feature Mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train_data.copy()\n",
    "train['country'] = pd.factorize(train['country'])[0]\n",
    "train['chronic_disease_binary'] = pd.factorize(train['chronic_disease_binary'])[0]\n",
    "new_label = {\"outcome_group\": {\"deceased\": 0, \"hospitalized\": 1, \"nonhospitalized\": 2}}\n",
    "train.replace(new_label, inplace = True)\n",
    "test = test_data.copy()\n",
    "test['country'] = pd.factorize(test['country'])[0]\n",
    "test['chronic_disease_binary'] = pd.factorize(test['chronic_disease_binary'])[0]\n",
    "train_data, test_data = train,test "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.3 Balancing Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Before Balancing,\n",
      "outcome_group\n",
      "0      997\n",
      "1    13241\n",
      "2     2974\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAGbCAYAAAAoSIKLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJm0lEQVR4nO3dd3hUdaLG8e9Mem+UEAiEkpCEXqSFqiCoqIiKuuwCu6LsetFrwV5ARbFcV1B3dd11CXbWgri6SFFgBZReJUAIJSEEAoQE0kgyM/eP0dEYWiDJmTnzfp4nT8jMmTPvJCHvnPL7HYvD4XAgIiIipmI1OoCIiIjUPRW8iIiICangRURETEgFLyIiYkIqeBERERNSwYuIiJiQCl5ERMSEVPAiIiImpIIXERExIRW8yBm8+OKLtGnTBh8fH7p27Wp0nHqzbNkyLBYLy5YtMzoKCQkJTJgwwegYIqagghePlp6ejsViqfbRpEkThgwZwoIFCy54vYsWLeKBBx4gLS2N2bNn8+yzz9Zh6gs3ePDgaq/V39+f1q1bc/vtt5OTk2N0PBFxI75GBxCpC0899RStW7fG4XBw+PBh0tPTufLKK/n3v//NyJEja72+b775BqvVyltvvYW/v389JL5wLVq0YMaMGQBUVFSwfft23njjDRYuXEhGRgbBwcEGJ7xwO3fuxGrVdodIXVDBiylcccUV9OzZ0/X1rbfeStOmTfnggw8uqODz8/MJCgqqs3J3OByUl5cTFBR00euKiIjgt7/9bbXbWrduzeTJk1m5ciXDhg276OcwSkBAgNERRExDb5XFlCIjIwkKCsLXt/p7WLvdzsyZM+nQoQOBgYE0bdqUSZMmcfz4cdcyFouF2bNnU1JS4toVnp6eDkBVVRVPP/00bdu2JSAggISEBB555BFOnTpV7XkSEhIYOXIkCxcupGfPngQFBfG3v/0NgMLCQu6++27i4+MJCAigXbt2PP/889jt9gt+vbGxsQDVXu/+/fu54447aN++PUFBQcTExHDjjTeyb9++c67v22+/5cYbb6Rly5YEBAQQHx/PPffcQ1lZWbXlJkyYQGhoKLm5uYwaNYrQ0FAaN27MlClTsNls1Za12+3MmjWLTp06ERgYSOPGjRkxYgTr1q1zLfPrY/A/HYJZuXIl9957L40bNyYkJITrrruOI0eO1Fj/tGnTiIuLIzg4mCFDhrB9+3Yd1xevpS14MYWioiKOHj2Kw+EgPz+fV199leLi4hpbupMmTSI9PZ3f//733HXXXezdu5fXXnuNjRs3snLlSvz8/HjnnXd48803WbNmDf/4xz8A6NevHwATJ05kzpw53HDDDdx3332sXr2aGTNmkJGRwbx586o9186dO7nllluYNGkSt912G+3bt6e0tJRBgwaRm5vLpEmTaNmyJatWreLhhx8mLy+PmTNnnvO12mw2jh49CkBlZSUZGRlMnTqVdu3akZaW5lpu7dq1rFq1iptvvpkWLVqwb98+Xn/9dQYPHsz27dvPuiv/o48+orS0lD/96U/ExMSwZs0aXn31VQ4cOMBHH31UI8/w4cPp3bs3//d//8eSJUt46aWXaNu2LX/6059cy916662kp6dzxRVXMHHiRKqqqvj222/5/vvvq+19OZ0777yTqKgopk6dyr59+5g5cyaTJ09m7ty5rmUefvhhXnjhBa6++mqGDx/O5s2bGT58OOXl5ef8noqYkkPEg82ePdsB1PgICAhwpKenV1v222+/dQCO9957r9rtX331VY3bx48f7wgJCam23KZNmxyAY+LEidVunzJligNwfPPNN67bWrVq5QAcX331VbVln376aUdISIhj165d1W5/6KGHHD4+Po7s7Oyzvt5Bgwad9vWmpKQ49uzZU23Z0tLSGo//7rvvHIDj7bffdt22dOlSB+BYunTpWR87Y8YMh8Vicezfv9912/jx4x2A46mnnqq2bLdu3Rw9evRwff3NN984AMddd91VY712u93171atWjnGjx/v+vqnn+/QoUOrLXfPPfc4fHx8HIWFhQ6Hw+E4dOiQw9fX1zFq1Khq6542bZoDqLZOEW+hXfRiCn/5y19YvHgxixcv5t1332XIkCFMnDiRTz/91LXMRx99REREBMOGDePo0aOujx49ehAaGsrSpUvP+hz/+c9/ALj33nur3X7fffcB8OWXX1a7vXXr1gwfPrzabR999BEDBgwgKiqqWoahQ4dis9n473//e87XmpCQ4HqtCxYsYObMmRQVFXHFFVdU2239y+P9lZWVHDt2jHbt2hEZGcmGDRvO+hy/fGxJSQlHjx6lX79+OBwONm7cWGP5P/7xj9W+HjBgAHv27HF9/cknn2CxWJg6dWqNx1oslnO+5ttvv73acgMGDMBms7F//34Avv76a6qqqrjjjjuqPe7OO+8857pFzEq76MUUevXqVW037y233EK3bt2YPHkyI0eOxN/fn8zMTIqKimjSpMlp15Gfn3/W59i/fz9Wq5V27dpVuz02NpbIyEhX2fykdevWNdaRmZnJli1baNy48QVlAAgJCWHo0KGur0eMGEH//v3p2bMnzz33HC+99BIAZWVlzJgxg9mzZ5Obm4vD4XA9pqio6KzPkZ2dzRNPPMHnn39e7fyE0z32p+PpvxQVFVXtcVlZWcTFxREdHX3O13c6LVu2rLF+wPUcP33vf/2ziY6Odi0r4m1U8GJKVquVIUOGMGvWLDIzM+nQoQN2u50mTZrw3nvvnfYxZyrdXzufLU7gtGfM2+12hg0bxgMPPHDaxyQlJZ3Xun+tR48eREREVNsDcOeddzJ79mzuvvtu+vbtS0REBBaLhZtvvvmsJ/TZbDaGDRtGQUEBDz74IMnJyYSEhJCbm8uECRNqPNbHx+eCMtfGmZ7jl29aRKQ6FbyYVlVVFQDFxcUAtG3bliVLlpCWlnZBw9VatWqF3W4nMzOTlJQU1+2HDx+msLCQVq1anXMdbdu2pbi4uNoWeF2x2Wyu1wrw8ccfM378eNcWPUB5eTmFhYVnXc/WrVvZtWsXc+bMYdy4ca7bFy9efMHZ2rZty8KFCykoKLjgrfiz+el7v3v37mp7To4dO1ZjD4SIt9AxeDGlyspKFi1ahL+/v6uMx4wZg81m4+mnn66xfFVV1TmL78orrwSocab7n//8ZwCuuuqqc+YaM2YM3333HQsXLqxxX2FhoetNSW0tXbqU4uJiunTp4rrNx8enxhbuq6++WmP42q/9tLX8y8c6HA5mzZp1QdkArr/+ehwOB08++WSN++piK/yyyy7D19eX119/vdrtr7322kWvW8RTaQteTGHBggXs2LEDcB7Hfv/998nMzOShhx4iPDwcgEGDBjFp0iRmzJjBpk2buPzyy/Hz8yMzM5OPPvqIWbNmccMNN5zxObp06cL48eN58803KSwsZNCgQaxZs4Y5c+YwatQohgwZcs6c999/P59//jkjR45kwoQJ9OjRg5KSErZu3crHH3/Mvn37aNSo0VnXUVRUxLvvvgs435js3LmT119/naCgIB566CHXciNHjuSdd94hIiKC1NRUvvvuO5YsWUJMTMxZ15+cnEzbtm2ZMmUKubm5hIeH88knn1zUlvCQIUP43e9+xyuvvEJmZiYjRozAbrfz7bffMmTIECZPnnzB6wZo2rQp//u//8tLL73ENddcw4gRI9i8eTMLFiygUaNG531YRcRMVPBiCk888YTr34GBgSQnJ/P6668zadKkasu98cYb9OjRg7/97W888sgj+Pr6kpCQwG9/+9tqY8jP5B//+Adt2rQhPT2defPmERsby8MPP3zas8NPJzg4mOXLl/Pss8/y0Ucf8fbbbxMeHk5SUhJPPvkkERER51zHgQMH+N3vfgc4zweIiopi0KBBTJ06tdpFcWbNmoWPjw/vvfce5eXlpKWlsWTJkhpn9v+an58f//73v7nrrruYMWMGgYGBXHfddUyePLnaHoLamj17Np07d+att97i/vvvJyIigp49e7rmGLhYzz//PMHBwfz9739nyZIl9O3bl0WLFtG/f38CAwPr5DlEPInFobNURMSkCgsLiYqKYvr06Tz66KNGxxFpUDoGLyKm8OtpdOHn8yUGDx7csGFE3IB20YuIKcydO9d1FcHQ0FBWrFjBBx98wOWXX35eh19EzEYFLyKm0LlzZ3x9fXnhhRc4ceKE68S76dOnGx1NxBA6Bi8iImJCOgYvIiJiQip4ERERE1LBi4iImJAKXkRExIRU8CIiIiakghcRETEhFbyIiIgJqeBFRERMSAUvIiJiQip4ERERE1LBi4iImJAKXkRExIRU8CIiIiakghcRETEhFbyIiIgJqeBFRERMSAUvIiJiQip4ERERE1LBi4iImJAKXkRExIRU8CIiIiakghcRETEhFbyIiIgJqeBFRERMSAUvIiJiQip4ERERE1LBi4iImJAKXkRExIRU8CIiIiakghcRETEhFbyIiIgJqeBFRERMSAUvIiJiQip4ERERE1LBi4iImJAKXkRExIRU8CIiIiakghcRETEhFbyIiIgJ+RodQETOofIUVJZCRSlUlP34+ccPWyU47M4Pux2wg8PhfJzFChaL8zMW8PEFv0DwCwb/IOeHXxD4Bzs/+/ob+SpFpI6p4EWMZKuC0gIoPgYlx5yfy0/8XOCVZWC3NUwWq8/P5R8QAsHREBoDITEQEg3BkT++WRART2BxOH56uy8i9aa8+McCP/pzkZccg7LCn7e43Z3VB4KjnIXvKv4YCGvk3AMgIm5FBS9S1+x2OJEHBTlwPAeOH4BTxUanql8hMRDVAqLinZ9DGzkPD4iIYVTwIherstxZ4sdznKVedNB5bNyb+QX9WPg/ln5kHPj4GZ1KxKuo4EVqq6oCju6Bo3uhIBtOHgH03+isLFYIj4WYVtCkHUS1BKuO54vUJxW8yPkoK4LDuyA/E47ta7gT38zKNxAat4Emic7C9w82OpGI6ajgRc6k+CjkZcChDDhx2Og05mWxQHQriE12fgSGGZ1IxBRU8CK/dDIf8rY7i734qNFpvFNUC4hNgbgOKnuRi6CCF6mqgIM/QPYG5wly4h4sFucu/Phuzt34GoMvUisqePFeRXnOUj+4zVny4r4Cw6BFV4jv6pxwR0TOSQUv3qWqwlno2RucBS8exuI8OS++GzRNck6+IyKnpYIX7/DT1nruNrBpa90UAkKgeWdo1cM5w56IVKOCF3M7ugcyv3WOVxdzsligWQdo1x/CGhudRsRtqODFnPIzncVemGt0EmlIscnQbgBExBqdRMRwKngxD4fDORnN7m91fN3bNUl0Fn1Uc6OTiBhGBS+ez+FwTkaT+a1zHLvITxq1dhZ9TCujk4g0OBW8eC6Hw3lG/O4VmpRGzi66JSQOdBa+iJdQwYtnKsiGH77SFLJSO03bQ+ownXUvXkEFL56l/ARkLHHOPCdyIaw+0LqP86x7X3+j04jUGxW8eAZbFez93rk73tuvtS51IzAMki+D5p2MTiJSL1Tw4v4O74Tti6H0uNFJxIyiWkCHERDRzOgkInVKBS/uq/gobF8ER7KMTiKmZ4EWXSD5UucMeSImoIIX92OrhF3LYe9qcNiNTiPexDfAudu+ZXfnDHkiHkwFL+7l+AHYPB9KCoxOIt4spjV0Hqkr14lHU8GLe7BVwa5lzhPp9Csp7sDXH1KGObfmRTyQCl6MV5QHm+ZD8RGjk4jU1KgNdLnGeda9iAdRwYtxHA7IWuXcctexdnFnfkHQ6SpolmJ0EpHzpoIXY5SdgE2fQcF+o5OInL8WXZxD6jRBjngAFbw0vLztsPVLqCw3OolI7QVHQbfREBlndBKRs1LBS8Nx2J3TzO5dbXQSkYtj9YGOV0F8F6OTiJyRCl4aRkUZbPwEju41OolI3Um4BFIuB6vV6CQiNajgpf6dzId1/9JUs2JO0a2gxw3gH2x0EpFqVPBSvw7tcA6Bs1UYnUSk/gRFQI8xEBFrdBIRFxW81A+HAzL/6/wQ8QZWX+h8NTTvaHQSEUAFL/WhqsI5BO7wTqOTiDS8Nn2d89lrLnsxmApe6lZJAaz/F5zUrHTixRq3hW7Xg1+A0UnEi6ngpe4UHYI170NFidFJRIwX0Qx6/UYn34lhVPBSNwpyYO2HUKXJa0RcQhtB77EQGG50EvFCKni5eEf3OIfB2SqNTiLifoIinSUfEm10EvEyKni5OId2OiewsduMTiLivgJCnSUf1sToJOJFVPBy4XK3wubPdSU4kfPhFwS9boHI5kYnES+hgpcLs389bFsA6NdH5Lz5+EPPm6BRgtFJxAuo4KX2slbBjq+NTiHimay+0H00NG1vdBIxORW81M6uZZD5rdEpRDybxQrdr4fYZKOTiInpEkhy/vauVrmL1AWHHTZ+Csf2GZ1ETEwFL+cndytsX2R0ChHzsNucw0uL8oxOIialgpdzy9/tPFteROpW1SlY84FzimeROqaCl7M7ngsbPtZQOJH6UlECq9+D8pNGJxGTUcHLmRUfhbUfaIY6kfpWVui8jkOlpnqWuqOCl9MrO/HjH5wyo5OIeIeT+c7rOegNtdQRFbzUVFnmLPeyIqOTiHiX4zmw4ROw65CYXDwVvFRnq3RuRRTreu4ihsjPhK1fGJ1CTEAFL9Vt+TccP2B0ChHvdmCzc94JkYuggpef7fkODv5gdAoRAchYAkf3Gp1CPJgKXpyO7tH88iLuxGF3Ho8vLTQ6iXgoFbw4/4Bs+BR0WQIR91JZBuv/pTPr5YKo4L2drcr5B0TD4UTc04nDsPVLo1OIB1LBe7sfvnL+ARER95W7FfavNzqFeBgVvDc7sAVyNhqdQkTOx/aFujCN1IoK3ludzIdt/zE6hYicL7sN1n+s6WzlvKngvZGt0nl2rk7cEfEsZYXOuSpEzoMK3hvt+MZ5IRkR8TyHdkDuNqNTiAdQwXubgmzYt8boFCJyMX74CsqLjU4hbk4F701slbBZu/dEPF5lGWzT0Dk5OxW8N9nxDZQWGJ1CROrC4V3OkTAiZ6CC9xbaNS9iPtsXQvlJo1OIm1LBewPtmhcxp8pyzXInZ6SC9wbaNS9iXvmZzsvLivyKCt7sCrJh31qjU4hIffphEZSfMDqFuBkVvJm5ds3rKnEiplZVDls1M6VUp4I3s6yV2jUv4i3yMyF/t9EpxI2o4M2q/ATs+d7oFCLSkDIWg91udApxEyp4s9qxVHPNi3ib4qOQvcHoFOImVPBmVJQHuZoAQ8QrZS7XFecEUMGb0/bFRicQEaNUlELmt0anEDeggjebQzugYL/RKUTESPvXQolOsPV2Kngzsdtgx9dGpxARo9ltkLHE6BRiMBW8mexfp3ftIuJ0eCcc0948b6aCN4uKMsj8r9EpRMSdbF8EDk105a1U8Gax+1udOSsi1Z04BAe3GZ1CDKKCN4NTJbB/vdEpRMQdZa0yOoEYRAVvBvvWgL3K6BQi4o5O5sPhTKNTiAFU8J6uqsJ5cp2IyJlkrTQ6gRhABe/psjfo2LuInN3xHOelo8Wr1KrgBw8ezN13311PUc5t2rRpdO3atc7XM2HCBEaNGnXR6z2bhIQEZs6cWbcrtdtg7+q6XaeImJOOxXsdX6MD1MaUKVO48847XV9PmDCBwsJCPvvss4ta76xZs3B44lCS3G3Oq8aJiJxLfiacOAzhTY1OIg3Eo3bRh4aGEhMTU+frjYiIIDIyss7XW68cDtijd+QiUgvaivcqtS54u93OAw88QHR0NLGxsUybNs11X3Z2Ntdeey2hoaGEh4czZswYDh8+7Lp/8+bNDBkyhLCwMMLDw+nRowfr1jlPEEtPTycyMpLPPvuMxMREAgMDGT58ODk5Oa7H/3LX+rRp05gzZw7z58/HYrFgsVhYtmwZAA8++CBJSUkEBwfTpk0bHn/8cSorz3zp1F/uot+3b59rfb/8GDx4sGv5FStWMGDAAIKCgoiPj+euu+6ipKTEdX9+fj5XX301QUFBtG7dmvfee6+23+ZzO7zLeWlIEZHzlfcDlB43OoU0kFoX/Jw5cwgJCWH16tW88MILPPXUUyxevBi73c61115LQUEBy5cvZ/HixezZs4ebbrrJ9dixY8fSokUL1q5dy/r163nooYfw8/Nz3V9aWsozzzzD22+/zcqVKyksLOTmm28+bY4pU6YwZswYRowYQV5eHnl5efTr1w+AsLAw0tPT2b59O7NmzeLvf/87L7/88nm9vvj4eNf68vLy2LhxIzExMQwcOBCArKwsRowYwfXXX8+WLVuYO3cuK1asYPLkya51TJgwgZycHJYuXcrHH3/MX//6V/Lz82v7rT47vRMXkdpyOCDrO6NTSAOp9TH4zp07M3XqVAASExN57bXX+Ppr5wVOtm7dyt69e4mPjwfg7bffpkOHDqxdu5ZLLrmE7Oxs7r//fpKTk12P/6XKykpee+01evfuDTjfTKSkpLBmzRp69epVbdnQ0FCCgoI4deoUsbGx1e577LHHXP9OSEhgypQpfPjhhzzwwAPnfH0+Pj6u9ZWXlzNq1Cj69u3r2lMxY8YMxo4d6zrZMDExkVdeeYVBgwbx+uuvk52dzYIFC1izZg2XXHIJAG+99RYpKSnnfO7zVpANhQfqbn0i4j0ObIb2g8E/2OgkUs9qvQXfuXPnal83a9aM/Px8MjIyiI+Pd5U7QGpqKpGRkWRkZABw7733MnHiRIYOHcpzzz1HVlZWtXX5+vq6ShEgOTm52uPP19y5c0lLSyM2NpbQ0FAee+wxsrNrP0TkD3/4AydPnuT999/HanV+qzZv3kx6ejqhoaGuj+HDh2O329m7dy8ZGRn4+vrSo0ePGq+jzuxbW3frEhHvYq+CA1uMTiENoNYF/8td6gAWiwW73X5ej502bRo//PADV111Fd988w2pqanMmzevthHO6rvvvmPs2LFceeWVfPHFF2zcuJFHH32UioqKWq1n+vTpLFy4kM8//5ywsDDX7cXFxUyaNIlNmza5PjZv3kxmZiZt27at09dyWhWlzqtEiYhcqJyNRieQBlBnw+RSUlLIyckhJyfHtRW/fft2CgsLSU1NdS2XlJREUlIS99xzD7fccguzZ8/muuuuA6Cqqop169a5dsfv3LmTwsLCM+7e9vf3x2azVbtt1apVtGrVikcffdR12/79tbtk4ieffMJTTz3FggULapR29+7d2b59O+3atTvtY5OTk6mqqmL9+vWuvRE/vY46kbvVOf5dRORCFR91HuqLbml0EqlHdTZMbujQoXTq1ImxY8eyYcMG1qxZw7hx4xg0aBA9e/akrKyMyZMns2zZMvbv38/KlStZu3ZttfL28/PjzjvvZPXq1axfv54JEybQp0+fGsfff5KQkMCWLVvYuXMnR48epbKyksTERLKzs/nwww/JysrilVdeqdVegm3btjFu3DgefPBBOnTowKFDhzh06BAFBc7rrD/44IOsWrWKyZMns2nTJjIzM5k/f77rJLv27dszYsQIJk2a5HodEydOJCgo6CK+u7+QrXfeIlIH9LfE9Oqs4C0WC/PnzycqKoqBAwcydOhQ2rRpw9y5cwHnyWvHjh1j3LhxJCUlMWbMGK644gqefPJJ1zqCg4N58MEH+c1vfkNaWhqhoaGux5/ObbfdRvv27enZsyeNGzdm5cqVXHPNNdxzzz1MnjyZrl27smrVKh5//PHzfh3r1q2jtLSU6dOn06xZM9fH6NGjAec5CMuXL2fXrl0MGDCAbt268cQTTxAXF+dax+zZs4mLi2PQoEGMHj2a22+/nSZNmtT2W1rT8QNQfOTi1yMiXq+iuJCKqvM7vCqeyeJwkync0tPTufvuu+tuV7YZbf3SOfe8iMgFcPiHcCyyA1t8UjlQFU3fBH8Sm/id+4HikTxqqlqvZquCg9uNTiEiHsZhsVIW1ZbdAR3YUpWA3WGFH68uvedYlQrexFTwniI/E6p01TgROT+2kMYcDOvABkcyRfZgV6n/0uGTdopP2QkN8KhZy+U8uc0uejmHdXOd09OKiJyBwzeAoqgUfvDtQFbV+V1UpmsLPzrH+ddzMjGCtuA9QUUp5O82OoWIuCEHUBGRwN7gDmyytaHC4XfarfUz2XO0SgVvUip4T3BoJzh0tquI/MweGMHhiI5sJoV8e3itSv2XTpQ7OFpso1GoT90GFMOp4D1BvnbNiwg4rL4UR7Vnp38Htlc2B7ulTtabc1wFb0YqeHdnq4Kje41OISIGqgyLIzu0IxttiZQ6Apxb63XT7QDkFtnoFn/u5cSzqODdXcF+sJ35WvYiYk6/HrN+obvgz0dBqZ2ySgdBfnX4rkEMp4J3d/mZRicQkQZytjHr9S2vyEabRqoEM9FP093p7HkR0zufMev1LbeoSgVvMvppurPio1B63OgUIlIPnGPWU/nBN9U5Zt3gi0TmFdlwOBxYLNpNbxYqeHem3fMipnKxY9brU3mV81h8TIjOpjcLFbw7O6yCFzGDn8asbyKVI/Ywtyn1X8stsqngTUQF764qy+F4jtEpROQC1deY9fp0sNBG57hzLyeeQQXvro7u0ex1Ih6ovses16cjJXYqbA78fTwksJyVCt5dHdtvdIILlvCHP7M/v7DG7Xdc1Yv7R6fR+taXT/u4fz00hhv7d6TgZCnj//wpS7fuIzEumn/+73V0a9vMtdz/vP4FbZpGcd/otPp6CSK10pBj1uuTw+E82a5VtKrBDPRTdFeFuUYnuGBrX56Ezf7z3odt+/MZ9tgcbkzrQHyjCPLeub/a8m9+tY4XP13JFT0SAXhm7n85WVbBhll/5PX/rOW2V+ezbuYfAfh+Rw6rdx7glduvbLgXJHIaP49Z78iWqlYNOma9Ph2s54IfPHgwXbt2ZebMmfX2HEazWCzMmzePUaNGGZpDBe+ObFVw4rDRKS5Y44iQal8/99G3tG0WzaBOCVgsFmKjwqrdP++7DMb070hoUAAAGTlHuHlgJ5KaN+L2ET1586t1AFRW2fjjX/7NP+66Fh8fXb9ajOEas25PpshhzJj1+nSwyODxelJnVPDu6MQh0xx/r6is4t1lW7h3VN/Tjq9dv/sgm/Yc4i9/Gum6rUvrWL7ZsoeJw7uzcEMmnROc17V+4ZMVDO6UQM/E5g2WXwR+HrO+zTeVPW4wZr0+lVQ4KKmwE+KvN9GeTj9Bd+TBu+d/7bPvd1BYXM6Ey7qd9v63Fq0nJb4x/VJaum576MYB+PpYaTtxJvO+y+Ct/x1FZu4x5ny9icdvHswfX/ucNre+zJjn5lJUUt5QL0W8jAM4FZHAjmZXMTf6dj5niLPcvcCxkrrZwCgpKWHcuHGEhobSrFkzXnrppWr3nzp1iilTptC8eXNCQkLo3bs3y5Ytq7bMypUrGTx4MMHBwURFRTF8+HCOH3dOAGa325kxYwatW7cmKCiILl268PHHH7sea7PZuPXWW133t2/fnlmzZlVb/7Jly+jVqxchISFERkaSlpbG/v0/nwM1f/58unfvTmBgIG3atOHJJ5+kqurn3TaZmZkMHDiQwMBAUlNTWbx4cZ187+qCtuDdUeFBoxPUmbcWreeKHu2IiwmvcV/ZqUreX76Vx28aVO32iJBA3r//xmq3XfrIbF78w+W8t2wLew4fZ+ff7uK2V+fz1AfLeGniiHp9DeJdPGXMen0qKLHTMuri13P//fezfPly5s+fT5MmTXjkkUfYsGEDXbt2BWDy5Mls376dDz/8kLi4OObNm8eIESPYunUriYmJbNq0icsuu4w//OEPzJo1C19fX5YuXYrN5tyFMmPGDN59913eeOMNEhMT+e9//8tvf/tbGjduzKBBg7Db7bRo0YKPPvqImJgYVq1axe23306zZs0YM2YMVVVVjBo1ittuu40PPviAiooK1qxZ49rb+O233zJu3DheeeUVBgwYQFZWFrfffjsAU6dOxW63M3r0aJo2bcrq1aspKiri7rvvvvhvXB2xOBwOh9Eh5FeWvmaKKWr35xfSZuLLfPrIzVzbJ6XG/e98s4lbX5lP7pwpNY7b/9LsxRv495qdfProLYx+5gOGdm3LHVf14su1O3ni3W9YP+tP9fkyxAvUGLPu5dO1No/w4bL2gRe1juLiYmJiYnj33Xe58UbnG/aCggJatGjB7bffzr333kubNm3Izs4mLu7nwfdDhw6lV69ePPvss/zmN78hOzubFStW1Fj/qVOniI6OZsmSJfTt29d1+8SJEyktLeX9998/ba7Jkydz6NAhPv74YwoKCoiJiWHZsmUMGjSoxrJDhw7lsssu4+GHH3bd9u677/LAAw9w8OBBFi1axFVXXcX+/ftdr+Grr77iiiuu0El2choVpaYod3AWc5OIEK66JOm097+1aAPX9Gp/1nI/UlTCUx8uY8XzEwGw2R1UVjnfvVdW2bHZ9f5ULpwnj1mvT8dKL34XfVZWFhUVFfTu3dt1W3R0NO3btwdg69at2Gw2kpKq/304deoUMTExAGzatMn15uDXdu/eTWlpKcOGDat2e0VFBd26/XxI8C9/+Qv//Oc/yc7OpqysjIqKCtcehOjoaCZMmMDw4cMZNmwYQ4cOZcyYMTRr5hyWu3nzZlauXMkzzzzjWp/NZqO8vJzS0lIyMjKIj4+v9gbll282jKaCdzcm2T1vt9uZvWQj4y/riq9Pzakvdx88xn9/2M9/pv32rOu5+80F3DcqjeaNnLv401LieWfpZi7v3o43v1pHWmrLsz5e5Ncc/iEcjezAVg8fs16fyivr/0S74uJifHx8WL9+PT6/+hsRGhoKQFBQ0FkfD/Dll1/SvHn1E28DApwjcj788EOmTJnCSy+9RN++fQkLC+PFF19k9erVrmVnz57NXXfdxVdffcXcuXN57LHHWLx4MX369KG4uJgnn3yS0aNH13j+wMCL28PREFTw7sYkJ9gt2bSH7CNF/GFY99Pe/8/FG2jRKJzLu7U94zoWrs9kd94x3rnv5/9ck0f2Zt3ug/S+9016JTVn6i2D6zq6mJBzzHq7H6+zbp4x6/WpoOTiCr5t27b4+fmxevVqWrZ0vhE/fvw4u3btYtCgQXTr1g2bzUZ+fj4DBgw47To6d+7M119/zZNPPlnjvtTUVAICAsjOzj7t7nVwnqDXr18/7rjjDtdtWVlZNZbr1q0b3bp14+GHH6Zv3768//779OnTh+7du7Nz507atWt32vWnpKSQk5NDXl6ea6v/+++/P/s3pgGp4N2NSbbgL+/eDscXT53x/mfHD+PZ8cPOeD/A8B6JDP9x8pufBAf686+HbqqTjGJ+zjHrHdlgb2/KMev1qbDMTvxFnGgXGhrKrbfeyv33309MTAxNmjTh0UcfxWp1vmlISkpi7NixjBs3jpdeeolu3bpx5MgRvv76azp37sxVV13Fww8/TKdOnbjjjjv44x//iL+/P0uXLuXGG2+kUaNGTJkyhXvuuQe73U7//v0pKipi5cqVhIeHM378eBITE3n77bdZuHAhrVu35p133mHt2rW0bt0agL179/Lmm29yzTXXEBcXx86dO8nMzGTcuHEAPPHEE4wcOZKWLVtyww03YLVa2bx5M9u2bWP69OkMHTqUpKQkxo8fz4svvsiJEyd49NFHL/p7X1dU8O6m+IjRCUQ8mjeNWa9PhWUXfxz+xRdfpLi4mKuvvpqwsDDuu+8+ioqKXPfPnj2b6dOnc99995Gbm0ujRo3o06cPI0c658VISkpi0aJFPPLII/Tq1YugoCB69+7NLbfcAsDTTz9N48aNmTFjBnv27CEyMpLu3bvzyCOPADBp0iQ2btzITTfdhMVi4ZZbbuGOO+5gwYIFAAQHB7Njxw7mzJnDsWPHaNasGf/zP//DpEmTABg+fDhffPEFTz31FM8//zx+fn4kJyczcaLznCCr1cq8efO49dZb6dWrFwkJCbzyyiuMGOEeI3t0Fr07sdvgqxnOCaFF5Lw5gIrIBPYEdWBTVVsqte1y0aKCrVzd8czHwMX96X+BOykrVLmL1ILGrNefojI7docDq5cPGfRkKnh3UmKO4XEi9ckTr7PuiewOKC53EB6k76+n0lS17qS00OgEIm6rMiyOrGaX83GjScyzDmd7VYs6n5CmIP8gb0y9lTuGtWTiwEY8+pte7M3Y4Lr/P+/OYvKIBCaPSGDBe69Ue2zWtrU8Ma4/tirz7Eaoi+PwYhxtwbsTk0xwI1JXfhqzvsWaSq6tfsesl5w4zjO3DyW5+0Dum/kp4VGNOJSdRXBYJADZmduY9+Z07vnzRzgcDl6+70Y69r6U+HYdsVVVkf78//L7h1/Fx9c8f1ZPntIhQ09mnt9EM1DBi5x+zHoDnAn/5TsvE92kObc98YbrtsZxCa5/5+3fSXy7jqT2HAxAfLuO5O3fRXy7jvzn3Zm075pGm9Qe9R+0AZVVquA9mQrenajgxYsZPWZ943+/pGOfobz28G/ZsXEFUY3juOz62xg86vcAxLftwKGc3Rw7lIPD4eBQ9m5atEnl8IE9fPvFuzw559uGDdwAylXwHk0F705U8OJl3GnM+pGD+1j66T8YfsudXD3hfvZsX8+7f74fXz9/+l81lrjWydzwp6m8cOc1ANx4xzTiWifz/OSR3HTn02z7fgnz/vEsPr5+jL33BZK79TfuxdSRskodg/dkKnh3caoYbJVGpxCpd6cds+4G56XZ7XZap3TnxjumAdCqfRdy92znm0/fov9VYwG4dPRELh090fWYFV++R2BwKO069uKhMd2ZOns5x/Nzef2xCfzfvB/w8w8w4qXUmXL9SfJoKnh3oTPoxeTsQZEcDu/IJlLccsx6ZKNY4lonV7utWUJ71i6df9rlTxYe5bN/zOCRNxaS9cM6mrZsR+yPH7aqSg5lZxLfrmNDRK83ZVXaRe/JVPDuovyE0QlE6txPY9Z3+Hcgw83HrCd27sOh/buq3XYoezeNYk9/xcL3X36I4bf8D9FNm7MnYz22qp83d202G3a75+/ePlXp0GQ3HkwF7y4qy41OIFJnKsOakx3agQ22RMo85Drrw2+ZzPSJl/Hv9Bfpddlo9mxfz7LPZvP7h1+tsey21d9wKGc3t019E4A2KT3I27+LzasWUXD4AFarlWYtE2s8ztM4gFNVEORndBK5EJqL3l3sXgk7vzE6hcgFqzFm3QNtWrGAj/46lcM5WTSKa8WIW+50nUX/k4ryMh7/XT/ueGYOrZI6u25fNj+dT994Cl//AMbd/zJd+7vHBUcu1tUdg4gK1pxonkgF7y4ylsCe74xOIVIrDouVssh2ZAZ2YGtVK+yaHNN0hrYPIC5CO3s9kX5q7qKizOgEIufNFtKY3LCObNR11k1Pk914LhW8u6hUwYt7c/gGUhSV4hZj1qXhaKic51LBuwtbhdEJRGpw1zHr0nC0Be+5VPDuokoFL+7D3cesS8M5pbHwHksF7y60BS8G86Qx69JwbHYVvKdSwbuLKh3oEmN44ph1aTgaZ+W5VPDuQlvw0oAa8jrr4tk8fz4+76WCdxcmmNZS3FuNMesNdJ118WzagvdcKnh3YdUEIVI/NGZdLoYOwXsuFby7sKjgpe44fAMpjErhB41Zl4ukyU49lwreXajgpY44sPBxl06UUAVs//FD5MJU+TUC+hgdQy6ACt5dqOCljlhwkHSihK9DS4yOIiYQRoDREeQCqVXchY7BSx3quHsHEdZgo2OICVhUEx5LPzl3oS14qUM+Djt9j2tkhlw8qyZF8FhqFXehgpc6lrJnFzHWUKNjiIezWFTwnkqt4i5U8FLHLDjof6Tc6Bji4XxUEx5LPzl3oWPwUg/aZmfRzBpudAzxYIFWnWTnqdQq7kJb8FJP+h88YXQE8WABVn+jI8gFUqu4Cx8/oxOIScXn7aeVJdLoGOKhAlXwHksF7y78Q4xOICbWP+eI0RHEQwVatIveU6ng3UWAznaW+tP0yEESiTI6hnggbcF7LhW8uwjQFrzUr7S9B7BoTLPUUoBOsvNYKnh3oS14qWfRhUdJtUcaHUM8jLbgPZcK3l2o4KUB9Mvaq3HNUiuBFhW8p9L/dHehXfTSAMKKi+hSFWF0DPEgGgfvuVTw7kJb8NJAeu3OxM+iC0nKuflafPC3agivp1LBuwv/YNCcz9IAgstK6HFKe4zk3CJ8woyOIBdBBe8uLBaNhZcG0yNzl46tyjlF+mrPoidTwbsTHYeXBhJQUU6vUh1blbPTFrxnU8G7k0BdFEQaTtfMHYRagoyOIW4s0lcF78lU8O4ktJHRCcSL+Nqq6HNC533ImWkL3rOp4N1JWGOjE4iX6Zi1k0irDg3J6WkL3rOp4N1JWBOjE4iXsTrs9CuoMjqGuCELFsJ9dJKdJ1PBu5PQRqC5wqWBtd+7i8ZWbalJdWE+IfhYVBGeTD89d+LjB8GRRqcQL2MB0vJLjY4hbiZSW+8eTwXvbnQcXgzQJmcPzS2awlZ+Fu2n3wdPp4J3N6EqeDFG/4PHjY4gbiTWT6N6PJ0K3t1oC14M0vxQDq0tkUbHEDfR1C/G6AhykVTw7kYFLwZKyz5sdARxA/4WP6J9tYve06ng3U1II110RgzT5Ogh2hNldAwxWBO/aCz6O+TxVPDuxsfXWfIiBum3Jwerhmt6NR1/NwcVvDuKamF0AvFiUUXH6GCPNDqGGCjWX8ffzUAF746i441OIF6uT9YefPAxOoYYpKm24E1BBe+OolsanUC8XFjxCbpWanY7bxRkDSBC14E3BRW8OwqOgkD9cRVj9dqdib/Fz+gY0sA0PM48VPDuKkq76cVYQeWl9CwPNjqGNLAW/k2NjiB1RAXvrmISjE4gQvfMnQRbAoyOIQ0oITDO6AhSR1Tw7qpRgtEJRPCvPEWvEn+jY0gDCbYG0tg32ugYUkdU8O4qJAYCw41OIULn3RmEWYOMjiENoFVAnCa4MREVvDtr1NroBCL42mz0LTQ6hTSEVgHaPW8mKnh3puPw4iZS9+wk2hpidAypZwkBzYyOIHXI1+gAchaN2wIWwGF0EvFyVoeDfscq+cLNp6nP+n4Hy15fwIGt+zhxuJAJb91FpxE9XPff13z8aR838rGbGPKnK6k6Vcm/pvyTbYs2ENY4guufHU/SwA6u5Za+/h+O5x5j9PTf1ftraWhN/KIJ9tGhGDNRwbuzgBDnpDcF+41OIkLSvkyaxnTisP2E0VHOqKL0FHGp8fS6eQDpE1+tcf/UjbOqfb1j6Rb+dd8/6XxlTwC+e28ZB7bu467PHydj6Rbem/w60za/isVi4Vj2Eb5/bxn3LHiyQV5LQ0vQ7nnTUcG7u2YpKnhxG2mHivm0idEpzizl0i6kXNrljPeHN4ms9vW2hRtp2y+FmFbOF5WfeZDUy7sR274FMS2b8MXTcykpOEloTDifPDyHkY+OITDMnFu5Ov5uPjoG7+5ik0FX9hI3kZC7lxYWc1wn/OSRIjK+3kzvWwa6botLbcneNbuoLKtgx/KthDeNJCQ6jPWfrsI3wI9OV/Q0MHH98bf4Eefvxu/c5IJoC97dBYY5Lz5TkG10EhEA+uce48M4z//TsfajFQSEBtLpip+P0fe6eQAHM3J4YcjDhESH8bs3/oeywhIW/t+n/Omjh1nw/Mds/Hw1Ma2acPNLtxLRzBxjxtsFtsTHou09s/H8/6XeIDZFBS9uI+5wLm3jupBFodFRLsqaD7+l+3V98Qv8eSIfHz9frn92XLXlPrzn7/T/w+Xk/rCfbQs3cN/i6Sz965fMe+I9Jvz9zoaOXS+SgzQk14z0ls0TNEs2OoFINWnZeVg8+NDRntU7OZKVR59bBp11ud0rMzi0K5f+vx9K1qodJF/ahYDgALpe3YusVRkNlLZ+BVsDaRkQa3QMqQcqeE8QGA5RLYxOIeLS6Fg+yY5Io2NcsNUf/JcWnROI63DmSzNXllfw6aNvc8Pzv8fqY8Vht2OvrALAVmnDbjfH8NWkoASs2j1vSvqpeopmqUYnEKmm795srG72J+RUSTm52/aTu8058qQg+wi52/ZzPPeYa5nyk2Vs+WINvc+x9b545uckX9qZFh1bAZDQM5GtC9ZzcHs2K9KX0LpnYv29kAaUot3zpqVj8J4iNhm2LzI6hYhLZFEBnWwt2exz3OgoLjmb9/L6jc+5vv78yQ8A6Hljf26ZeRsAG+d/j8MB3Ub1OeN68nYcYPO/13Dv4qddt3UeeQlZ3+3gL6OfpXHbWH772p/q6VU0nAifMJr5NzY6htQTi8PhMMd+Jm+w8p9QmGt0ChGXkuAw3mrfmCpsRkeRC9A7tBNp4d2MjiH1xL32r8nZxes/oriXkNKTdKsMMzqGXKCUoDZGR5B6pIL3JM07gm+g0SlEqrlk1y4CLH5Gx5BaauIXTbSfOSYtktNTwXsSHz+IP/M0nCJGCKwo45Iyc07famYdg9sZHUHqmQre07Tqce5lRBpYt8ydhFgCjI4h5ynA4kdqUFujY0g9U8F7mpAYaKTjZuJe/Koq6F2s3fSeokNwO/yt+nmZnQreE7Uy5wUvxLN1yswgwhpsdAw5BwsWuoZodkxvoIL3RE0TIUgnx4h78XHY6VtoNzqGnEPrgOZE+mrkgzdQwXsiixVadjc6hUgNKVm7iLGGGh1DzqJbqLbevYUK3lPFdwOrj9EpRKqx4CDtaLnRMeQMYnwjaBUQZ3QMaSAqeE8VEOK8jKyIm2m3P4tmlnCjY8hp6Ni7d1HBe7I2Z55LW8RIaYdOGB1BfiXA4q+hcV5GBe/JIppB0ySjU4jU0PLgflpaIo2OIb/QJSQJP6uuL+ZNVPCeLunsl7wUMUr/A0eNjiA/CrD40TO0g9ExpIGp4D1deKzzUrIibiY2P5d2RBkdQ4DuoakEWjXToLdRwZtB0iDAYnQKkRrS9uVi0e+moQIt/nQP0Qm53kgFbwZhTaCZ/gOL+4k5foRUe6TRMbxaz9AOBFj9jY4hBlDBm4W24sVN9d2zDx/9qTFEsDWQbtp691r6X2cWoY0gTifRiPsJP1lI5ypNrWyEXqEddea8F1PBm0nSQLBoK17cT+/du/GzqGgaUqg1mM4h7Y2OIQZSwZtJSAw072R0CpEagsuK6X5Kc9Q3pN5hnfC1aDprb6aCN5ukwaBdcuKGembuJNCik70aQpRPOB2DE42OIQZTwZtNUAS0SzM6hUgNARXlXFIaaHQMrzA44hJ8LPrz7u30G2BGbfpBUKTRKURq6JqZQYhFJV+f2gS0oHVgc6NjiBtQwZuRjy+kDjM6hUgNfrYq+p7UceH64oOVwRGXGB1D3IQK3qxik6FRG6NTiNTQcfcOIq3BRscwpR6hqUT6hhkdQ9yECt7MOo7QCXfidqwOO/2O24yOYToRPqH0DutsdAxxIyp4MwuJ0Ql34pba79lFY6u2NOvSkIhebj3XwODBg7n77rsNzTBt2jS6du1a5+uZMGECo0aNuuj1nk1CQgIzZ86s1WNU8GbXNs1Z9CJuxAKk5ZcZHcM0EgNb0iawhdEx3N6UKVP4+uuvXV/XVTHPmjWL9PT0i15PXVPBm53VBzpdZXQKkRra5GQRZ9EUthfL3+LH4IheRsfwCKGhocTE1P0GT0REBJGRkXW+3oulgvcGMa0gvqvRKURq6J9XaHQEjzc4oidhPhd30uLgwYO56667eOCBB4iOjiY2NpZp06a57s/Ozubaa68lNDSU8PBwxowZw+HDh133/7TL+p133iEhIYGIiAhuvvlmTp48We157Hb7GZ/jfJ5n8+bNDBkyhLCwMMLDw+nRowfr1q0DID09ncjISD777DMSExMJDAxk+PDh5OTk1Mj507/nzJnD/PnzsVgsWCwWli1bBsCDDz5IUlISwcHBtGnThscff5zKysozfv9+uSdg3759rvX98mPw4MGu5VesWMGAAQMICgoiPj6eu+66i5KSEtf9+fn5XH311QQFBdG6dWvee++9Mz732ajgvUXq5RAcZXQKkWpa5GWTYIk0OobHahfYss5mrJszZw4hISGsXr2aF154gaeeeorFixdjt9u59tprKSgoYPny5SxevJg9e/Zw0003VXt8VlYWn332GV988QVffPEFy5cv57nnnjuv5wDO63nGjh1LixYtWLt2LevXr+ehhx7Cz8/PdX9paSnPPPMMb7/9NitXrqSwsJCbb775tK93ypQpjBkzhhEjRpCXl0deXh79+vUDICwsjPT0dLZv386sWbP4+9//zssvv3xe38f4+HjX+vLy8ti4cSMxMTEMHDjQ9X0aMWIE119/PVu2bGHu3LmsWLGCyZMnu9YxYcIEcnJyWLp0KR9//DF//etfyc/PP6/n/yX3PSND6pZvAHS9Dr5LB4fd6DQiLv2z89kXrylsayvEGsSwiD51tr7OnTszdepUABITE3nttddcx6u3bt3K3r17iY+PB+Dtt9+mQ4cOrF27lksucY67t9vtpKenExbmPHnyd7/7HV9//TXPPPPMOZ9j2LBhfP311+d8nuzsbO6//36Sk5Nd6/ilyspKXnvtNXr37g0431CkpKSwZs0aevWqfhgjNDSUoKAgTp06RWxsbLX7HnvsMde/ExISmDJlCh9++CEPPPDAOb+PPj4+rvWVl5czatQo+vbt69pbMWPGDMaOHes64TAxMZFXXnmFQYMG8frrr5Odnc2CBQtYs2aN63v71ltvkZJS+8v+agvem0Q1h8SBRqcQqabJ0TyS0N6l2ro8sh9BPnU3K2DnztWH2DVr1oz8/HwyMjKIj493lS5AamoqkZGRZGRkuG5LSEhwlfsvH38+zwGc1/Pce++9TJw4kaFDh/Lcc8+RlZVVbX2+vr6uUgRITk6ukfN8zJ07l7S0NGJjYwkNDeWxxx4jOzu7VusA+MMf/sDJkyd5//33sVqddbt582bS09MJDQ11fQwfPhy73c7evXvJyMjA19eXHj161HgdtaWC9zbt+kN0S6NTiFSTtjcHK7rU8fnqEty+zqej/eWubgCLxYLdfv57+87n8Rf7HNOmTeOHH37gqquu4ptvviE1NZV58+ad9+PPx3fffcfYsWO58sor+eKLL9i4cSOPPvooFRUVtVrP9OnTWbhwIZ9//nm1Nz7FxcVMmjSJTZs2uT42b95MZmYmbdu2rdPXooL3NhYLdB0FvpoPXNxHVOExUu2RRsfwCNG+EQyM6HHuBetISkoKOTk51U5W2759O4WFhaSmpjb48yQlJXHPPfewaNEiRo8ezezZs133VVVVuU66A9i5cyeFhYVn3L3t7++PzVZ90qVVq1bRqlUrHn30UXr27EliYiL79++v1Wv55JNPeOqpp/jXv/5Vo7S7d+/O9u3badeuXY0Pf39/kpOTqaqqYv369TVeR22p4L1RUAR0utLoFCLV9M3aiw+ap/5srFi5IrJ/g05oM3ToUDp16sTYsWPZsGEDa9asYdy4cQwaNIiePXs22POUlZUxefJkli1bxv79+1m5ciVr166tVt5+fn7ceeedrF69mvXr1zNhwgT69OlT4/j7TxISEtiyZQs7d+7k6NGjVFZWkpiYSHZ2Nh9++CFZWVm88sortdpLsG3bNsaNG8eDDz5Ihw4dOHToEIcOHaKgoABwnqG/atUqJk+ezKZNm8jMzGT+/Pmuk+zat2/PiBEjmDRpkut1TJw4kaCgoFp/T1Xw3iquA7ToYnQKEZew4iK6VoUbHcOt9Q3rTFP/hp24ymKxMH/+fKKiohg4cCBDhw6lTZs2zJ07t0Gfx8fHh2PHjjFu3DiSkpIYM2YMV1xxBU8++aRrHcHBwTz44IP85je/IS0tjdDQ0LPmvO2222jfvj09e/akcePGrFy5kmuuuYZ77rmHyZMn07VrV1atWsXjjz9+3q9j3bp1lJaWMn36dJo1a+b6GD16NOA8D2H58uXs2rWLAQMG0K1bN5544gni4uJc65g9ezZxcXEMGjSI0aNHc/vtt9OkSZPafkuxOBwOR60fJeZQVQHf/h1KC4xOIgJAWVAIb6XEUuGoMjqK22kV0Izroi/Dquu8n1Z6ejp33333Be3KNiv9pngzX3/odp1ztjsRNxBUVkKP8hCjY7idCJ8wrooaqHKXWtFvi7eLjIOOOh4v7qPH7p0EWTQu/id+Fl+ujR5CoDXA6CjiYbSLXpy2L4a93xudQgSA9ckdWR508twLeoGrowaTGKShrVJ72oIXp5Sh0KRuprwUuVhdMjMIs9T+rGGz6R3aWeUuF0wFL04Wi/N4fFjtz9QUqWu+Nht9Tnj3xDdtA1vQL0wjXeTCqeDlZ74B0PMm8NdJTmK8Dlk7iLJ65+9itG8EV0QOwGLx7jc5cnFU8FJdcCT0uFFn1ovhrA4HacfOfIlOswqw+HFt9BD8rX7nXljkLFTwUlN0PHQaaXQKERL3ZdLEGnbuBU3C1+LDtdGXEuWrCX/k4qng5fRadIa2/YxOIV7OAvQ/XGJ0jAZhxcLIqEG0CGhqdBQxCRW8nFn7SyGuo9EpxMslHNhLC0uE0THq3eWRabQJbGF0DDERFbycmcUCXa+F2NNfiUmkofTPNfd0yoPDLyE1uI3RMcRkVPBydharc/hc0ySjk4gXizt8gDZEGR2jXvQO7Uz3UL2Jlrqngpdzs/pA9xugcTujk4gXS8vOw4K5ho11CW5PWnhXo2OISang5fxYfZzD5xq1NjqJeKnGxw7T3hFpdIw60z4ogUsjTn+dcpG6oIKX8+fj65wIJ7qV0UnES/Xbm43VBH+2EgNbMiKyvyaykXrl+f9TpGH5+MElN0OUzvaVhhdZVEBHm2efUd8hqC1XRQ3ER5d+lXqm3zCpPV9/uOQ3zkvNijSwPll78MUzZ1rsGpLM5ZH9dF13aRD6LZML4xcAvcZCVLzRScTLhJacoGul581u1zu0E5dG9NJueWkwKni5cH6B0HusLjMrDe6S3ZkEWDxnrvYB4d1JC+9mdAzxMip4uTg+ftBjDLTQZS2l4QSVl9KzPNjoGOdkwcJlEb25JFQzQkrDU8HLxbNaocs1mrteGlT3XTsItgQYHeOMrFgYEZlGl5D2RkcRL6WCl7qTfBl0uMI5xa1IPfOrqqB3iXvupnde8vVSUjT9rBjI4nA4HEaHEJPJ3w0bP4GqCqOTiMnZfHyY3TWJE/ZSo6O4RPmEc23MEKJ9PXs4n3g+bcFL3WvSDvqOh0Bd01rql4/NRt9C99lGaRXQjFsaX6lyF7egLXipP+UnYd1cKMozOomYmAMLb/dI5Zi92NAc3UNSGBjeQ2PcxW3oN1HqT2AY9J0ALXsYnURMzIKDfkdPGfb8Pli5PLIvgyMuUbmLW9EWvDSMg9tgy5dg03F5qR/v9+jEIfuJBn3OYGsgV0cNpnlAkwZ9XpHzoYKXhlN8FDZ8AifzjU4iJpQdl8DHTRvuz1msXyNGRg0k3De0wZ5TpDZU8NKwbJWw7Ss4sMnoJGJCH3fvQrajsF6fw4KFXqEd6RvWRbvkxa2p4MUYBzbDtgXOwhepI3lNmvNBc996W3+YTzBXRA6gRUDTensOkbqighfjnMx37rIvPmp0EjGRz7t1ZTfH63y9SYGtGBrZh0Cr+86eJ/JLKngxVlUFbF8EORuNTiImcSy6MW+3CsFB3fxp87P4MiSiFx2D29XJ+kQaigpe3MPRfbD1Cyit+y0v8T5fde3KdsvF/y419YvhyqgBRPlq0ibxPCp4cR+2Sti1HPZ+D/q1lItQFB5FettIbNgv6PE+WOkV1oleoZ3w0Yl04qFU8OJ+ivJgy7/hxGGjk4gH+6ZzVzb51H4rvqV/My6L7K2tdvF4KnhxT3Y77PkOMv8L9iqj04gHKgkO5Z/JTal0nN/vT7A1kEHhPXUFODENFby4t+JjzmPzBdlGJxEPtKJjF9b4FZ51GQsWOgcn0j+8OwFW/4YJJtIAVPDi/hwO51n2u5bBqRKj04gHKfcP4q2OzTnlOP18C038orksog/N/Bs1cDKR+qeCF89RVeHcbb/ne81pL+dtTUonVgRWn6M+wOJP37AudA1pr9noxLRU8OJ5TpU4j81nbwDHhZ0lLd6j0seXf3ZpS4mjHB986BaaTK/QTgRqd7yYnApePFdJAez8BvIyjE4ibm5LYgfyGjelX3hXwnxCjI4j0iBU8OL5Cg9CxhIo2G90EnFHzVIgcRCENTY6iUiDUsGLeeRnws6lGj8vTk3bQ9IgCNeFYcQ7qeDFfPJ3Q9YqbdF7I4sFmiRB4gCIaGZ0GhFDqeDFvApznUV/aCfU0YVHxE35BkJ8V0i4BIIjjU4j4hZU8GJ+JQWwby0c2OQcaifmERIDCb2gRWfw1VnxIr+kghfvUXnKWfL71uqqdZ6ucVtnsTdu69wtLyI1qODF+zgczuP0BzY7T8zTXPeewccPmneG1r0gVDPPiZyLCl68W2W5cxx97ladlOeOLFbnVnpcB+dZ8doNL3LeVPAiPykrgtxtzrIvPmJ0Gi9mgZhWzlKPTQH/IKMDiXgkFbzI6RQdchb9wR/g1Emj03iHyDiI6wjNUiEwzOg0Ih5PBS9yNg4HHM+BI1lwZA8U5aEhd3XEYoGIOGiS6NxaD4k2OpGIqajgRWqjohSO7v258LV1XzshMdCoDTRq7dwN7xdodCIR01LBi1yMk/k/l31Bts7I/7WAMGiU4Cz0Rq0hMNzoRCJeQwUvUldslc5j90V5cOLHz8VHveiSthYIjYHwWIhs7ix0XeBFxDAqeJH6ZKuCk4edZf9T+RcfAbvN6GQXx8fPORY9rImz0CNinZ81jE3EbajgRRqa3ebctV98zDmjXmmh83NZIZSdwG1O4vMLcp7NHhju/BwU4dwiD2sCwVGaQU7EzangRdyJ3e48ca/8JJSf+PHzSagscx4CqKpwfrZVQFXlz/+2/fjvX7P6go8vWP1+/Ozz421+zs++/s4CD/pFkf/02cev4V+/iNQZFbyIWTgczpJ32H8scB+jE4mIgVTwIiIiJmQ1OoCIiIjUPRW8iIiICangRURETEgFLyIiYkIqeBERERNSwYuIiJiQCl5ERMSEVPAiIiImpIIXERExIRW8iIiICangRURETEgFLyIiYkIqeBERERNSwYuIiJiQCl5ERMSEVPAiIiImpIIXERExIRW8iIiICangRURETEgFLyIiYkIqeBERERNSwYuIiJiQCl5ERMSEVPAiIiImpIIXERExIRW8iIiICangRURETEgFLyIiYkIqeBERERNSwYuIiJiQCl5ERMSEVPAiIiImpIIXERExIRW8iIiICangRURETEgFLyIiYkIqeBERERNSwYuIiJiQCl5ERMSEVPAiIiImpIIXERExIRW8iIiICangRURETEgFLyIiYkIqeBERERP6f+Eyh6i4Lz/OAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "After Balancing,\n",
      "outcome_group\n",
      "0     9970\n",
      "1    10571\n",
      "2     9814\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAc0AAAGbCAYAAACmksv3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNiklEQVR4nO3dd3yV5f3/8ddZmSebkAQIJIRAmLKnDBUFxYGoKOIX+Slqi2DRgtQ6sVht/dqKRf3WUXBgsRUVxSoigspeMmTvMMIKmWSe8fsj9ZSUdQJJ7jPez8cjD8gZ9/nch5D3+Vz3dV+3ye12uxEREZELMhtdgIiIiL9QaIqIiHhJoSkiIuIlhaaIiIiXFJoiIiJeUmiKiIh4SaEpIiLiJYWmiIiIlxSaIiIiXlJoilyE9957j6ysLGw2G7GxsUaXU6tGjx5NWlqa0WWwePFiTCYTixcvNroUEQ+Fpsh/ee211zCZTPTo0eOs92/bto3Ro0eTkZHBm2++yRtvvEFJSQnPPPNMvf6C37dvHyaTqdpXdHQ0HTt2ZPr06TidznqrRSRYWI0uQMTXzJo1i7S0NFatWsWuXbto0aJFtfsXL16My+Vi2rRpnvtOnDjBlClTABgwYEC91jtixAiuu+46AAoKCvjXv/7F+PHj2b9/Py+++GK91lKb+vXrR2lpKSEhIUaXIuKhTlPkNHv37mXZsmX86U9/IjExkVmzZp3xmGPHjgHUy7DsqVOnLviYzp07c9ddd3HXXXfx4IMPMm/ePLp168YHH3xQ5/XVJbPZTFhYGGazfk2J79BPo8hpZs2aRVxcHEOGDOHWW289IzTT0tJ4+umnAUhMTMRkMjF69GgSExMBmDJlimeo9JlnnvE8b9u2bdx6663Ex8cTFhZG165d+eyzz6pte+bMmZhMJr777jvGjh1Lw4YNadKkSY33wWQykZSUhNVafSBp7ty5DBkyhEaNGhEaGkpGRga/+93vvBrG/d///V969+5NQkIC4eHhdOnShY8++uisrz1u3Dg+/fRT2rVrR2hoKG3btuWrr74647GHDh3i3nvv9dSTnp7OL3/5SyoqKoCzH9McMGAA7dq1Y8uWLVxxxRVERETQuHFj/vjHP56x/f3793PjjTcSGRlJw4YNefjhh5k/f76Ok8ol0fCsyGlmzZrFsGHDCAkJYcSIEbz++uusXr2abt26AfDyyy/z7rvv8sknn/D6669jt9tp3749PXv25Je//CU333wzw4YNA6BDhw4AbN68mT59+tC4cWN+85vfEBkZyT/+8Q+GDh3KnDlzuPnmm6vVMHbsWBITE3nqqae86jRLSko4ceIEAIWFhXz55Zd89dVXPPbYY9UeN3PmTOx2O4888gh2u51vv/2Wp556isLCwgsO406bNo0bb7yRkSNHUlFRwezZs7ntttuYN28eQ4YMqfbYJUuW8PHHHzN27FiioqJ45ZVXuOWWW8jOziYhIQGAw4cP0717d/Lz87n//vvJysri0KFDfPTRR5SUlJx3SDYvL4/BgwczbNgwhg8fzkcffcTkyZNp37491157LVDVoV955ZXk5OTwq1/9iuTkZD744AMWLVp0wfdT5LzcIuJ2u93uNWvWuAH3ggUL3G632+1yudxNmjRx/+pXv6r2uKefftoNuI8fP+657fjx427A/fTTT5+x3auuusrdvn17d1lZmec2l8vl7t27tzszM9Nz24wZM9yA+/LLL3c7HI4L1rt37143cNavX/7yl26Xy1Xt8SUlJWds44EHHnBHRERUq+3uu+92N2vW7LzPraiocLdr18595ZVXVrsdcIeEhLh37drluW3Dhg1uwP2Xv/zFc9uoUaPcZrPZvXr16jNq+rnuRYsWuQH3okWLPPf179/fDbjfffddz23l5eXu5ORk9y233OK57aWXXnID7k8//dRzW2lpqTsrK+uMbYrUhIZnRf5t1qxZJCUlccUVVwBVQ4233347s2fPvuiZqCdPnuTbb79l+PDhFBUVceLECU6cOEFubi6DBg1i586dHDp0qNpz7rvvPiwWi9evcf/997NgwQIWLFjAnDlzePDBB/nrX//KI488Uu1x4eHhnr//XEvfvn0pKSlh27Zt532N05+bl5dHQUEBffv2Zd26dWc8duDAgWRkZHi+79ChA9HR0ezZswcAl8vFp59+yg033EDXrl3PeL7JZDpvLXa7nbvuusvzfUhICN27d/dsH+Crr76icePG3HjjjZ7bwsLCuO+++867bZEL0fCsCOB0Opk9ezZXXHEFe/fu9dzeo0cPXnrpJRYuXMg111xT4+3u2rULt9vNk08+yZNPPnnWxxw7dozGjRt7vk9PT6/Ra2RmZjJw4EDP98OGDcNkMvHyyy9zzz330L59e6BqmPiJJ57g22+/pbCwsNo2CgoKzvsa8+bNY+rUqaxfv57y8nLP7WcLuKZNm55xW1xcHHl5eQAcP36cwsJC2rVr5/1OnqZJkyZnvG5cXBwbN270fL9//34yMjLOeNx/z4QWqSmFpgjw7bffkpOTw+zZs5k9e/YZ98+aNeuiQtPlcgEwceJEBg0adNbH/Pcv8tO7uot11VVXMX36dL7//nvat29Pfn4+/fv3Jzo6mmeffZaMjAzCwsJYt24dkydP9tR5Nj/88AM33ngj/fr147XXXiMlJQWbzcaMGTPOOkP3XF2y2+2+5P2qj+2LnI9CU4SqUGzYsCGvvvrqGfd9/PHHfPLJJ/zf//3fOQPtXEOKzZs3B8Bms1XrBuuaw+EAoLi4GKiaiZqbm8vHH39Mv379PI87vas+lzlz5hAWFsb8+fMJDQ313D5jxoyLqi0xMZHo6Gh++umni3q+N5o1a8aWLVtwu93V/m127dpVZ68pwUHHNCXolZaW8vHHH3P99ddz6623nvE1btw4ioqKzjhF5HQREREA5OfnV7u9YcOGDBgwgL/+9a/k5OSc8bzjx4/X6r787PPPPwfgsssuA/7TnZ3ejVVUVPDaa69dcFsWiwWTyVTtuO6+ffv49NNPL6o2s9nM0KFD+fzzz1mzZs0Z99dGxzho0CAOHTpU7d+srKyMN99885K3LcFNnaYEvc8++4yioqJqk0ZO17NnT89CB7fffvtZHxMeHk6bNm348MMPadmyJfHx8bRr14527drx6quvcvnll9O+fXvuu+8+mjdvztGjR1m+fDkHDx5kw4YNl1T/unXreP/994GqCT4LFy5kzpw59O7d2zOk3Lt3b+Li4rj77rt56KGHMJlMvPfee14F1JAhQ/jTn/7E4MGDufPOOzl27BivvvoqLVq0qHYcsSZ+//vf8/XXX9O/f3/uv/9+WrduTU5ODv/85z9ZsmTJJS8c8cADDzB9+nRGjBjBr371K1JSUpg1axZhYWHAhScbiZyLQlOC3s+/TK+++uqz3m82mxkyZAizZs0iNzf3nNt56623GD9+PA8//DAVFRU8/fTTtGvXjjZt2rBmzRqmTJnCzJkzyc3NpWHDhnTq1Imnnnrqkuv/+9//zt///ncArFYrTZs2ZdKkSTz11FOe1XQSEhKYN28ev/71r3niiSeIi4vjrrvu4qqrrjrnsdafXXnllbz99tu88MILTJgwgfT0dP7whz+wb9++iw7Nxo0bs3LlSp588klmzZpFYWEhjRs35tprr/V07Zfi5/NQx48fz7Rp07Db7YwaNYrevXtzyy23eMJTpKZMbh09F5Eg8fLLL/Pwww9z8ODBajOWRbyl0BSRgFRaWlpt4lZZWRmdOnXC6XSyY8cOAysTf6bhWREJSMOGDaNp06Z07NiRgoIC3n//fbZt23bWRfhFvKXQFJGANGjQIN566y1mzZqF0+mkTZs2zJ49+5yTuUS8oeFZERERL+k8TRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETESwpNERERLyk0RUREvKTQFBER8ZJCU0RExEsKTRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETESwpNERERLyk0RUREvKTQFBER8ZJCU0RExEsKTRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETESwpNERERLyk0RUREvKTQFBER8ZJCU0RExEsKTRERES8pNEVERLyk0BQREfGS1egCRKR2VDjclFS6Ka10U+l043SBw+nG4QKH699/nva92w0mE5gwYTKB+d9fFrMJqxksZrCaTYRaTYTZTITbqv4MtYLZZDJ6d0UModAU8QMul5uCMjdF5S5KKtz/+ar8z/cOV/3UYgJCrRD27xANs5qIDDUTE2YiOsxMdJiZMJtCVQKTye12u40uQkSquN1uisrd5Je6yC9xkVfqIr/URVGZG5cf/U8NtUJ0mJmYMDPR4SZiwszEhJuJCjVhUpcqfkyhKWKgwjIXx4qcHCt2kXfKRUGZq946RiOEWCDRbqGB3ez5M8QSHCE6YMAAOnbsyMsvv2x0KXXGZDLxySefMHToUKNLqTManhWpJ263m7xSF8eKXBwtcnKsyEVpZXB9Zq1wwqECJ4cKnEAlJiA63ESi3UJipJnEKAux4ZqfKL5LoSlSh/JLXBwscHC00MXxYicVTqMr8i1uoKDUTUGpg13Hq24Lt5loHGuhcYyFlBhL0HSi4h/0kU6kFrncbo4WOlmTXc4nG0r47KdS1h2o5FCBAtNbpZVudh138N2ucv6xroT5W0v5KaeC/BL/Gbc+deoUo0aNwm63k5KSwksvvVTt/vLyciZOnEjjxo2JjIykR48eLF68uNpjli5dyoABA4iIiCAuLo5BgwaRl5cHgMvl4vnnnyc9PZ3w8HAuu+wyPvroI89znU4n9957r+f+Vq1aMW3atGrbX7x4Md27dycyMpLY2Fj69OnD/v37PffPnTuXzp07ExYWRvPmzZkyZQoOh8Nz/86dO+nXrx9hYWG0adOGBQsW1Nbb59PUaYpcIofTzeECJwfynRzMd1DuuPBzxDsuNxwtcnG0yMW6A5VEhlR1oc3irSRHmX12UtGkSZP47rvvmDt3Lg0bNuS3v/0t69ato2PHjgCMGzeOLVu2MHv2bBo1asQnn3zC4MGD2bRpE5mZmaxfv56rrrqKe+65h2nTpmG1Wlm0aBFOZ9Unr+eff57333+f//u//yMzM5Pvv/+eu+66i8TERPr374/L5aJJkyb885//JCEhgWXLlnH//feTkpLC8OHDcTgcDB06lPvuu4+///3vVFRUsGrVKs/7+cMPPzBq1CheeeUV+vbty+7du7n//vsBePrpp3G5XAwbNoykpCRWrlxJQUEBEyZMMOKtrneaCCRyEZwuNwfynOzJdZBT4MSp/0X1LiLERHq8leYNrMRF+M6gWXFxMQkJCbz//vvcdtttAJw8eZImTZpw//3388gjj9C8eXOys7Np1KiR53kDBw6ke/fu/P73v+fOO+8kOzubJUuWnLH98vJy4uPj+eabb+jVq5fn9jFjxlBSUsIHH3xw1rrGjRvHkSNH+Oijjzh58iQJCQksXryY/v37n/HYgQMHctVVV/HYY495bnv//fd59NFHOXz4MF9//TVDhgxh//79nn346quvuPbaazURSET+I/eUk13HHezNdWi41WAlFW42H6lk85FKYsNNNG9gpXmClYgQYwN09+7dVFRU0KNHD89t8fHxtGrVCoBNmzbhdDpp2bJlteeVl5eTkJAAwPr16z2B+9927dpFSUkJV199dbXbKyoq6NSpk+f7V199lb/97W9kZ2dTWlpKRUWFp9ONj49n9OjRDBo0iKuvvpqBAwcyfPhwUlJSANiwYQNLly7lueee82zP6XRSVlZGSUkJW7duJTU1tVronx7ggUyhKXIBZZVu9uQ62H3cQV6p/xxXCyb5pW7WHajkxwOVJEWZad7ASlq8FasPTiIqLi7GYrGwdu1aLBZLtfvsdjsA4eHh530+wBdffEHjxo2r3RcaGgrA7NmzmThxIi+99BK9evUiKiqKF198kZUrV3oeO2PGDB566CG++uorPvzwQ5544gkWLFhAz549KS4uZsqUKQwbNuyM1w8LC7u4HQ8QCk2Rs3C73RwqqOoqD+Y7/WphgWDmBo4UuThSVMGa7AoyE220SrJiD62/7jMjIwObzcbKlStp2rQpAHl5eezYsYP+/fvTqVMnnE4nx44do2/fvmfdRocOHVi4cCFTpkw54742bdoQGhpKdnb2WYdWoWoSUe/evRk7dqzntt27d5/xuE6dOtGpUycee+wxevXqxQcffEDPnj3p3Lkz27dvp0WLFmfdfuvWrTlw4AA5OTme7nTFihXnf2MChEJT5DROl5vdJxxsOVJJYZmS0p9VOGHzkUq2HKkkNc5C62QbSVGWCz/xEtntdu69914mTZpEQkICDRs25PHHH8dsrgruli1bMnLkSEaNGsVLL71Ep06dOH78OAsXLqRDhw4MGTKExx57jPbt2zN27Fh+8YtfEBISwqJFi7jtttto0KABEydO5OGHH8blcnH55ZdTUFDA0qVLiY6O5u677yYzM5N3332X+fPnk56eznvvvcfq1atJT08HYO/evbzxxhvceOONNGrUiO3bt7Nz505GjRoFwFNPPcX1119P06ZNufXWWzGbzWzYsIGffvqJqVOnMnDgQFq2bMndd9/Niy++SGFhIY8//nidv7e+QKEpApQ73Gw/Vsm2ow7KgmzBgUDnBrLznGTnOYmPMNM6yUpaghWLue6Gbl988UWKi4u54YYbiIqK4te//jUFBQWe+2fMmMHUqVP59a9/zaFDh2jQoAE9e/bk+uuvB6qC9euvv+a3v/0t3bt3Jzw8nB49ejBixAgAfve735GYmMjzzz/Pnj17iI2NpXPnzvz2t78F4IEHHuDHH3/k9ttvx2QyMWLECMaOHcuXX34JQEREBNu2beOdd94hNzeXlJQUHnzwQR544AEABg0axLx583j22Wf5wx/+gM1mIysrizFjxgBgNpv55JNPuPfee+nevTtpaWm88sorDB48uM7eU1+h2bMS1IrLXWw5Usmu446AXr5OqguzmWjV0ErrJBshVt877im+S6EpQenkKSc/5VSy/6QT/QcIXiEWaJNio3WSDZsPThoS36PQlKBSUOpi/cEK9ufpfBH5j1ArtE2xkZVkw1qHw7bi/xSaEhSKy11sOFTJnhMOdZZyTuE2E+1SbLRsWLfHPMV/KTQlsFWWws4lnCosYI75WqOrET8REWKiQyMbLRpYMSs85TQKTQlMLifsWw27llQFJ7A8ZQQ7HSkGFyb+JCbMRNemITSO1YkGUkWhKYHn6A7Y8jWU5FW7uSI6ldmhZ1+aTOR8msRa6NY0hKgw31njVoyh0JTAUVYEm7+CI9vO+ZANKcPY4Eirv5okYJhNVZOF2jfSZKFgptAU/+d2w/41sH0ROMrP+1BnZCIfhN+F20cvKSW+zx5qonuzEJpoyDYoKTTFvxUehU1fQP4hr5+yI+U6Vjiy6rAoCQZN4yx0bxZi+FVVpH4pNMU/OSth5/ewZwW4a7aUjyssmtlRo3FoFUm5RCEW6N4slOYN9LMULBSa4n9O7IFN/zpjok9NHEi+gkXOThd+oIgXmsVZ6JEWSphNw/6BTqEp/sPlhG0LYe/KCz/2Aty2cObE3UOJO7QWChOpWhihV7qOdQY6hab4h1O5sO5jKDxSa5s8ltSLr1zBcbV5qT+ZiVa6Ng3RWrYBSqEpvu/gBvjpK3BW1Opm3RYb8xrcQ54rsla3K2IPNdGneWi9XL9T6pdCU3yXo7zq2OXhn+rsJfITO/IZV9bZ9iV4mYAOjW10aGTDpFOcAoZCU3xT/mH48eNLmuzjDbfJzDdJd5PjjKvT15Hg1TjGQt+MUF23M0AoNMX37FkO276t8akkF+tUfBZzLNfVy2tJcIoKNTEgM4y4CJ3T6e/0Lyi+w+mAHz+Brd/UW2ACRJ7cRob1aL29ngSfonI3X24pZW+uw+hS5BIpNMU3lBfDinfr9Pjl+XQtWWLI60rwcLjgh93lrN5fjksDfH5LoSnGKzwKS/9Wo6XwaltowX7aW7MNe30JHluPOliwrYzSSgWnP1JoirGO7oBlM6G0wOhKaF/0Q9Xi7yJ17GiRiy82l5JfWn+HIaR2KDTFOHuWw5p/1Pr5lxfLWnyU7rYdRpchQaKkws1XW0o5VuQ0uhSpAYWm1D+XEzZ+XjXhB9/q7FrmL8WKfolJ/ahwwoJtZWTnaYKQv1BoSv1yVMDqv8OB9UZXclbm0nz6WI2ZjCTByemG73aWs+NYpdGliBcUmlJ/Ksth1QdwYq/RlZxX0xPLCcc3howlOLiBFfsq2HBIP3e+TqEp9aOyFFa+D3kHjK7kgkyVJfQ1rzO6DAlCGw5VsmKvTknxZQpNqXvlp2DFe1Bw2OhKvJZ0Yg0xphKjy5AgtOO4g+93leNyKTh9kUJT6lZZUdWiBYX+teKOyVlBP/cqo8uQIJWd5+T73eo4fZFCU+pOaQEsfxeKTxhdyUWJPbGeZIvx549KcMrOc/KDgtPnKDSlbpw6CcvfgZKTRldy0UxuF70rlxldhgSx/SedLN1Tjq6r4TsUmlL7SvKrhmR9YJWfSxWZu5U0y3Gjy5AgtjfXybK9FQpOH6HQlNpVUVJ1WklZkdGV1AoT0L3sB6PLkCC3+4SDFfsUnL5AoSm15+eFC07lGl1JrQrL30cbq++fKiOBbedxB6v26zxOoyk0pXa4nLDuI8j3n9NKauKy4iVazF0Mt/2Yg7UHFJxGUmjKpXO7q9aSPb7b6ErqjK0oh662XUaXIcLmnEotuWcghaZcuq3fwKFNRldR51rlL8GMLuUkxlu5v4LDBbqwgBEUmnJpdi+HvSuMrqJeWErz6GPdbHQZIrjd8N2uMvJL9CGuvik05eId2gTbvjG6inqVlruMUJOGxsR4lU5YuKOM0goFZ31SaMrFyT9cdRwzyJgqTtHXvN7oMkQAOFXh5tud5TicmqRWXxSaUnPlp2DtP6tmzAahlBOriDKXGl2GCAC5p1ws0apB9UahKTXjdsGPH0NZodGVGMbkKKefe7XRZYh4ZOc5WXdQhw3qg0JTambbt5C7z+gqDBd//EcamoP3g4P4ns05lRzIcxhdRsBTaIr3crbAnuVGV+ETTG4nfZx6L8S3LN1TTnG5JgbVJYWmeKfoGGwIvok/52M/sYWmlsBaMlD8W4UTXcC6jik05cIqy6om/ji1fNfpTLjpUb7E6DJEqjlxyqWl9uqQQlMubMPcqutjyhnC83aTZT1kdBki1Ww96iD7pI5v1gWFppxf9jo4usPoKnxap2JdOkx8z7K95RTp+GatU2jKuZXkwZYFRlfh82xFh+lsDdzF6sU//Xx806njm7VKoSln53bDhs90HNNLrQuWYHbrU734ltxTLtYf0vmbtUmhKWe3dyWczDa6Cr9hKcmlp22r0WWInGFLTiUnioNz9a66oNCUMxUdh+2LjK7C7zQ/uZQQLeYuPsZN1fFNDdPWDoWmVOdyVc2WdWnmXU2Zy4u53LzR6DJEzpBf6mbTYX2gqw1WowsQH7N7CRTkGF2F32p8YiWR8W055Q4zuhS/sHDOm3z78VucOFx1KKBx89bcdO9vuKz3NQDMeH48m1cvJv9EDmHhkbRo35Ph456lUVorAIoLTvLmsw+wde33JKVmMOaJ12nW6jLP9t/948MkNk7n2pEP1f/O+ZifcippGm8hPsJidCl+TZ2m/EdBDuzU6ROXwuQoox9rjC7Db8Q3bMzwsc8y5Z0fmPLO97Tp2o9pk27n4J4tAKRldWLMk6/z/Oy1TJw2FzduXnzoJlzOqmN0n898kbKSIp59dwlZnfvyt9+P82x716ZV7N68hkF3PGjIvvkalxuW7anApauhXBKFplRxu2DjvKo/5ZI0OLGOBHOR0WX4hU59r+OyPoNIbtqC5KaZ3PrLZwiLsLP7p6qryFxx8z1kdbqcxEbNSMvqyC0PPMXJowc5nrMfgMN7t9Pj6ltJbprJFUP/H4f3bQfA4ajknT/8itG/mYbZos7qZydLXGzO0TDtpVBoSpX9a6HwiNFVBASTy8HlzhVGl+F3XE4nK77+J+Wlp2jRrvsZ95eXnuKHee+R2CiNhKQmAKRmtmfLmu9wOhxsWrGQ1BbtAPjXe38mq3Nf0lt3rtd98AcbDlWSX6oPxxfL5NaVS6X8FCx+FRzlRlcSMNwmE4uS7+agI97oUnzegV0/8bsxV1FZUUZYuJ1fPPs3LuszyHP/wo/e4MPpT1JeeoqUZpk8/Kc5JDVpDkBJcQHv/GECOzeuoEFKU+5+9GUsVht/euRWnnp7IR+9PoWfVn5LWutO3PPb6UTYY4zaTZ/SMMrM4NbhRpfhlxSaAuvnwiHN+qxtJXGZfGS9wegyfJ6jsoLcIwcoKS5k9bef8v1nM3ns9a9o3Lw1UBWMhSePk597hC9nvULescM88eY3hISefbLVC2Ov45rbx3LiSDbrl3zFI3+ew4zfj8MeE8+IXz1fn7vm0/pmhJKeoLmgNaXh2WB38oACs45E5O0k06qZyBditYWQlJpBeutODH9wCqmZ7fn6w9c890fYY0hu2oKsTpcz/vn3ydm/g7WLPzvrtr7//D0iomLo3P96tq37gS79r8dqtdHtypvZulaT3E637kAFDp27WWMKzWDmdsOW+UZXEdC6nNIv6ppyu1w4Ks++fKPb7Qa3+6z3F+YdZ+7bL3DXr18CwOV04XRUTXpxOitxu7QqzulOVbj5Sedu1ph682B2aKPOyaxjIYUHuSxlLxsc6UaX4pP+8erTdOh9NQlJqZSVFLF8/j/Ztu4HJk6by7FDe1m5YA7telxFdFwDTh47xLx3/4QtNNxzHufpPvjzZAaPHE98w0YAZF7Wk6VfzqZdj6tY/MkMMjv0rO/d83mbj1SSmWglMlT9k7cUmsHKUQHbtFRefWhXuISN4Wm4TSajS/E5RXnHeXPK/eSfOEK4PZrUFu2YOG0u7XpcSd7xHHasX8bXs1/lVFE+MfENadWpD0++9Q3R8Q2rbWfTim84enAP9z/zlue2gbc9wN6t65hyzxU0b9uFoWMeq+/d83lOF/x4sJLLM0KNLsVvaCJQsNq+GHZp6LC+7Ey5luWO1kaXIXIGE3Bd2zASInU+qzfUkwejipKqq5hIvck4uRQbWs9XfI8bWHtAlwD0lkIzGO1Zrutk1jNzeSGXWzRLWXzTkUIXh/L1oc4bCs1gU1EC+7Q2qhGa5K4kwqQFJMQ3bdRMWq8oNIPNnhXqMg1iqiyln2mt0WWInNXxYhc5hTot50IUmsGkogT2rza6iqCWeGItceZTRpchclabDukD9YUoNIPJnhVVp5qIYUzOSvq6NAlLfNORIhfHitRtno9CM1hUlKrL9BExJzbSyJJndBkiZ7VJxzbPS6EZLPYsV5fpI0xuF70qlhldhshZHSpwkntK3ea5KDSDQaW6TF8TeXI7GdajRpchclbqNs9NoRkMsn9Ul+mDupZoRSbxTdl5TvJLdKHqs1FoBjq3G7LXGV2FnEVoQTbtrfuNLkPkrLYcUbd5NgrNQHd8F5Ro0omval/0Q9UHGxEfs++kgwqHfjb/m0Iz0Gn1H59mLT5Gd9t2o8sQOYPDBbtPaGm9/6bQDGQleXB8t9FVyAW0zFuKFc1WFN+z47iGaP+bQjOQ7V9L1TUMxJeZywroY9lkdBkiZygodXNUS+tVo9AMVE4HHFhvdBXipaa5KwhHM5zF92w/pm7zdArNQJWzuer8TPELpsoS+po1y1l8T3aek7JKjVj9TKEZqPZrApC/STqxmhhzidFliFTjcsMuHdv0UGgGoqJjkH/Y6CqkhkzOSvq5VhldhsgZdhx34NapUYBCMzAd3mJ0BXKRYk+sJ9lSYHQZItUUl7s5XqwVgkChGZhyFJr+yuR20btSi7mL79l3UudsgkIz8BQegVO5RlchlyAydytpluNGlyFSTfZJp4ZoUWgGHg3N+j0T0KNMi7mLbympdHNMQ7QKzYCjodmAEJq/j7bWA0aXIVLNfg3RKjQDSkGOFmcPIJdpMXfxMfs1RKvQDCiHNxtdgdQia/ERutl2GV2GiEephmgVmgElZ6vRFUgta5m/BIsWcxcfEuxDtArNQJF/GErzja5CapmlNI/eVh2nFt8R7EO0Cs1AcVzDeIEqLXcZoSYtYya+obTSzcmS4B2iVWgGihN7ja5A6oip4hT9zD8aXYaIR04QXy5MoRkInJWQf8joKqQOJR9fRZRZV60R35BToE5T/NnJbHAF7ye/YGByVtDfvdroMkQAOF7sxOkKzuOaCs1AoKHZoBB3/EcamguNLkMEh4ugXcBdoRkIFJpBweR20sex3OgyRIDgPa6p0PR3FSVVi7RLULDnbqGZ9YTRZYiQU6DQFH+Uu8/oCqQemXDTo2yJ0WWIkHvKRYUz+I5rKjT9nYZmg05Y3h6yrAeNLkOCnBs4GoRDtApNf5e73+gKxACditVtivGOFgXfZCCFpj9zlOuC00HKVnSYzlatAiXGOlmiTlP8SeFRoysQA7UuWIrZHXyf9MV3nDwVfD9/Ck1/VqBZs8HMUpJLL5sWcxfjVDihqDy4glOh6c90qknQSz+5jBAt5i4GCrZuU6HpzxSaQc9cXkxf8wajy5AglqvQFL/gckLRcaOrEB/Q6MRK7CYt5i7GCLbLhCk0/VXRMdAkEAFMjnL6sdboMiRInTwVXDNoFZr+SpOA5DQJJ9aSYC4yugwJQmUOOBVEk4EUmv5KxzPlNCaXk77OFUaXIUEqL4iGaBWa/qromNEViI+JOvETqRYtdiH1r6g8eNagVWj6q9ICoysQH2PCTc/ypUaXIUGoWMOz4tNcLijTxYjlTOF5u2hpPWx0GRJk1GmKbysrBHfw/JBKzXQ+pcXcpX6p0xTfpqFZOY+QwoN0tOqScVJ/itVpik8rzTe6AvFxbQuXaDF3qTcOF5RVBkdwKjT9kTpNuQDLqeP0sG03ugwJIsEyRKvQ9EcKTfFCxsml2HAYXYYEiWCZDKTQ9EcKTfGCubyQyy0bjS5DgoQ6TfFdCk3xUpPclUSayowuQ4KAjmmK7yrVOZriHVNlKX1N64wuQ4JARZCs267Q9DcuJ7h0nEq8l3h8DXHmU0aXIQGu3KFOU3yRo9zoCsTPmFwO+rq0mLvUrUqnQlN8kaPC6ArED8Wc2EgjS57RZUgAU6cpvkmdplwEk9tNr4plRpchAUzHNMU3qdOUixR5cjsZ1qNGlyEBqkKdpvgkdZpyCbqV/GB0CRKgHC5wBcGFJBSa/kahKZcgpCCbDtb9RpchAaoiCCb2KzT9jYZn5RK1K/wBUxB0BFL/gmEGrULT36jTlEtkPXWM7lrMXepAMHwWq1FoDhgwgAkTJtRRKRf2zDPP0LFjx1rfzujRoxk6dOglb/d80tLSePnlly99Q87KS99GPXv9X6voMO5Vom97jujbnqPXr9/gyzU7PPcP+M3fMF3/VLWvX0z/zHP/yaISbpjyPvZbp9Lpodf4cXdOte0/+Po8Xvp4ab3tTyDIzFuKlSCZ7ij1JhhWn7UaXUBNTJw4kfHjx3u+Hz16NPn5+Xz66aeXtN1p06bh9puPSP5S5380SYjmhbuvJrNRAm7cvLNwPTdN/Ts/TvslbZs1BOC+QV149q4rPc+JCLV5/v7ch99TVFrBumm/4PV/rea+v8xlzcu/AGDFtgOs3H6QV+6/rn53ys+Zywq4PHYTi50djS5FAojf/Bq9BH4Vmna7HbvdXuvbjYmJqfVt1h2T0QXU2A09sqp9/9yogbz+r9Ws2H7AE5oRoTaS46LO+vytB45zR7/2tGzcgPsHd+WNr9YAUOlw8otXP+eth27CYtGRhppKzV1BeFxrSt2hRpci4jdqHJoul4tHH32Ut956i5CQEH7xi1/wzDPPAJCdnc348eNZuHAhZrOZwYMH85e//IWkpCQANmzYwIQJE1izZg0mk4nMzEz++te/0rVrV2bOnMmECROYOXMmkyZN4sCBA/Tv35+33nqL1NRUoGpY9dNPP2X9+vU888wzvPPOOwCYTFVBsmjRIgYMGMDkyZP55JNPOHjwIMnJyYwcOZKnnnoKm8125g5RvWPdt28f6enpZzymf//+LF68GIAlS5bw2GOPsWbNGho0aMDNN9/M888/T2RkJADHjh3j3nvv5ZtvviE5OZmpU6fW9G0OWE6ni38u2cypsgp6ZaV6bp+1eCPvL95IcqydG7q34sk7+hMRFgLAZenJfLtxD2MGdWb+up10SKv6efrjnCUMaJ9G18zGhuyLvzNVlmBtsgqXKQimPEq9cFj7Av7UhNRcjUPznXfe4ZFHHmHlypUsX76c0aNH06dPH6666ipuuukm7HY73333HQ6HgwcffJDbb7/dEzYjR46kU6dOvP7661gsFtavX18tyEpKSnjuued49913CQkJYezYsdxxxx0sXXrm8aqJEyeydetWCgsLmTFjBgDx8fEAREVFMXPmTBo1asSmTZu47777iIqK4tFHH73g/qWmppKT859jZkeOHGHgwIH069cPgN27dzN48GCmTp3K3/72N44fP864ceMYN26cp47Ro0dz+PBhFi1ahM1m46GHHuLYsWM1favPzuR/nSbApn1H6TXxTcoqHNjDQ/jk8RG0aVrVZd45oAPNEmNolBDNxr1HmDxzAdsPneDjx0cA8Jvb+vLL1z4nY8zLpCXF8vavhrLzUC7vLFzP8v+9j19M/4yvf9xN18xGvDn+JmIiw4zcVb+xJzWDve5j/jjiL77KFPhHNWscmh06dODpp58GIDMzk+nTp7Nw4UIANm3axN69ez2d4bvvvkvbtm1ZvXo13bp1Izs7m0mTJpGVleV5/ukqKyuZPn06PXr0AKoCunXr1qxatYru3btXe6zdbic8PJzy8nKSk5Or3ffEE094/p6WlsbEiROZPXu2V6FpsVg82ysrK2Po0KH06tXL000///zzjBw50jMhKjMzk1deeYX+/fvz+uuvk52dzZdffsmqVavo1q0bAG+//TatW7e+4Gt7xU9Ds1XjBNa/8ksKSsr5aMlm7v7zx3z3wj20adqQ+wd39TyufVoSKfFRXPX4THbnnCQjJZ6YyDA+mHRbte1d+dsZvHjPNcxavJE9R/PY/teHuO8vc3n274t5aczg+t49v7S8YQS4dJk5qT1mPzx8VFM1PhDUoUOHat+npKRw7Ngxtm7dSmpqqicwAdq0aUNsbCxbt24F4JFHHmHMmDEMHDiQF154gd27d1fbltVq9QQNQFZWVrXne+vDDz+kT58+JCcnY7fbeeKJJ8jOzq7prnLPPfdQVFTEBx98gNlc9VZt2LCBmTNneo6v2u12Bg0ahMvlYu/evWzduhWr1UqXLl3O2I9aYbLUznbqWYjNSotGCXRp0YjnR1/NZenJTPvs7Ffe6NGqCQC7Duee9f4ZC9YRGxnGTT1bs3jTXob2bI3NauG2y9uyeNPeOtuHQLI3tTlHFZhSy0wKzTP993FBk8mEy+VdS/7MM8+wefNmhgwZwrfffkubNm345JNPalrCeS1fvpyRI0dy3XXXMW/ePH788Ucef/xxKipqtijA1KlTmT9/Pp999hlRUf+ZoFJcXMwDDzzA+vXrPV8bNmxg586dZGRk1Oq+nJXZr+ZunZPL7aa88uzH0tbvqRoeT4k/c2LQ8YJTPDt7MX95YAgATpebSkfVqROVDhdOl8YavbE8MdLoEiQAmfx0JKwmau03cOvWrTlw4AAHDhzwdJtbtmwhPz+fNm3aeB7XsmVLWrZsycMPP8yIESOYMWMGN998MwAOh4M1a9Z4hmK3b99Ofn7+OYc2Q0JCcDqrn2u2bNkymjVrxuOPP+65bf/+mi0bNmfOHJ599lm+/PLLM4Kwc+fObNmyhRYtWpz1uVlZWTgcDtauXevpmn/ej1ph9r9O87GZC7i2ayZNE2MoKq3gg8UbWbxpH/Of/R9255zkg8Ubua5bSxKiwtm47ygPv/kl/do1o0N68hnbmvDGl/x6aB8aN4gGoE/rVN5btIFrOrfgja/W0KdN0/rePb+zr0k6R9zqMqX2WU2B8aH+fGptDwcOHEj79u0ZOXIkL7/8Mg6Hg7Fjx9K/f3+6du1KaWkpkyZN4tZbbyU9PZ2DBw+yevVqbrnlFs82bDYb48eP55VXXsFqtTJu3Dh69ux5xvHMn6WlpTF//ny2b99OQkICMTExZGZmkp2dzezZs+nWrRtffPFFjbrZn376iVGjRjF58mTatm3LkSNHgKqAjo+PZ/LkyfTs2ZNx48YxZswYIiMj2bJlCwsWLGD69Om0atWKwYMH88ADD/D6669jtVqZMGEC4eHhl/YG/8wPQ/NYwSlG/eljck4WERMZRoe0JOY/+z9c3akFB44X8M2G3bz82XJOlVWS2iCaW3q34Yk7+p+xnflrd7IrJ5f3fj3Mc9u463uwZtdhejzyBt1bNubpEQPqcc/80/Iku45lSp0IM4UYXUKdq7XQNJlMzJ07l/Hjx9OvX79qp5xA1QSb3NxcRo0axdGjR2nQoAHDhg1jypQpnm1EREQwefJk7rzzTg4dOkTfvn15++23z/ma9913H4sXL6Zr164UFxezaNEibrzxRh5++GHGjRtHeXk5Q4YM4cknn/RM5LmQNWvWUFJSwtSpU6udKvLzKScdOnTgu+++4/HHH6dv37643W4yMjK4/fbbPY+dMWMGY8aMoX///iQlJTF16lSefPLJGr6j52Dxv09yb/9q6DnvS02M4bsX7vVqO4O6ZDKoS/XJYxFhIfzjN7ef4xny3/Y3TidHgSl1wIIZW4AcPjofk9tHlsL5+TzNWhvGDFQn9sLK942uQvzU7M4dOOwuMLoMCUAR5jB+kTzc6DLqnJZR8TehmsAhFye7UZoCU+pMaBAMzYJC0/+ERBhdgfip5cnRRpcgASzUrNCsVz8vZScXEBKBP64/K8bKbtSMQ+oypQ4pNMU3mcwQUkszcSVorEgO7PVAxXjBMHMWFJr+SUO0UgMHUppxUF2m1DF1muK7QjQZSLy3PEVdptS9cHNwXGJOoemPQtVpincOpjRVlyn1ItpS+9c69kUKTX+kTlO8tDwl1ugSJEjEWBWa4qt0rqZ44VByKgfUZUo9UacpvisizugKxA8sbxRvdAkSJEyYiLYEx4d5haY/ikwwugLxcYeTm5Dtzje6DAkSUZYIzKbgiJPg2MtAY1doyvktb6SfEak/wTI0CwpN/2QNhdAzL9AsAnA4qTH71WVKPYpRaIrPU7cp57CicQOjS5AgEx0kM2dBoem/dFxTziKnYWP2qcuUehZjCZ6RL4Wmv7Krm5AzrWiinwupf4m24JnRr9D0Vxqelf9ypGFj9qrLlHpmwUKCNXiWalRo+isNz8p/0bFMMUIDW2zQnG4CCk3/FR4DFpvRVYiPOJrYiD3kG12GBKGGtuBaREOh6a9MJohONroK8RErmiQaXYIEKYWm+I+4JkZXID7gWIMUdqvLFIM0tAXXoSKFpj+LSzW6AvEBy1OTjC5BgpQZU1DNnAWFpn9Tpxn0jjdIZjd5RpchQSrOGoPVZDG6jHql0PRnoZG64kmQW56q49pinGA7ngkKTf+nbjNoHU9IYpe6TDFQ45CGRpdQ7xSa/k7HNYPWiqYpRpcgQa5paPD9DCo0/V1cY6MrEAOcSGjITnWZYqBoi51Ya/CsOfszhaa/i0oCa4jRVUg9W9G0kdElSJBLDQ3O4+kKTX9nMkGsjmsGk9z4RHbqvEwxWNMQhab4q4YtjK5A6tGKZo1x4za6DAly6jTFfzXMNLoCqSe5cYnsUJcpBou3xmC3RBhdhiEUmoEgMl5XPQkSK9Vlig9oGqRdJig0A0dSS6MrkDp2MrYB2035RpchQmpI8J1q8jOFZqDQEG3AW5nWRF2mGM5qstAsCM/P/JlCM1DEpYItzOgqpI7kxSawTV2m+IBmoY0IMQfvtXwVmoHCbIbEDKOrkDqyMi1VXab4hJZhzYwuwVAKzUCiIdqAlBeTwFZ1meIDLJhpHhbc54UrNANJwxZVix1IQFmlLlN8RNPQFELNwb0CmUIzkNjCISHN6CqkFuXHxLPVnG90GSIAtAwP7qFZUGgGnsYdjK5AatHKtKa41GWKDzBjJiNMV1VSaAaalNZawD1AFETHqcsUn5EamkyYOdToMgyn0Aw0FhsktzG6CqkFK9ObqcsUnxHss2Z/ptAMRKmXGV2BXKLCqFi2mAuMLkMEAJvJSqvwNKPL8AkKzUAUlwoRcUZXIZdgZfM0XLiMLkMEgFbhaUG9oMHpFJqByGSCJpoQ5K8Ko2LZrC5TfEiHCK1t/TOFZqDSLFq/tSpdXab4joa2eJJDGhhdhs9QaAaqiFiI14F7f1Nkj2GzRV2m+I72EVpp7HQKzUCW2tHoCqSGVjVPx6kuU3yEzWSldXhzo8vwKQrNQJbSBkLtRlchXiqyR/OTukzxIZoAdCaFZiCzWKFZV6OrEC+tat5cXab4FE0AOpNCM9A161q14IH4NHWZ4muSbAmaAHQWCs1AFxKuY5t+YHW6ukzxLd3s7YwuwScpNINBeg9dMsyHFUdGsclaaHQZIh7x1hgyw5oaXYZPUmgGg4g4SM4yugo5h9XNM3DiNLoMEY9u9raY9EH7rBSawaJ5L6MrkLM4FRHFRnWZ4kOiLZE6zeQ8FJrBIrYxxGu4xdeszmiuLlN8Shd7W8wmRcO56J0JJuo2fcqpCDsbrUVGlyHiEWEO0wpAF6DQDCYNMyGmkdFVyL+tycjAoS5TfEjnyNZYTRajy/BpCs1gYjJB1lVGVyFASbidDeoyxYeEmkK4LLKV0WX4PIVmsGmQBokZRlcR9NRliq/pam9DqDnE6DJ8nkIzGGVdaXQFQa0kPJINIcVGlyHiEWkOp3NkG6PL8AsKzWAUnQyNtNqHUdZmtKDS7TC6DBGPXlGXYTNbjS7DL+hdClatroAjW8GlIcL6VBoWwfqQYnAbXYnvWPbOQpa99y0nD5wAILllY65++CZaX3kZJXnFfPXSJ+z47ifyDudij4+i3eAuDJ40jPDoCABK8or5+4Q32bVsKw3Sk7j9T2No0u4/15Kd89t3SWiayIBfXGvI/vm6eGsM7SJaGF2G31BoBquIWGjaBfatMrqSoLK2RSaV7nyjy/ApMSnxDHlsOA3Sk8ANq/+5hBn3TOOR+c/idkPh0XxuePIOklo2Iu9gLh/9ZiaFR/K4+83xAHzzyueUnyrjka+eZdm7C/nnpL/x8JdTANi/dhfZP+7m5t/dZeQu+rS+0Z11XmYN6J0KZpl9wRpqdBVBw9NlSjVtr+lE66suI7F5MokZyVz3m1sJiQxj/7rdpGQ1YfSb42l7TScapCWReXkbrpt8K5u/WY/TUTVKcnTXYTre2IPEjGR63nUFx3YeBsBZ6eCj37zDrS+MxmzRr7qzaRaaQkZYqtFl+BX9JAWzkAgteFCP1mW0oELHMs/L5XTx49wVVJSU06zL2YcMS4tKCLOHY7FWnU/YqE0qu5Zuwelwsn3xJlJaV4XAotf+RUavLFIvS6+3+v2JCRP9o3W93ZrS8Gywa94LDm6AkjyjKwloZaHh/BhaomOZ55Cz9QCv3Pg7HOWVhESG8f/eeojklo3PeFzxySK+efkzeo4c4LntygevZ85j7/B870nEpTZg+Ev3cnzPEVb/cwkPffYUH02eyfbvfyK1Qxq3vXiP51hosGsfkUkDW5zRZfgdk9vt1n/jYHd8N6z6wOgqAtrSth1YGaKLTJ+Lo8JB/qFcSotK2PjFalZ+8D1j5zxWLTjLikr564g/EhEbyT0zJmCxnfsz/+u3vUDfMdeQd/AEW75Zz5h3H+Efk2YQGWfnxqdH1Mcu+bRwcyijE28i3BJmdCl+R8OzUrXYQYrO0aorZSHhrA8tMboMn2YNsdIgPYnUDukMeWw4jdqk8sNbX3vuLysu5Y2R/0toZBij33rovIG56sPvCY+JoN2gzuxevo12g7pgsVm57Ppu7F6+tT52x+cNiO6mwLxICk2p0uYaTQqqI+syMyl3Vxpdhl9xu9w4KqqO/5YVlfLGiBexhli5Z+YEbGHnXrWmOLeQBX+e65kt63K6PBOGnA4nLqcG1tJDG9M6Qpf+ulgKTakSFgWttFJQbSsPCas6linn9MXz/2D3im2cPHCcnK0Hqr5fvo3Ow3r9e0j2RSpKyxn+v/dQVlRK4bF8Co/l43K6ztjWp09/QP8HBhOTEg9AWrdM1s5ZytGdh1kxazFp3YL7Ch42k5WrYnoYXYZf00Qg+Y9mXeDwT5B3wOhKAsa6Fi0pd+tY5vkUnyji7796k8Jj+YRHhZPSOpX7PphIq37t2LVsK9k/7gbg+T6PVnve4yv+l/jURM/32xZvInffUe585X7PbZf/v4Ec3LCXaddPoWnH5lzzyNB62SdfdXlUJ6KtdqPL8GuaCCTVFZ+AH97QSkG1oDwkjLfaNdHQrPiEFFsidzQYjMlkMroUv6bhWanO3gAy+xldRUD4sYWOZYpvsGDmmtheCsxaoNCUMzXvDTEpRlfh1ypCQlkXVmZ0GSIAdI9qT4It1ugyAoJCU85kNkOnYWDVtfUu1o8ZLSlzVxhdhghJtgS629sbXUbAUGjK2UXGQ7vrjK7CL1XYQlkbri5TjBdisjEkrh8WLchea/ROyrk1bg9NLjO6Cr+zvoW6TPENV8f2ItYaZXQZAUWhKefXdjBEJhhdhd+otIaoyxSf0D4ik1bhaUaXEXAUmnJ+1hDoPAzMFqMr8QvrW7SkVF2mGCzBGsuAmG5GlxGQFJpyYdHJkDXQ6Cp8XqU1hDURCkwxltVk5fq4fthMWrumLig0xTvp3SGppdFV+LQNLVpR6i43ugwJcldEd9PpJXVIoSne63AjhMcYXYVPquoyFZhirFbhabSPDO71deuaQlO8FxIOXW/X+ZtnsbFFS0rUZYqBkmwJXBPb2+gyAp5CU2omOgk6DgO0HNfPKi1WVutYphjIbo7gpvgrdByzHig0peaSMqG1Jgb9bFNmlrpMMYzNZGVowhXYLRFGlxIUFJpycZr3hKadja7CcA6LldURWpRdjDM49nIa2nQudX1RaMrFa3stJKQbXYWhNrbI4pRbixmIMS6P6kRmeFOjywgqCk25eGYzdLk1aFcMclgsrI5UlynGaBPenO5RWoi9vik05dLYwqDbHWALN7qSercpQ12mGKNRSCJXx/YyuoygpNCUSxcZD12Hg8VmdCX1xmGxsNruMLoMCUKJ1jiGxl+FxaSlLY2g0JTaEd+0KjjNwTHl/aeMLIrVZUo9i7fGcEvC1YSZda60URSaUnsaNIfOtwb84u5Oi4XVdqfRZUiQibHYuTXhaiIsYUaXEtQUmlK7kjKh0zAI4Ive/tS8FUXuUqPLkCBiN0dwa8I1OhfTBwTubzYxTnIWdLyJQFw1yGmxsCraZXQZEkQizGHc1uAaYqx2o0sRFJpSVxq1g8tuMLqKWre5eSuKXOoypX6EmUK4JeFq4qzRRpci/6bQlLrT5DJod53RVdQap8nMqih1mVI/Qk0hDEsYSKItzuhS5DTBMdVRjNOsC7hdsPkroyu5ZJszWlHoLjG6DAkCkeZwBaaPUmhK3UvrVrUIwobPqgLUDzlNZlbFAP5ZvviRGIudWxKuJtYaZXQpchYKTakfjdtDSCSs/Sc4/e8yWlsyWlHoUpcpdauBNY5bEgYSaQm+Fbb8hY5pSv1JbA69RlWFpx9x/dxlitShRiGJDG8wSIHp4xSaUr9iUqD3aIjwn2M1W5q3pEBdptShtNDG3BKvlX78gUJT6l9kfFVwRicbXckFuUxmVsYG3vmm4juywtO5Kf4KbEGyBKW/U2iKMULtVUO1Pn49zq3qMqUO9bR34NrYy7EE8ApagUb/UmIcayh0HwFNOhpdyVm5TCZ1mVInbCYrN8QNoHd0R0wm/Yz5E5Pb7XYbXYQI2euqzuV0+c5C6FsyWvFVtK5kIrUrxmLnpvgraKBzMP2SBtHFNzTtDNFJsPYjKCs0upp/d5kWnZcptappSDJD4vsTbg41uhS5SOo0xbeUn4IfP4bcfYaWsbV5K76MUZcptadzZGv6RXfBrOOXfk3/euJbQiOhx0ho3suwEtyYWBkX2NcElfpjwcyg2N4MiOmmwAwA6jTFd+VsgQ2f1/sKQtuat+RfMeX1+poSmBKssVwX11dryAYQHdMU35XSBuyJVcO1Rcfq5SXdmFgRZwWXQlMuTcfILPpFd8Fq0qhFIFGnKb7P5YQdi2HPcqjjH9ft6S35IlaBKRcvwhzGoNg+pIc1NroUqQPqNMX3mS2QdRUktYINc+HUyTp5GTewIt6mLlMuWnpoYwbF9iZC68cGLHWa4l+clbBtIexbXeub3pGWybw4/7sCixjPgoV+MV3oFJlldClSx9Rpin+x2KDt4Kquc+PnUFpQK5t1A8sTQsCl0JSaSbY14JrY3jSwxRpditQDdZrivyrLYcvXcHD9JW9KXabUVIjJRp/oTnSMaKWl8IKIQlP83/HdsHk+nMq9qKe7gfe6tOWEq7h265KA1Ty0CVfF9iDK4l/XhpVLp9CUwOBywr5VsPMHcNRsIs/OZi34PL6yjgqTQBJtsXNFTDcywlKNLkUMotCUwFJeDNu+hYMbvHq4G3i/SzuOu4rqti7xaxbMdLW3pXtUe2wmTQUJZgpNCUz5h6qumpJ/+LwP29Usg8/iHfVUlPijFmFN6RvdmThrtNGliA9QaErgcrvh4EbYvrBqIfizeL9LO46py5SzSA1J5vLozqSENDC6FPEhCk0JfJXlsGcZ7F1VbR3b3U0zmJugLlOqS7TG0Te6M2la0UfOQqEpwaOiBHYvg/1rwFnJrC7tOKouU/4txmKnT1QnWoWn6RQSOSddp0aCR0gEtB4IV4zjZPuB5LpLjK5IfECkOZwrY7ozuuFQsiLS/SowBwwYwIQJEwyt4ZlnnqFjx461vp3Ro0czdOjQS97u+aSlpfHyyy/X6DmaBibBJ9ROfNNejHF2ZO2pLWw4tYMKt045CTYJ1li62tuQFZ6ORVciuWgTJ05k/Pjxnu9Hjx5Nfn4+n3766SVtd9q0afjiQKhCU4JWhCWcvtFd6GZvx4+ntrHh1HZKXGVGlyV1LDUkma72tqSFNvKrrtJX2e127HZ7rW83Jiam1rdZGzQ8K0EvzBxKr6jLuC/pFq6NvZwUm2ZLBhoTJlqFpTGywRBua3AN6WGN6zwwBwwYwEMPPcSjjz5KfHw8ycnJPPPMM577s7Ozuemmm7Db7URHRzN8+HCOHj3quf/n4cr33nuPtLQ0YmJiuOOOOygqqn4c3uVynfM1vHmdDRs2cMUVVxAVFUV0dDRdunRhzZo1AMycOZPY2Fg+/fRTMjMzCQsLY9CgQRw4cOCMOn/++zvvvMPcuXMxmUyYTCYWL14MwOTJk2nZsiURERE0b96cJ598ksrKc4/wnD48u2/fPs/2Tv8aMGCA5/FLliyhb9++hIeHk5qaykMPPcSpU/+ZNX/s2DFuuOEGwsPDSU9PZ9asWed87fNRaIr8m8VkoXVEc0YkXsfIBkNoG9FCFxD2c6GmEDpFZnFPw6EMie9HUkhCvb7+O++8Q2RkJCtXruSPf/wjzz77LAsWLMDlcnHTTTdx8uRJvvvuOxYsWMCePXu4/fbbqz1/9+7dfPrpp8ybN4958+bx3Xff8cILL3j1GoBXrzNy5EiaNGnC6tWrWbt2Lb/5zW+w2Wye+0tKSnjuued49913Wbp0Kfn5+dxxxx1n3d+JEycyfPhwBg8eTE5ODjk5OfTu3RuAqKgoZs6cyZYtW5g2bRpvvvkmf/7zn716H1NTUz3by8nJ4ccffyQhIYF+/fp53qfBgwdzyy23sHHjRj788EOWLFnCuHHjPNsYPXo0Bw4cYNGiRXz00Ue89tprHDtW84vba3hW5CySQhIYFNKbftFd2Fyyiw2ntlPg1Nq0/sCEiaahKbSLyCAjrKmhH3w6dOjA008/DUBmZibTp09n4cKFAGzatIm9e/eSmlq1JN+7775L27ZtWb16Nd26dQOqQm/mzJlERUUB8D//8z8sXLiQ55577oKvcfXVV7Nw4cILvk52djaTJk0iKyvLs43TVVZWMn36dHr06AFUhXTr1q1ZtWoV3bt3r/ZYu91OeHg45eXlJCcnV7vviSee8Pw9LS2NiRMnMnv2bB599NELvo8Wi8WzvbKyMoYOHUqvXr08XfXzzz/PyJEjPZOiMjMzeeWVV+jfvz+vv/462dnZfPnll6xatcrz3r799tu0bt36gq/93xSaIucRbg6lq70tXSLbsK/8MFtL97C77ACVbp3f6WtiLFG0jcigbUSGzyyk3qFDh2rfp6SkcOzYMbZu3UpqaqonyADatGlDbGwsW7du9fxiT0tL8wTm6c/35jUAr17nkUceYcyYMbz33nsMHDiQ2267jYyMDM/jrVarpx6ArKwsz/P/OzTP58MPP+SVV15h9+7dFBcX43A4iI6u+SpL99xzD0VFRSxYsACzuWqwdMOGDWzcuLHakKvb7cblcrF371527NiB1WqlS5cuZ+xHTSk0RbxgMplID2tMelhjKt0O9pYdZFvpPvaWHcSJy+jygpbVZKVlWFPaRrSgSUiSz03sOX2YE6p+jlwu739evHn+pb7GM888w5133skXX3zBl19+ydNPP83s2bO5+eabvd7GhSxfvpyRI0cyZcoUBg0aRExMDLNnz+all16q0XamTp3K/PnzWbVqVbUPE8XFxTzwwAM89NBDZzynadOm7Nix45L34WcKTZEaspmstAxPo2V4GuWuCnaVHWBb6V4OlOfgwvemyAeacHMozUObkBGWSrOwRn65gHrr1q05cOAABw4c8HSBW7ZsIT8/nzZt2tT767Rs2ZKWLVvy8MMPM2LECGbMmOEJTYfDwZo1azxd5fbt28nPzz/n0GZISAhOp7PabcuWLaNZs2Y8/vjjntv2799fo32ZM2cOzz77LF9++WW1Thigc+fObNmyhRYtWpz1uVlZWTgcDtauXevpmn/ej5ryv582ER8Sag7xDAmWOMvYVZbN3vJDHCg/onM/a1GMJYoWYalkhKXSKCQRs8m/5zAOHDiQ9u3bM3LkSF5++WUcDgdjx46lf//+dO3atd5ep7S0lEmTJnHrrbeSnp7OwYMHWb16NbfccotnGzabjfHjx/PKK69gtVoZN24cPXv2POfQbFpaGvPnz2f79u0kJCQQExNDZmYm2dnZzJ49m27duvHFF1/wySefeL0fP/30E6NGjWLy5Mm0bduWI0eOAFUBHR8fz+TJk+nZsyfjxo1jzJgxREZGsmXLFhYsWMD06dNp1aoVgwcP5oEHHuD111/HarUyYcIEwsPDa/ye+vdPnogPibCE0SGyJTfFX8HY5Nu5PWEQPeztSbIlYMK3hg19nRkzKbZE+kR1ZFTiDdybdDP9Y7rSJDTJ7wMTqoZQ586dS1xcHP369WPgwIE0b96cDz/8sF5fx2KxkJuby6hRo2jZsiXDhw/n2muvZcqUKZ5tREREMHnyZO6880769OmD3W4/b5333XcfrVq1omvXriQmJrJ06VJuvPFGHn74YcaNG0fHjh1ZtmwZTz75pNf7sWbNGkpKSpg6dSopKSmer2HDhgFVx3W/++47duzYQd++fenUqRNPPfUUjRo18mxjxowZNGrUiP79+zNs2DDuv/9+GjZsWNO3VGvPitSHUlcZ+8tz2F92mP3lORS7tITf6SxYSAlpQJOQJJqEJpFiS8Rm1kCY0WbOnMmECRMuahgzUOmnUqQehJvDyApPJys8HYBiZwlHKk5wpPIERypyOVqZS7m74gJbCRwhJhspIYlVIRnSkOSQBlrKTvyCQlPEAHZLBC3Cm9IivClQNT0+z1nIkYpcjlSe4GhFLicdBX4fpGZMxFmjaWCNo4Etlga2OBpYY4m22H1upquINzQ8K+LDSl1l5DmKyHcUVv3pLCTfUUSeo9BnJhqZMRFpiSDKHIHdEkG01U4Da1VAxltjtKqSBBSFpoifKnGWUuwqocRZTqmrjFJX9T9LXOWUucooc1Xgwo3L7cKNGxcuXO6qP/+bGTM2kwWryYrVZCHUHEKIyUaoOYRQk41wcxhRlgjslkiiLBFEWSKJNIera5SgodAUCWIutwsXbtxuNxaTOSBmporUJYWmiIiIl/SxUkRExEsKTRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETESwpNERERLyk0RUREvKTQFBER8ZJCU0RExEsKTRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETESwpNERERLyk0RUREvKTQFBER8ZJCU0RExEsKTRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETESwpNERERLyk0RUREvKTQFBER8ZJCU0RExEsKTRERES8pNEVERLyk0BQREfGSQlNERMRLCk0REREvKTRFRES8pNAUERHxkkJTRETES/8fuQerO05JNGIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def show_train_dataset_pie_chart(train_dataset: pd.DataFrame, title: str):\n",
    "    plt.figure()\n",
    "    data = train_dataset.groupby(\"outcome_group\").size()\n",
    "    print(\"\\n\" + title + \",\")\n",
    "    print(data)\n",
    "    data = [int(data[0]), int(data[1]), int(data[2])]\n",
    "    labels = [\"deceased\", \"hospitalized\", \"nonhospitalized\"]\n",
    "    colours = sns.color_palette('pastel')[0:4]\n",
    "    plt.pie(x=data, labels=labels, colors=colours, autopct='%.0f%%')\n",
    "    plt.title(title)\n",
    "    plt.show()\n",
    "\n",
    "show_train_dataset_pie_chart(train_data, \"Before Balancing\")\n",
    "\n",
    "deceased = train_data[train_data[\"outcome_group\"] == 0]\n",
    "new_deceased = deceased.sample(frac=10, replace=True, random_state=1)\n",
    "new_deceased.reset_index(inplace=True, drop=True)\n",
    "\n",
    "hospitalized = train_data[train_data[\"outcome_group\"] == 1]\n",
    "hospitalized_sample = np.random.choice(hospitalized.index, 3000, replace=True)\n",
    "new_hospitalized = hospitalized.drop(hospitalized_sample)\n",
    "new_hospitalized.reset_index(inplace=True, drop=True)\n",
    "\n",
    "nonhospitalized = train_data[train_data[\"outcome_group\"] == 2]\n",
    "new_nonhospitalized = nonhospitalized.sample(frac=3.3, replace=True, random_state=1)\n",
    "new_nonhospitalized.reset_index(inplace=True, drop=True)\n",
    "\n",
    "new_train = pd.concat([new_deceased, new_hospitalized, new_nonhospitalized])\n",
    "new_train.sort_index(axis = 0, inplace=True)\n",
    "new_train.reset_index(inplace=True, drop=True)\n",
    "\n",
    "show_train_dataset_pie_chart(new_train, \"After Balancing\")\n",
    "\n",
    "train_data = new_train"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.4 Building Models"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train/Validation Split,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, validation_data = train_test_split(train_data, test_size=0.2)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XG Boost,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n",
      "[CV 1/5; 1/12] START learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 1/12] END learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.827 total time=   0.6s\n",
      "[CV 2/5; 1/12] START learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 1/12] END learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.816 total time=   0.6s\n",
      "[CV 3/5; 1/12] START learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 1/12] END learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.812 total time=   0.7s\n",
      "[CV 4/5; 1/12] START learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 1/12] END learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.802 total time=   0.7s\n",
      "[CV 5/5; 1/12] START learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 1/12] END learning_rate=0.2, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.800 total time=   0.6s\n",
      "[CV 1/5; 2/12] START learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 2/12] END learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.828 total time=   1.1s\n",
      "[CV 2/5; 2/12] START learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 2/12] END learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.818 total time=   1.1s\n",
      "[CV 3/5; 2/12] START learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 2/12] END learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.813 total time=   1.0s\n",
      "[CV 4/5; 2/12] START learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 2/12] END learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.803 total time=   1.0s\n",
      "[CV 5/5; 2/12] START learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 2/12] END learning_rate=0.2, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.801 total time=   1.0s\n",
      "[CV 1/5; 3/12] START learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 3/12] END learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.827 total time=   0.8s\n",
      "[CV 2/5; 3/12] START learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 3/12] END learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.819 total time=   0.8s\n",
      "[CV 3/5; 3/12] START learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 3/12] END learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.814 total time=   0.7s\n",
      "[CV 4/5; 3/12] START learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 3/12] END learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.803 total time=   0.7s\n",
      "[CV 5/5; 3/12] START learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 3/12] END learning_rate=0.2, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.801 total time=   0.8s\n",
      "[CV 1/5; 4/12] START learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 4/12] END learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.827 total time=   1.3s\n",
      "[CV 2/5; 4/12] START learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 4/12] END learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.818 total time=   1.4s\n",
      "[CV 3/5; 4/12] START learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 4/12] END learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.812 total time=   1.3s\n",
      "[CV 4/5; 4/12] START learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 4/12] END learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.805 total time=   1.4s\n",
      "[CV 5/5; 4/12] START learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 4/12] END learning_rate=0.2, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.802 total time=   1.3s\n",
      "[CV 1/5; 5/12] START learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 5/12] END learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.828 total time=   1.0s\n",
      "[CV 2/5; 5/12] START learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 5/12] END learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.819 total time=   1.0s\n",
      "[CV 3/5; 5/12] START learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 5/12] END learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.812 total time=   1.0s\n",
      "[CV 4/5; 5/12] START learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 5/12] END learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.804 total time=   1.0s\n",
      "[CV 5/5; 5/12] START learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 5/12] END learning_rate=0.2, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.803 total time=   1.0s\n",
      "[CV 1/5; 6/12] START learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 6/12] END learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.828 total time=   1.6s\n",
      "[CV 2/5; 6/12] START learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 6/12] END learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.818 total time=   1.6s\n",
      "[CV 3/5; 6/12] START learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 6/12] END learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.812 total time=   1.6s\n",
      "[CV 4/5; 6/12] START learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 6/12] END learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.804 total time=   1.6s\n",
      "[CV 5/5; 6/12] START learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 6/12] END learning_rate=0.2, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.804 total time=   1.6s\n",
      "[CV 1/5; 7/12] START learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 7/12] END learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.828 total time=   0.6s\n",
      "[CV 2/5; 7/12] START learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 7/12] END learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.818 total time=   0.5s\n",
      "[CV 3/5; 7/12] START learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 7/12] END learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.812 total time=   0.5s\n",
      "[CV 4/5; 7/12] START learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 7/12] END learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.803 total time=   0.6s\n",
      "[CV 5/5; 7/12] START learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 7/12] END learning_rate=0.3, max_depth=6, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.801 total time=   0.6s\n",
      "[CV 1/5; 8/12] START learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 8/12] END learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.827 total time=   1.0s\n",
      "[CV 2/5; 8/12] START learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 8/12] END learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.819 total time=   1.0s\n",
      "[CV 3/5; 8/12] START learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 8/12] END learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.812 total time=   1.0s\n",
      "[CV 4/5; 8/12] START learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 8/12] END learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.804 total time=   1.0s\n",
      "[CV 5/5; 8/12] START learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 8/12] END learning_rate=0.3, max_depth=6, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.803 total time=   1.2s\n",
      "[CV 1/5; 9/12] START learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 9/12] END learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.827 total time=   0.7s\n",
      "[CV 2/5; 9/12] START learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 9/12] END learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.819 total time=   0.7s\n",
      "[CV 3/5; 9/12] START learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 9/12] END learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.812 total time=   0.7s\n",
      "[CV 4/5; 9/12] START learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 9/12] END learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.804 total time=   0.7s\n",
      "[CV 5/5; 9/12] START learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 9/12] END learning_rate=0.3, max_depth=8, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.802 total time=   0.7s\n",
      "[CV 1/5; 10/12] START learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 10/12] END learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.827 total time=   1.3s\n",
      "[CV 2/5; 10/12] START learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 10/12] END learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.819 total time=   1.3s\n",
      "[CV 3/5; 10/12] START learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 10/12] END learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.812 total time=   1.3s\n",
      "[CV 4/5; 10/12] START learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 10/12] END learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.805 total time=   1.3s\n",
      "[CV 5/5; 10/12] START learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 10/12] END learning_rate=0.3, max_depth=8, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.803 total time=   1.3s\n",
      "[CV 1/5; 11/12] START learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 11/12] END learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.827 total time=   1.0s\n",
      "[CV 2/5; 11/12] START learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 11/12] END learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.819 total time=   1.0s\n",
      "[CV 3/5; 11/12] START learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 11/12] END learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.812 total time=   0.9s\n",
      "[CV 4/5; 11/12] START learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 11/12] END learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.804 total time=   0.9s\n",
      "[CV 5/5; 11/12] START learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 11/12] END learning_rate=0.3, max_depth=10, n_estimators=150, num_class=3, objective=multi:softmax;, score=0.804 total time=   1.0s\n",
      "[CV 1/5; 12/12] START learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 1/5; 12/12] END learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.827 total time=   1.7s\n",
      "[CV 2/5; 12/12] START learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 2/5; 12/12] END learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.819 total time=   1.6s\n",
      "[CV 3/5; 12/12] START learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 3/5; 12/12] END learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.811 total time=   1.6s\n",
      "[CV 4/5; 12/12] START learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 4/5; 12/12] END learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.805 total time=   1.6s\n",
      "[CV 5/5; 12/12] START learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax\n",
      "[CV 5/5; 12/12] END learning_rate=0.3, max_depth=10, n_estimators=250, num_class=3, objective=multi:softmax;, score=0.804 total time=   1.6s\n",
      "XG Boost GridSearchCV best score = 0.8133098712925386\n",
      "XG Boost GridSearchCV best parameters = {'learning_rate': 0.2, 'max_depth': 10, 'n_estimators': 250, 'num_class': 3, 'objective': 'multi:softmax'}\n",
      "XG Boost GridSearchCV deceased class f1-score = 0.7329870471431389\n",
      "XG Boost GridSearchCV hospitalized class f1-score = 0.9305241753276097\n",
      "XG Boost GridSearchCV nonhospitalized class f1-score = 0.7992334717342703\n",
      "XG Boost GridSearchCV accuracy score = 0.8263465656399275\n"
     ]
    }
   ],
   "source": [
    "# Takes about 1 - 2 minutes to run.\n",
    "\n",
    "# Decide number of k-fold splits\n",
    "k = 5\n",
    "# Create model with blank parameters\n",
    "xgb_model = xgb.XGBClassifier(random_state = 1)\n",
    "# Create space of possible parameters\n",
    "parameter_search_space = {\n",
    "    \"learning_rate\": [0.2, 0.3],\n",
    "    \"max_depth\": [6, 8, 10],\n",
    "    \"n_estimators\": [150, 250],\n",
    "    \"objective\": [\"multi:softmax\"],\n",
    "    \"num_class\": [3]\n",
    "}\n",
    "# Create grid search cross validation object\n",
    "grid_search_cv = GridSearchCV(\n",
    "    estimator=xgb_model,\n",
    "    param_grid=parameter_search_space,\n",
    "    scoring=\"f1_macro\",\n",
    "    cv=k,\n",
    "    verbose=10\n",
    ")\n",
    "# Put data and labels in proper format\n",
    "data = train_data.iloc[:, :4].values\n",
    "labels = train_data.iloc[:, 4].values.reshape(-1, 1)\n",
    "# Fit grid search object\n",
    "grid_search_cv.fit(data, labels)\n",
    "# Print and save results.\n",
    "print(\"XG Boost GridSearchCV best score = \" + str(grid_search_cv.best_score_))\n",
    "print(\"XG Boost GridSearchCV best parameters = \" + str(grid_search_cv.best_params_))\n",
    "predictions = grid_search_cv.predict(data)\n",
    "_, _, fscore, _ = precision_recall_fscore_support(predictions, labels)\n",
    "print(\"XG Boost GridSearchCV deceased class f1-score = \" + str(fscore[0]))\n",
    "print(\"XG Boost GridSearchCV hospitalized class f1-score = \" + str(fscore[1]))\n",
    "print(\"XG Boost GridSearchCV nonhospitalized class f1-score = \" + str(fscore[2]))\n",
    "accuracy = accuracy_score(predictions, labels)\n",
    "print(\"XG Boost GridSearchCV accuracy score = \" + str(accuracy))\n",
    "pd.DataFrame(grid_search_cv.cv_results_).to_csv(\"xgboost_results.csv\")\n",
    "xgb_model = grid_search_cv.best_estimator_"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 32 candidates, totalling 160 fits\n",
      "[CV 1/5; 1/32] START criterion=gini, max_features=1, n_estimators=50............\n",
      "[CV 1/5; 1/32] END criterion=gini, max_features=1, n_estimators=50;, score=0.827 total time=   0.2s\n",
      "[CV 2/5; 1/32] START criterion=gini, max_features=1, n_estimators=50............\n",
      "[CV 2/5; 1/32] END criterion=gini, max_features=1, n_estimators=50;, score=0.819 total time=   0.2s\n",
      "[CV 3/5; 1/32] START criterion=gini, max_features=1, n_estimators=50............\n",
      "[CV 3/5; 1/32] END criterion=gini, max_features=1, n_estimators=50;, score=0.813 total time=   0.2s\n",
      "[CV 4/5; 1/32] START criterion=gini, max_features=1, n_estimators=50............\n",
      "[CV 4/5; 1/32] END criterion=gini, max_features=1, n_estimators=50;, score=0.804 total time=   0.2s\n",
      "[CV 5/5; 1/32] START criterion=gini, max_features=1, n_estimators=50............\n",
      "[CV 5/5; 1/32] END criterion=gini, max_features=1, n_estimators=50;, score=0.805 total time=   0.2s\n",
      "[CV 1/5; 2/32] START criterion=gini, max_features=1, n_estimators=100...........\n",
      "[CV 1/5; 2/32] END criterion=gini, max_features=1, n_estimators=100;, score=0.827 total time=   0.5s\n",
      "[CV 2/5; 2/32] START criterion=gini, max_features=1, n_estimators=100...........\n",
      "[CV 2/5; 2/32] END criterion=gini, max_features=1, n_estimators=100;, score=0.820 total time=   0.5s\n",
      "[CV 3/5; 2/32] START criterion=gini, max_features=1, n_estimators=100...........\n",
      "[CV 3/5; 2/32] END criterion=gini, max_features=1, n_estimators=100;, score=0.812 total time=   0.5s\n",
      "[CV 4/5; 2/32] START criterion=gini, max_features=1, n_estimators=100...........\n",
      "[CV 4/5; 2/32] END criterion=gini, max_features=1, n_estimators=100;, score=0.804 total time=   0.5s\n",
      "[CV 5/5; 2/32] START criterion=gini, max_features=1, n_estimators=100...........\n",
      "[CV 5/5; 2/32] END criterion=gini, max_features=1, n_estimators=100;, score=0.806 total time=   0.5s\n",
      "[CV 1/5; 3/32] START criterion=gini, max_features=1, n_estimators=150...........\n",
      "[CV 1/5; 3/32] END criterion=gini, max_features=1, n_estimators=150;, score=0.827 total time=   0.8s\n",
      "[CV 2/5; 3/32] START criterion=gini, max_features=1, n_estimators=150...........\n",
      "[CV 2/5; 3/32] END criterion=gini, max_features=1, n_estimators=150;, score=0.820 total time=   0.8s\n",
      "[CV 3/5; 3/32] START criterion=gini, max_features=1, n_estimators=150...........\n",
      "[CV 3/5; 3/32] END criterion=gini, max_features=1, n_estimators=150;, score=0.813 total time=   0.8s\n",
      "[CV 4/5; 3/32] START criterion=gini, max_features=1, n_estimators=150...........\n",
      "[CV 4/5; 3/32] END criterion=gini, max_features=1, n_estimators=150;, score=0.804 total time=   0.8s\n",
      "[CV 5/5; 3/32] START criterion=gini, max_features=1, n_estimators=150...........\n",
      "[CV 5/5; 3/32] END criterion=gini, max_features=1, n_estimators=150;, score=0.805 total time=   0.8s\n",
      "[CV 1/5; 4/32] START criterion=gini, max_features=1, n_estimators=200...........\n",
      "[CV 1/5; 4/32] END criterion=gini, max_features=1, n_estimators=200;, score=0.827 total time=   1.2s\n",
      "[CV 2/5; 4/32] START criterion=gini, max_features=1, n_estimators=200...........\n",
      "[CV 2/5; 4/32] END criterion=gini, max_features=1, n_estimators=200;, score=0.819 total time=   1.1s\n",
      "[CV 3/5; 4/32] START criterion=gini, max_features=1, n_estimators=200...........\n",
      "[CV 3/5; 4/32] END criterion=gini, max_features=1, n_estimators=200;, score=0.813 total time=   1.2s\n",
      "[CV 4/5; 4/32] START criterion=gini, max_features=1, n_estimators=200...........\n",
      "[CV 4/5; 4/32] END criterion=gini, max_features=1, n_estimators=200;, score=0.804 total time=   1.1s\n",
      "[CV 5/5; 4/32] START criterion=gini, max_features=1, n_estimators=200...........\n",
      "[CV 5/5; 4/32] END criterion=gini, max_features=1, n_estimators=200;, score=0.806 total time=   1.2s\n",
      "[CV 1/5; 5/32] START criterion=gini, max_features=2, n_estimators=50............\n",
      "[CV 1/5; 5/32] END criterion=gini, max_features=2, n_estimators=50;, score=0.827 total time=   0.2s\n",
      "[CV 2/5; 5/32] START criterion=gini, max_features=2, n_estimators=50............\n",
      "[CV 2/5; 5/32] END criterion=gini, max_features=2, n_estimators=50;, score=0.819 total time=   0.2s\n",
      "[CV 3/5; 5/32] START criterion=gini, max_features=2, n_estimators=50............\n",
      "[CV 3/5; 5/32] END criterion=gini, max_features=2, n_estimators=50;, score=0.813 total time=   0.2s\n",
      "[CV 4/5; 5/32] START criterion=gini, max_features=2, n_estimators=50............\n",
      "[CV 4/5; 5/32] END criterion=gini, max_features=2, n_estimators=50;, score=0.804 total time=   0.2s\n",
      "[CV 5/5; 5/32] START criterion=gini, max_features=2, n_estimators=50............\n",
      "[CV 5/5; 5/32] END criterion=gini, max_features=2, n_estimators=50;, score=0.805 total time=   0.3s\n",
      "[CV 1/5; 6/32] START criterion=gini, max_features=2, n_estimators=100...........\n",
      "[CV 1/5; 6/32] END criterion=gini, max_features=2, n_estimators=100;, score=0.827 total time=   0.6s\n",
      "[CV 2/5; 6/32] START criterion=gini, max_features=2, n_estimators=100...........\n",
      "[CV 2/5; 6/32] END criterion=gini, max_features=2, n_estimators=100;, score=0.820 total time=   0.6s\n",
      "[CV 3/5; 6/32] START criterion=gini, max_features=2, n_estimators=100...........\n",
      "[CV 3/5; 6/32] END criterion=gini, max_features=2, n_estimators=100;, score=0.812 total time=   0.6s\n",
      "[CV 4/5; 6/32] START criterion=gini, max_features=2, n_estimators=100...........\n",
      "[CV 4/5; 6/32] END criterion=gini, max_features=2, n_estimators=100;, score=0.804 total time=   0.6s\n",
      "[CV 5/5; 6/32] START criterion=gini, max_features=2, n_estimators=100...........\n",
      "[CV 5/5; 6/32] END criterion=gini, max_features=2, n_estimators=100;, score=0.805 total time=   0.6s\n",
      "[CV 1/5; 7/32] START criterion=gini, max_features=2, n_estimators=150...........\n",
      "[CV 1/5; 7/32] END criterion=gini, max_features=2, n_estimators=150;, score=0.827 total time=   0.9s\n",
      "[CV 2/5; 7/32] START criterion=gini, max_features=2, n_estimators=150...........\n",
      "[CV 2/5; 7/32] END criterion=gini, max_features=2, n_estimators=150;, score=0.820 total time=   0.9s\n",
      "[CV 3/5; 7/32] START criterion=gini, max_features=2, n_estimators=150...........\n",
      "[CV 3/5; 7/32] END criterion=gini, max_features=2, n_estimators=150;, score=0.813 total time=   0.9s\n",
      "[CV 4/5; 7/32] START criterion=gini, max_features=2, n_estimators=150...........\n",
      "[CV 4/5; 7/32] END criterion=gini, max_features=2, n_estimators=150;, score=0.804 total time=   0.9s\n",
      "[CV 5/5; 7/32] START criterion=gini, max_features=2, n_estimators=150...........\n",
      "[CV 5/5; 7/32] END criterion=gini, max_features=2, n_estimators=150;, score=0.805 total time=   0.9s\n",
      "[CV 1/5; 8/32] START criterion=gini, max_features=2, n_estimators=200...........\n",
      "[CV 1/5; 8/32] END criterion=gini, max_features=2, n_estimators=200;, score=0.827 total time=   1.2s\n",
      "[CV 2/5; 8/32] START criterion=gini, max_features=2, n_estimators=200...........\n",
      "[CV 2/5; 8/32] END criterion=gini, max_features=2, n_estimators=200;, score=0.819 total time=   1.2s\n",
      "[CV 3/5; 8/32] START criterion=gini, max_features=2, n_estimators=200...........\n",
      "[CV 3/5; 8/32] END criterion=gini, max_features=2, n_estimators=200;, score=0.812 total time=   1.2s\n",
      "[CV 4/5; 8/32] START criterion=gini, max_features=2, n_estimators=200...........\n",
      "[CV 4/5; 8/32] END criterion=gini, max_features=2, n_estimators=200;, score=0.804 total time=   1.2s\n",
      "[CV 5/5; 8/32] START criterion=gini, max_features=2, n_estimators=200...........\n",
      "[CV 5/5; 8/32] END criterion=gini, max_features=2, n_estimators=200;, score=0.805 total time=   1.2s\n",
      "[CV 1/5; 9/32] START criterion=gini, max_features=3, n_estimators=50............\n",
      "[CV 1/5; 9/32] END criterion=gini, max_features=3, n_estimators=50;, score=0.827 total time=   0.3s\n",
      "[CV 2/5; 9/32] START criterion=gini, max_features=3, n_estimators=50............\n",
      "[CV 2/5; 9/32] END criterion=gini, max_features=3, n_estimators=50;, score=0.819 total time=   0.3s\n",
      "[CV 3/5; 9/32] START criterion=gini, max_features=3, n_estimators=50............\n",
      "[CV 3/5; 9/32] END criterion=gini, max_features=3, n_estimators=50;, score=0.813 total time=   0.3s\n",
      "[CV 4/5; 9/32] START criterion=gini, max_features=3, n_estimators=50............\n",
      "[CV 4/5; 9/32] END criterion=gini, max_features=3, n_estimators=50;, score=0.804 total time=   0.3s\n",
      "[CV 5/5; 9/32] START criterion=gini, max_features=3, n_estimators=50............\n",
      "[CV 5/5; 9/32] END criterion=gini, max_features=3, n_estimators=50;, score=0.805 total time=   0.3s\n",
      "[CV 1/5; 10/32] START criterion=gini, max_features=3, n_estimators=100..........\n",
      "[CV 1/5; 10/32] END criterion=gini, max_features=3, n_estimators=100;, score=0.827 total time=   0.6s\n",
      "[CV 2/5; 10/32] START criterion=gini, max_features=3, n_estimators=100..........\n",
      "[CV 2/5; 10/32] END criterion=gini, max_features=3, n_estimators=100;, score=0.819 total time=   0.6s\n",
      "[CV 3/5; 10/32] START criterion=gini, max_features=3, n_estimators=100..........\n",
      "[CV 3/5; 10/32] END criterion=gini, max_features=3, n_estimators=100;, score=0.813 total time=   0.6s\n",
      "[CV 4/5; 10/32] START criterion=gini, max_features=3, n_estimators=100..........\n",
      "[CV 4/5; 10/32] END criterion=gini, max_features=3, n_estimators=100;, score=0.804 total time=   0.6s\n",
      "[CV 5/5; 10/32] START criterion=gini, max_features=3, n_estimators=100..........\n",
      "[CV 5/5; 10/32] END criterion=gini, max_features=3, n_estimators=100;, score=0.806 total time=   0.7s\n",
      "[CV 1/5; 11/32] START criterion=gini, max_features=3, n_estimators=150..........\n",
      "[CV 1/5; 11/32] END criterion=gini, max_features=3, n_estimators=150;, score=0.827 total time=   1.0s\n",
      "[CV 2/5; 11/32] START criterion=gini, max_features=3, n_estimators=150..........\n",
      "[CV 2/5; 11/32] END criterion=gini, max_features=3, n_estimators=150;, score=0.819 total time=   1.0s\n",
      "[CV 3/5; 11/32] START criterion=gini, max_features=3, n_estimators=150..........\n",
      "[CV 3/5; 11/32] END criterion=gini, max_features=3, n_estimators=150;, score=0.813 total time=   1.0s\n",
      "[CV 4/5; 11/32] START criterion=gini, max_features=3, n_estimators=150..........\n",
      "[CV 4/5; 11/32] END criterion=gini, max_features=3, n_estimators=150;, score=0.804 total time=   1.0s\n",
      "[CV 5/5; 11/32] START criterion=gini, max_features=3, n_estimators=150..........\n",
      "[CV 5/5; 11/32] END criterion=gini, max_features=3, n_estimators=150;, score=0.805 total time=   1.0s\n",
      "[CV 1/5; 12/32] START criterion=gini, max_features=3, n_estimators=200..........\n",
      "[CV 1/5; 12/32] END criterion=gini, max_features=3, n_estimators=200;, score=0.827 total time=   1.3s\n",
      "[CV 2/5; 12/32] START criterion=gini, max_features=3, n_estimators=200..........\n",
      "[CV 2/5; 12/32] END criterion=gini, max_features=3, n_estimators=200;, score=0.819 total time=   1.3s\n",
      "[CV 3/5; 12/32] START criterion=gini, max_features=3, n_estimators=200..........\n",
      "[CV 3/5; 12/32] END criterion=gini, max_features=3, n_estimators=200;, score=0.813 total time=   1.4s\n",
      "[CV 4/5; 12/32] START criterion=gini, max_features=3, n_estimators=200..........\n",
      "[CV 4/5; 12/32] END criterion=gini, max_features=3, n_estimators=200;, score=0.805 total time=   1.4s\n",
      "[CV 5/5; 12/32] START criterion=gini, max_features=3, n_estimators=200..........\n",
      "[CV 5/5; 12/32] END criterion=gini, max_features=3, n_estimators=200;, score=0.805 total time=   1.4s\n",
      "[CV 1/5; 13/32] START criterion=gini, max_features=4, n_estimators=50...........\n",
      "[CV 1/5; 13/32] END criterion=gini, max_features=4, n_estimators=50;, score=0.827 total time=   0.3s\n",
      "[CV 2/5; 13/32] START criterion=gini, max_features=4, n_estimators=50...........\n",
      "[CV 2/5; 13/32] END criterion=gini, max_features=4, n_estimators=50;, score=0.819 total time=   0.3s\n",
      "[CV 3/5; 13/32] START criterion=gini, max_features=4, n_estimators=50...........\n",
      "[CV 3/5; 13/32] END criterion=gini, max_features=4, n_estimators=50;, score=0.812 total time=   0.3s\n",
      "[CV 4/5; 13/32] START criterion=gini, max_features=4, n_estimators=50...........\n",
      "[CV 4/5; 13/32] END criterion=gini, max_features=4, n_estimators=50;, score=0.804 total time=   0.3s\n",
      "[CV 5/5; 13/32] START criterion=gini, max_features=4, n_estimators=50...........\n",
      "[CV 5/5; 13/32] END criterion=gini, max_features=4, n_estimators=50;, score=0.805 total time=   0.3s\n",
      "[CV 1/5; 14/32] START criterion=gini, max_features=4, n_estimators=100..........\n",
      "[CV 1/5; 14/32] END criterion=gini, max_features=4, n_estimators=100;, score=0.827 total time=   0.7s\n",
      "[CV 2/5; 14/32] START criterion=gini, max_features=4, n_estimators=100..........\n",
      "[CV 2/5; 14/32] END criterion=gini, max_features=4, n_estimators=100;, score=0.819 total time=   0.7s\n",
      "[CV 3/5; 14/32] START criterion=gini, max_features=4, n_estimators=100..........\n",
      "[CV 3/5; 14/32] END criterion=gini, max_features=4, n_estimators=100;, score=0.812 total time=   0.7s\n",
      "[CV 4/5; 14/32] START criterion=gini, max_features=4, n_estimators=100..........\n",
      "[CV 4/5; 14/32] END criterion=gini, max_features=4, n_estimators=100;, score=0.804 total time=   0.7s\n",
      "[CV 5/5; 14/32] START criterion=gini, max_features=4, n_estimators=100..........\n",
      "[CV 5/5; 14/32] END criterion=gini, max_features=4, n_estimators=100;, score=0.805 total time=   0.7s\n",
      "[CV 1/5; 15/32] START criterion=gini, max_features=4, n_estimators=150..........\n",
      "[CV 1/5; 15/32] END criterion=gini, max_features=4, n_estimators=150;, score=0.827 total time=   1.1s\n",
      "[CV 2/5; 15/32] START criterion=gini, max_features=4, n_estimators=150..........\n",
      "[CV 2/5; 15/32] END criterion=gini, max_features=4, n_estimators=150;, score=0.819 total time=   1.1s\n",
      "[CV 3/5; 15/32] START criterion=gini, max_features=4, n_estimators=150..........\n",
      "[CV 3/5; 15/32] END criterion=gini, max_features=4, n_estimators=150;, score=0.812 total time=   1.1s\n",
      "[CV 4/5; 15/32] START criterion=gini, max_features=4, n_estimators=150..........\n",
      "[CV 4/5; 15/32] END criterion=gini, max_features=4, n_estimators=150;, score=0.804 total time=   1.1s\n",
      "[CV 5/5; 15/32] START criterion=gini, max_features=4, n_estimators=150..........\n",
      "[CV 5/5; 15/32] END criterion=gini, max_features=4, n_estimators=150;, score=0.804 total time=   1.1s\n",
      "[CV 1/5; 16/32] START criterion=gini, max_features=4, n_estimators=200..........\n",
      "[CV 1/5; 16/32] END criterion=gini, max_features=4, n_estimators=200;, score=0.827 total time=   1.5s\n",
      "[CV 2/5; 16/32] START criterion=gini, max_features=4, n_estimators=200..........\n",
      "[CV 2/5; 16/32] END criterion=gini, max_features=4, n_estimators=200;, score=0.819 total time=   1.5s\n",
      "[CV 3/5; 16/32] START criterion=gini, max_features=4, n_estimators=200..........\n",
      "[CV 3/5; 16/32] END criterion=gini, max_features=4, n_estimators=200;, score=0.812 total time=   1.5s\n",
      "[CV 4/5; 16/32] START criterion=gini, max_features=4, n_estimators=200..........\n",
      "[CV 4/5; 16/32] END criterion=gini, max_features=4, n_estimators=200;, score=0.804 total time=   1.5s\n",
      "[CV 5/5; 16/32] START criterion=gini, max_features=4, n_estimators=200..........\n",
      "[CV 5/5; 16/32] END criterion=gini, max_features=4, n_estimators=200;, score=0.805 total time=   1.5s\n",
      "[CV 1/5; 17/32] START criterion=entropy, max_features=1, n_estimators=50........\n",
      "[CV 1/5; 17/32] END criterion=entropy, max_features=1, n_estimators=50;, score=0.827 total time=   0.2s\n",
      "[CV 2/5; 17/32] START criterion=entropy, max_features=1, n_estimators=50........\n",
      "[CV 2/5; 17/32] END criterion=entropy, max_features=1, n_estimators=50;, score=0.819 total time=   0.2s\n",
      "[CV 3/5; 17/32] START criterion=entropy, max_features=1, n_estimators=50........\n",
      "[CV 3/5; 17/32] END criterion=entropy, max_features=1, n_estimators=50;, score=0.813 total time=   0.2s\n",
      "[CV 4/5; 17/32] START criterion=entropy, max_features=1, n_estimators=50........\n",
      "[CV 4/5; 17/32] END criterion=entropy, max_features=1, n_estimators=50;, score=0.805 total time=   0.2s\n",
      "[CV 5/5; 17/32] START criterion=entropy, max_features=1, n_estimators=50........\n",
      "[CV 5/5; 17/32] END criterion=entropy, max_features=1, n_estimators=50;, score=0.805 total time=   0.2s\n",
      "[CV 1/5; 18/32] START criterion=entropy, max_features=1, n_estimators=100.......\n",
      "[CV 1/5; 18/32] END criterion=entropy, max_features=1, n_estimators=100;, score=0.827 total time=   0.5s\n",
      "[CV 2/5; 18/32] START criterion=entropy, max_features=1, n_estimators=100.......\n",
      "[CV 2/5; 18/32] END criterion=entropy, max_features=1, n_estimators=100;, score=0.820 total time=   0.5s\n",
      "[CV 3/5; 18/32] START criterion=entropy, max_features=1, n_estimators=100.......\n",
      "[CV 3/5; 18/32] END criterion=entropy, max_features=1, n_estimators=100;, score=0.813 total time=   0.5s\n",
      "[CV 4/5; 18/32] START criterion=entropy, max_features=1, n_estimators=100.......\n",
      "[CV 4/5; 18/32] END criterion=entropy, max_features=1, n_estimators=100;, score=0.804 total time=   0.5s\n",
      "[CV 5/5; 18/32] START criterion=entropy, max_features=1, n_estimators=100.......\n",
      "[CV 5/5; 18/32] END criterion=entropy, max_features=1, n_estimators=100;, score=0.806 total time=   0.5s\n",
      "[CV 1/5; 19/32] START criterion=entropy, max_features=1, n_estimators=150.......\n",
      "[CV 1/5; 19/32] END criterion=entropy, max_features=1, n_estimators=150;, score=0.827 total time=   0.9s\n",
      "[CV 2/5; 19/32] START criterion=entropy, max_features=1, n_estimators=150.......\n",
      "[CV 2/5; 19/32] END criterion=entropy, max_features=1, n_estimators=150;, score=0.820 total time=   0.8s\n",
      "[CV 3/5; 19/32] START criterion=entropy, max_features=1, n_estimators=150.......\n",
      "[CV 3/5; 19/32] END criterion=entropy, max_features=1, n_estimators=150;, score=0.813 total time=   0.8s\n",
      "[CV 4/5; 19/32] START criterion=entropy, max_features=1, n_estimators=150.......\n",
      "[CV 4/5; 19/32] END criterion=entropy, max_features=1, n_estimators=150;, score=0.804 total time=   0.8s\n",
      "[CV 5/5; 19/32] START criterion=entropy, max_features=1, n_estimators=150.......\n",
      "[CV 5/5; 19/32] END criterion=entropy, max_features=1, n_estimators=150;, score=0.805 total time=   0.9s\n",
      "[CV 1/5; 20/32] START criterion=entropy, max_features=1, n_estimators=200.......\n",
      "[CV 1/5; 20/32] END criterion=entropy, max_features=1, n_estimators=200;, score=0.827 total time=   1.2s\n",
      "[CV 2/5; 20/32] START criterion=entropy, max_features=1, n_estimators=200.......\n",
      "[CV 2/5; 20/32] END criterion=entropy, max_features=1, n_estimators=200;, score=0.819 total time=   1.2s\n",
      "[CV 3/5; 20/32] START criterion=entropy, max_features=1, n_estimators=200.......\n",
      "[CV 3/5; 20/32] END criterion=entropy, max_features=1, n_estimators=200;, score=0.813 total time=   1.2s\n",
      "[CV 4/5; 20/32] START criterion=entropy, max_features=1, n_estimators=200.......\n",
      "[CV 4/5; 20/32] END criterion=entropy, max_features=1, n_estimators=200;, score=0.805 total time=   1.2s\n",
      "[CV 5/5; 20/32] START criterion=entropy, max_features=1, n_estimators=200.......\n",
      "[CV 5/5; 20/32] END criterion=entropy, max_features=1, n_estimators=200;, score=0.806 total time=   1.2s\n",
      "[CV 1/5; 21/32] START criterion=entropy, max_features=2, n_estimators=50........\n",
      "[CV 1/5; 21/32] END criterion=entropy, max_features=2, n_estimators=50;, score=0.827 total time=   0.2s\n",
      "[CV 2/5; 21/32] START criterion=entropy, max_features=2, n_estimators=50........\n",
      "[CV 2/5; 21/32] END criterion=entropy, max_features=2, n_estimators=50;, score=0.819 total time=   0.2s\n",
      "[CV 3/5; 21/32] START criterion=entropy, max_features=2, n_estimators=50........\n",
      "[CV 3/5; 21/32] END criterion=entropy, max_features=2, n_estimators=50;, score=0.813 total time=   0.2s\n",
      "[CV 4/5; 21/32] START criterion=entropy, max_features=2, n_estimators=50........\n",
      "[CV 4/5; 21/32] END criterion=entropy, max_features=2, n_estimators=50;, score=0.804 total time=   0.2s\n",
      "[CV 5/5; 21/32] START criterion=entropy, max_features=2, n_estimators=50........\n",
      "[CV 5/5; 21/32] END criterion=entropy, max_features=2, n_estimators=50;, score=0.806 total time=   0.2s\n",
      "[CV 1/5; 22/32] START criterion=entropy, max_features=2, n_estimators=100.......\n",
      "[CV 1/5; 22/32] END criterion=entropy, max_features=2, n_estimators=100;, score=0.827 total time=   0.6s\n",
      "[CV 2/5; 22/32] START criterion=entropy, max_features=2, n_estimators=100.......\n",
      "[CV 2/5; 22/32] END criterion=entropy, max_features=2, n_estimators=100;, score=0.820 total time=   0.6s\n",
      "[CV 3/5; 22/32] START criterion=entropy, max_features=2, n_estimators=100.......\n",
      "[CV 3/5; 22/32] END criterion=entropy, max_features=2, n_estimators=100;, score=0.813 total time=   0.6s\n",
      "[CV 4/5; 22/32] START criterion=entropy, max_features=2, n_estimators=100.......\n",
      "[CV 4/5; 22/32] END criterion=entropy, max_features=2, n_estimators=100;, score=0.804 total time=   0.6s\n",
      "[CV 5/5; 22/32] START criterion=entropy, max_features=2, n_estimators=100.......\n",
      "[CV 5/5; 22/32] END criterion=entropy, max_features=2, n_estimators=100;, score=0.806 total time=   0.6s\n",
      "[CV 1/5; 23/32] START criterion=entropy, max_features=2, n_estimators=150.......\n",
      "[CV 1/5; 23/32] END criterion=entropy, max_features=2, n_estimators=150;, score=0.827 total time=   0.9s\n",
      "[CV 2/5; 23/32] START criterion=entropy, max_features=2, n_estimators=150.......\n",
      "[CV 2/5; 23/32] END criterion=entropy, max_features=2, n_estimators=150;, score=0.820 total time=   0.9s\n",
      "[CV 3/5; 23/32] START criterion=entropy, max_features=2, n_estimators=150.......\n",
      "[CV 3/5; 23/32] END criterion=entropy, max_features=2, n_estimators=150;, score=0.813 total time=   0.9s\n",
      "[CV 4/5; 23/32] START criterion=entropy, max_features=2, n_estimators=150.......\n",
      "[CV 4/5; 23/32] END criterion=entropy, max_features=2, n_estimators=150;, score=0.804 total time=   0.9s\n",
      "[CV 5/5; 23/32] START criterion=entropy, max_features=2, n_estimators=150.......\n",
      "[CV 5/5; 23/32] END criterion=entropy, max_features=2, n_estimators=150;, score=0.805 total time=   0.9s\n",
      "[CV 1/5; 24/32] START criterion=entropy, max_features=2, n_estimators=200.......\n",
      "[CV 1/5; 24/32] END criterion=entropy, max_features=2, n_estimators=200;, score=0.827 total time=   1.2s\n",
      "[CV 2/5; 24/32] START criterion=entropy, max_features=2, n_estimators=200.......\n",
      "[CV 2/5; 24/32] END criterion=entropy, max_features=2, n_estimators=200;, score=0.819 total time=   1.2s\n",
      "[CV 3/5; 24/32] START criterion=entropy, max_features=2, n_estimators=200.......\n",
      "[CV 3/5; 24/32] END criterion=entropy, max_features=2, n_estimators=200;, score=0.813 total time=   1.3s\n",
      "[CV 4/5; 24/32] START criterion=entropy, max_features=2, n_estimators=200.......\n",
      "[CV 4/5; 24/32] END criterion=entropy, max_features=2, n_estimators=200;, score=0.804 total time=   1.2s\n",
      "[CV 5/5; 24/32] START criterion=entropy, max_features=2, n_estimators=200.......\n",
      "[CV 5/5; 24/32] END criterion=entropy, max_features=2, n_estimators=200;, score=0.806 total time=   1.2s\n",
      "[CV 1/5; 25/32] START criterion=entropy, max_features=3, n_estimators=50........\n",
      "[CV 1/5; 25/32] END criterion=entropy, max_features=3, n_estimators=50;, score=0.827 total time=   0.3s\n",
      "[CV 2/5; 25/32] START criterion=entropy, max_features=3, n_estimators=50........\n",
      "[CV 2/5; 25/32] END criterion=entropy, max_features=3, n_estimators=50;, score=0.819 total time=   0.3s\n",
      "[CV 3/5; 25/32] START criterion=entropy, max_features=3, n_estimators=50........\n",
      "[CV 3/5; 25/32] END criterion=entropy, max_features=3, n_estimators=50;, score=0.812 total time=   0.3s\n",
      "[CV 4/5; 25/32] START criterion=entropy, max_features=3, n_estimators=50........\n",
      "[CV 4/5; 25/32] END criterion=entropy, max_features=3, n_estimators=50;, score=0.804 total time=   0.3s\n",
      "[CV 5/5; 25/32] START criterion=entropy, max_features=3, n_estimators=50........\n",
      "[CV 5/5; 25/32] END criterion=entropy, max_features=3, n_estimators=50;, score=0.806 total time=   0.3s\n",
      "[CV 1/5; 26/32] START criterion=entropy, max_features=3, n_estimators=100.......\n",
      "[CV 1/5; 26/32] END criterion=entropy, max_features=3, n_estimators=100;, score=0.827 total time=   0.6s\n",
      "[CV 2/5; 26/32] START criterion=entropy, max_features=3, n_estimators=100.......\n",
      "[CV 2/5; 26/32] END criterion=entropy, max_features=3, n_estimators=100;, score=0.820 total time=   0.6s\n",
      "[CV 3/5; 26/32] START criterion=entropy, max_features=3, n_estimators=100.......\n",
      "[CV 3/5; 26/32] END criterion=entropy, max_features=3, n_estimators=100;, score=0.813 total time=   0.6s\n",
      "[CV 4/5; 26/32] START criterion=entropy, max_features=3, n_estimators=100.......\n",
      "[CV 4/5; 26/32] END criterion=entropy, max_features=3, n_estimators=100;, score=0.804 total time=   0.6s\n",
      "[CV 5/5; 26/32] START criterion=entropy, max_features=3, n_estimators=100.......\n",
      "[CV 5/5; 26/32] END criterion=entropy, max_features=3, n_estimators=100;, score=0.805 total time=   0.6s\n",
      "[CV 1/5; 27/32] START criterion=entropy, max_features=3, n_estimators=150.......\n",
      "[CV 1/5; 27/32] END criterion=entropy, max_features=3, n_estimators=150;, score=0.828 total time=   1.0s\n",
      "[CV 2/5; 27/32] START criterion=entropy, max_features=3, n_estimators=150.......\n",
      "[CV 2/5; 27/32] END criterion=entropy, max_features=3, n_estimators=150;, score=0.820 total time=   1.0s\n",
      "[CV 3/5; 27/32] START criterion=entropy, max_features=3, n_estimators=150.......\n",
      "[CV 3/5; 27/32] END criterion=entropy, max_features=3, n_estimators=150;, score=0.813 total time=   1.0s\n",
      "[CV 4/5; 27/32] START criterion=entropy, max_features=3, n_estimators=150.......\n",
      "[CV 4/5; 27/32] END criterion=entropy, max_features=3, n_estimators=150;, score=0.804 total time=   1.0s\n",
      "[CV 5/5; 27/32] START criterion=entropy, max_features=3, n_estimators=150.......\n",
      "[CV 5/5; 27/32] END criterion=entropy, max_features=3, n_estimators=150;, score=0.805 total time=   1.0s\n",
      "[CV 1/5; 28/32] START criterion=entropy, max_features=3, n_estimators=200.......\n",
      "[CV 1/5; 28/32] END criterion=entropy, max_features=3, n_estimators=200;, score=0.827 total time=   1.4s\n",
      "[CV 2/5; 28/32] START criterion=entropy, max_features=3, n_estimators=200.......\n",
      "[CV 2/5; 28/32] END criterion=entropy, max_features=3, n_estimators=200;, score=0.819 total time=   1.4s\n",
      "[CV 3/5; 28/32] START criterion=entropy, max_features=3, n_estimators=200.......\n",
      "[CV 3/5; 28/32] END criterion=entropy, max_features=3, n_estimators=200;, score=0.813 total time=   1.4s\n",
      "[CV 4/5; 28/32] START criterion=entropy, max_features=3, n_estimators=200.......\n",
      "[CV 4/5; 28/32] END criterion=entropy, max_features=3, n_estimators=200;, score=0.804 total time=   1.3s\n",
      "[CV 5/5; 28/32] START criterion=entropy, max_features=3, n_estimators=200.......\n",
      "[CV 5/5; 28/32] END criterion=entropy, max_features=3, n_estimators=200;, score=0.806 total time=   1.4s\n",
      "[CV 1/5; 29/32] START criterion=entropy, max_features=4, n_estimators=50........\n",
      "[CV 1/5; 29/32] END criterion=entropy, max_features=4, n_estimators=50;, score=0.827 total time=   0.3s\n",
      "[CV 2/5; 29/32] START criterion=entropy, max_features=4, n_estimators=50........\n",
      "[CV 2/5; 29/32] END criterion=entropy, max_features=4, n_estimators=50;, score=0.819 total time=   0.3s\n",
      "[CV 3/5; 29/32] START criterion=entropy, max_features=4, n_estimators=50........\n",
      "[CV 3/5; 29/32] END criterion=entropy, max_features=4, n_estimators=50;, score=0.812 total time=   0.3s\n",
      "[CV 4/5; 29/32] START criterion=entropy, max_features=4, n_estimators=50........\n",
      "[CV 4/5; 29/32] END criterion=entropy, max_features=4, n_estimators=50;, score=0.804 total time=   0.3s\n",
      "[CV 5/5; 29/32] START criterion=entropy, max_features=4, n_estimators=50........\n",
      "[CV 5/5; 29/32] END criterion=entropy, max_features=4, n_estimators=50;, score=0.805 total time=   0.3s\n",
      "[CV 1/5; 30/32] START criterion=entropy, max_features=4, n_estimators=100.......\n",
      "[CV 1/5; 30/32] END criterion=entropy, max_features=4, n_estimators=100;, score=0.827 total time=   0.7s\n",
      "[CV 2/5; 30/32] START criterion=entropy, max_features=4, n_estimators=100.......\n",
      "[CV 2/5; 30/32] END criterion=entropy, max_features=4, n_estimators=100;, score=0.820 total time=   0.7s\n",
      "[CV 3/5; 30/32] START criterion=entropy, max_features=4, n_estimators=100.......\n",
      "[CV 3/5; 30/32] END criterion=entropy, max_features=4, n_estimators=100;, score=0.812 total time=   0.7s\n",
      "[CV 4/5; 30/32] START criterion=entropy, max_features=4, n_estimators=100.......\n",
      "[CV 4/5; 30/32] END criterion=entropy, max_features=4, n_estimators=100;, score=0.804 total time=   0.7s\n",
      "[CV 5/5; 30/32] START criterion=entropy, max_features=4, n_estimators=100.......\n",
      "[CV 5/5; 30/32] END criterion=entropy, max_features=4, n_estimators=100;, score=0.805 total time=   0.7s\n",
      "[CV 1/5; 31/32] START criterion=entropy, max_features=4, n_estimators=150.......\n",
      "[CV 1/5; 31/32] END criterion=entropy, max_features=4, n_estimators=150;, score=0.827 total time=   1.1s\n",
      "[CV 2/5; 31/32] START criterion=entropy, max_features=4, n_estimators=150.......\n",
      "[CV 2/5; 31/32] END criterion=entropy, max_features=4, n_estimators=150;, score=0.820 total time=   1.1s\n",
      "[CV 3/5; 31/32] START criterion=entropy, max_features=4, n_estimators=150.......\n",
      "[CV 3/5; 31/32] END criterion=entropy, max_features=4, n_estimators=150;, score=0.812 total time=   1.1s\n",
      "[CV 4/5; 31/32] START criterion=entropy, max_features=4, n_estimators=150.......\n",
      "[CV 4/5; 31/32] END criterion=entropy, max_features=4, n_estimators=150;, score=0.804 total time=   1.1s\n",
      "[CV 5/5; 31/32] START criterion=entropy, max_features=4, n_estimators=150.......\n",
      "[CV 5/5; 31/32] END criterion=entropy, max_features=4, n_estimators=150;, score=0.804 total time=   1.1s\n",
      "[CV 1/5; 32/32] START criterion=entropy, max_features=4, n_estimators=200.......\n",
      "[CV 1/5; 32/32] END criterion=entropy, max_features=4, n_estimators=200;, score=0.827 total time=   1.5s\n",
      "[CV 2/5; 32/32] START criterion=entropy, max_features=4, n_estimators=200.......\n",
      "[CV 2/5; 32/32] END criterion=entropy, max_features=4, n_estimators=200;, score=0.819 total time=   1.5s\n",
      "[CV 3/5; 32/32] START criterion=entropy, max_features=4, n_estimators=200.......\n",
      "[CV 3/5; 32/32] END criterion=entropy, max_features=4, n_estimators=200;, score=0.812 total time=   1.5s\n",
      "[CV 4/5; 32/32] START criterion=entropy, max_features=4, n_estimators=200.......\n",
      "[CV 4/5; 32/32] END criterion=entropy, max_features=4, n_estimators=200;, score=0.804 total time=   1.5s\n",
      "[CV 5/5; 32/32] START criterion=entropy, max_features=4, n_estimators=200.......\n",
      "[CV 5/5; 32/32] END criterion=entropy, max_features=4, n_estimators=200;, score=0.805 total time=   1.5s\n",
      "Random Forest GridSearchCV best score = 0.8139897700097469\n",
      "Random Forest GridSearchCV best parameters = {'criterion': 'entropy', 'max_features': 2, 'n_estimators': 50}\n",
      "Random Forest GridSearchCV deceased class f1-score = 0.7382888114595433\n",
      "Random Forest GridSearchCV hospitalized class f1-score = 0.9305594920922852\n",
      "Random Forest GridSearchCV nonhospitalized class f1-score = 0.7962926955732712\n",
      "Random Forest GridSearchCV accuracy score = 0.8265524625267666\n"
     ]
    }
   ],
   "source": [
    "# Takes about 3-4 minutes to run.\n",
    "\n",
    "# Decide number of k-fold splits\n",
    "k = 5\n",
    "# Create model with blank parameters\n",
    "rf_model = RandomForestClassifier(random_state = 44)\n",
    "# Create space of possible parameters\n",
    "parameter_search_space = {\n",
    "    'n_estimators': [50, 100, 150, 200],\n",
    "    'criterion': [\"gini\", \"entropy\"],\n",
    "    'max_features': [1, 2, 3, 4]\n",
    "  }\n",
    "# Create grid search cross validation object\n",
    "grid_search_cv = GridSearchCV(\n",
    "    estimator=rf_model,\n",
    "    param_grid=parameter_search_space,\n",
    "    scoring=\"f1_macro\",\n",
    "    cv=k,\n",
    "    verbose=10\n",
    ")\n",
    "# Put data and labels in proper format\n",
    "data = train_data.iloc[:, :4].values\n",
    "labels = train_data.iloc[:, 4].values.ravel()\n",
    "# Fit grid search object\n",
    "grid_search_cv.fit(data, labels)\n",
    "# Print and save results.\n",
    "print(\"Random Forest GridSearchCV best score = \" + str(grid_search_cv.best_score_))\n",
    "print(\"Random Forest GridSearchCV best parameters = \" + str(grid_search_cv.best_params_))\n",
    "predictions = grid_search_cv.predict(data)\n",
    "_, _, fscore, _ = precision_recall_fscore_support(predictions, labels)\n",
    "print(\"Random Forest GridSearchCV deceased class f1-score = \" + str(fscore[0]))\n",
    "print(\"Random Forest GridSearchCV hospitalized class f1-score = \" + str(fscore[1]))\n",
    "print(\"Random Forest GridSearchCV nonhospitalized class f1-score = \" + str(fscore[2]))\n",
    "accuracy = accuracy_score(predictions, labels)\n",
    "print(\"Random Forest GridSearchCV accuracy score = \" + str(accuracy))\n",
    "pd.DataFrame(grid_search_cv.cv_results_).to_csv(\"rf_results.csv\")\n",
    "rf_model = grid_search_cv.best_estimator_"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 32 candidates, totalling 160 fits\n",
      "[CV 1/5; 1/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 1/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.696 total time=   3.4s\n",
      "[CV 2/5; 1/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 1/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.612 total time=   2.3s\n",
      "[CV 3/5; 1/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 1/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.745 total time=   4.4s\n",
      "[CV 4/5; 1/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 1/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.648 total time=   4.2s\n",
      "[CV 5/5; 1/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 1/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.541 total time=   1.8s\n",
      "[CV 1/5; 2/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 2/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.782 total time=   6.1s\n",
      "[CV 2/5; 2/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 2/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.759 total time=  10.2s\n",
      "[CV 3/5; 2/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 2/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.773 total time=   7.8s\n",
      "[CV 4/5; 2/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 2/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.755 total time=   7.5s\n",
      "[CV 5/5; 2/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 2/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.758 total time=   9.1s\n",
      "[CV 1/5; 3/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 3/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.781 total time=   9.4s\n",
      "[CV 2/5; 3/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 3/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.676 total time=   9.5s\n",
      "[CV 3/5; 3/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 3/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.759 total time=   9.5s\n",
      "[CV 4/5; 3/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 3/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.756 total time=   9.6s\n",
      "[CV 5/5; 3/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 3/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.576 total time=   9.5s\n",
      "[CV 1/5; 4/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 4/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.784 total time=   6.3s\n",
      "[CV 2/5; 4/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 4/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.776 total time=   7.9s\n",
      "[CV 3/5; 4/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 4/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.771 total time=  10.3s\n",
      "[CV 4/5; 4/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 4/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.757 total time=  10.3s\n",
      "[CV 5/5; 4/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 4/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.755 total time=   7.6s\n",
      "[CV 1/5; 5/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 5/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.742 total time=   3.3s\n",
      "[CV 2/5; 5/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 5/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.728 total time=   1.9s\n",
      "[CV 3/5; 5/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 5/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.724 total time=   3.4s\n",
      "[CV 4/5; 5/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 5/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.719 total time=   3.8s\n",
      "[CV 5/5; 5/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 5/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.727 total time=   2.7s\n",
      "[CV 1/5; 6/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 6/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.785 total time=   2.7s\n",
      "[CV 2/5; 6/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 6/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.773 total time=   2.7s\n",
      "[CV 3/5; 6/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 6/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.772 total time=   3.0s\n",
      "[CV 4/5; 6/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 6/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.765 total time=   3.6s\n",
      "[CV 5/5; 6/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 6/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.760 total time=   4.7s\n",
      "[CV 1/5; 7/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 7/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.785 total time=   5.3s\n",
      "[CV 2/5; 7/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 7/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.773 total time=   5.4s\n",
      "[CV 3/5; 7/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 7/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.770 total time=   5.3s\n",
      "[CV 4/5; 7/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n",
      "[CV 4/5; 7/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.750 total time=   4.2s\n",
      "[CV 5/5; 7/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 7/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.751 total time=   5.3s\n",
      "[CV 1/5; 8/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 8/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.787 total time=   4.7s\n",
      "[CV 2/5; 8/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 8/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.769 total time=   5.7s\n",
      "[CV 3/5; 8/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 3/5; 8/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.776 total time=   5.4s\n",
      "[CV 4/5; 8/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 8/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.750 total time=   3.7s\n",
      "[CV 5/5; 8/32] START activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 8/32] END activation=tanh, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.762 total time=   3.5s\n",
      "[CV 1/5; 9/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 9/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.733 total time=   5.3s\n",
      "[CV 2/5; 9/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 9/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.771 total time=   9.3s\n",
      "[CV 3/5; 9/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 9/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.611 total time=   3.4s\n",
      "[CV 4/5; 9/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 9/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.666 total time=   4.9s\n",
      "[CV 5/5; 9/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 9/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.177 total time=   1.7s\n",
      "[CV 1/5; 10/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 10/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.782 total time=   8.5s\n",
      "[CV 2/5; 10/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 10/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.777 total time=   6.5s\n",
      "[CV 3/5; 10/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 10/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.777 total time=   5.5s\n",
      "[CV 4/5; 10/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 10/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.757 total time=   2.8s\n",
      "[CV 5/5; 10/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 10/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.756 total time=   7.4s\n",
      "[CV 1/5; 11/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 11/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.559 total time=   9.4s\n",
      "[CV 2/5; 11/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 11/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.773 total time=   9.5s\n",
      "[CV 3/5; 11/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n",
      "[CV 3/5; 11/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.679 total time=   5.9s\n",
      "[CV 4/5; 11/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 11/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.760 total time=   9.5s\n",
      "[CV 5/5; 11/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n",
      "[CV 5/5; 11/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.747 total time=   8.9s\n",
      "[CV 1/5; 12/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 12/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.774 total time=   4.2s\n",
      "[CV 2/5; 12/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 12/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.762 total time=   7.4s\n",
      "[CV 3/5; 12/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 3/5; 12/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.775 total time=   7.4s\n",
      "[CV 4/5; 12/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 12/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.755 total time=   7.6s\n",
      "[CV 5/5; 12/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 12/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.760 total time=   7.3s\n",
      "[CV 1/5; 13/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 13/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.750 total time=   3.9s\n",
      "[CV 2/5; 13/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 13/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.776 total time=   4.6s\n",
      "[CV 3/5; 13/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 13/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.743 total time=   2.6s\n",
      "[CV 4/5; 13/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 13/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.736 total time=   4.3s\n",
      "[CV 5/5; 13/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 13/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.758 total time=   5.3s\n",
      "[CV 1/5; 14/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 14/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.784 total time=   1.9s\n",
      "[CV 2/5; 14/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 14/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.774 total time=   2.3s\n",
      "[CV 3/5; 14/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 14/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.767 total time=   4.0s\n",
      "[CV 4/5; 14/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 14/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.757 total time=   3.6s\n",
      "[CV 5/5; 14/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 14/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.756 total time=   3.6s\n",
      "[CV 1/5; 15/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 15/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.781 total time=   5.3s\n",
      "[CV 2/5; 15/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 15/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.771 total time=   5.3s\n",
      "[CV 3/5; 15/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 15/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.774 total time=   5.3s\n",
      "[CV 4/5; 15/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n",
      "[CV 4/5; 15/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.760 total time=   5.3s\n",
      "[CV 5/5; 15/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 15/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.753 total time=   5.4s\n",
      "[CV 1/5; 16/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 16/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.789 total time=   3.7s\n",
      "[CV 2/5; 16/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 16/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.778 total time=   4.5s\n",
      "[CV 3/5; 16/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 16/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.773 total time=   5.8s\n",
      "[CV 4/5; 16/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 16/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.754 total time=   2.9s\n",
      "[CV 5/5; 16/32] START activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 16/32] END activation=tanh, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.760 total time=   3.5s\n",
      "[CV 1/5; 17/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 17/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.760 total time=   7.7s\n",
      "[CV 2/5; 17/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 17/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.745 total time=   7.5s\n",
      "[CV 3/5; 17/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 17/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.755 total time=   4.4s\n",
      "[CV 4/5; 17/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 17/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.755 total time=   4.9s\n",
      "[CV 5/5; 17/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 17/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.752 total time=   3.3s\n",
      "[CV 1/5; 18/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 18/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.790 total time=   4.9s\n",
      "[CV 2/5; 18/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 18/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.755 total time=   3.4s\n",
      "[CV 3/5; 18/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 18/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.773 total time=   3.9s\n",
      "[CV 4/5; 18/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 18/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.761 total time=   3.0s\n",
      "[CV 5/5; 18/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 18/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.753 total time=   1.6s\n",
      "[CV 1/5; 19/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 19/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.782 total time=   7.9s\n",
      "[CV 2/5; 19/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 19/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.771 total time=   7.9s\n",
      "[CV 3/5; 19/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n",
      "[CV 3/5; 19/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.174 total time=   2.7s\n",
      "[CV 4/5; 19/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 19/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.757 total time=   7.9s\n",
      "[CV 5/5; 19/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 19/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.752 total time=   7.9s\n",
      "[CV 1/5; 20/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 20/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.789 total time=   5.2s\n",
      "[CV 2/5; 20/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 20/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.769 total time=   3.2s\n",
      "[CV 3/5; 20/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 3/5; 20/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.768 total time=   2.6s\n",
      "[CV 4/5; 20/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 20/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.767 total time=   4.2s\n",
      "[CV 5/5; 20/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 20/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.764 total time=   5.3s\n",
      "[CV 1/5; 21/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 21/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.737 total time=   3.3s\n",
      "[CV 2/5; 21/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 21/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.762 total time=   2.3s\n",
      "[CV 3/5; 21/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 21/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.734 total time=   3.2s\n",
      "[CV 4/5; 21/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 21/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.655 total time=   2.9s\n",
      "[CV 5/5; 21/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 21/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.699 total time=   2.3s\n",
      "[CV 1/5; 22/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 22/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.749 total time=   2.0s\n",
      "[CV 2/5; 22/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 22/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.768 total time=   3.0s\n",
      "[CV 3/5; 22/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 22/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.755 total time=   2.6s\n",
      "[CV 4/5; 22/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 22/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.758 total time=   2.4s\n",
      "[CV 5/5; 22/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 22/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.758 total time=   2.7s\n",
      "[CV 1/5; 23/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 23/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.782 total time=   4.7s\n",
      "[CV 2/5; 23/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n",
      "[CV 2/5; 23/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.768 total time=   4.2s\n",
      "[CV 3/5; 23/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 23/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.769 total time=   4.6s\n",
      "[CV 4/5; 23/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 23/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.754 total time=   4.7s\n",
      "[CV 5/5; 23/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 23/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.753 total time=   4.5s\n",
      "[CV 1/5; 24/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 24/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.767 total time=   2.2s\n",
      "[CV 2/5; 24/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 24/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.735 total time=   2.5s\n",
      "[CV 3/5; 24/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 3/5; 24/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.760 total time=   3.0s\n",
      "[CV 4/5; 24/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 24/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.754 total time=   1.7s\n",
      "[CV 5/5; 24/32] START activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 24/32] END activation=relu, alpha=0.0001, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.738 total time=   2.2s\n",
      "[CV 1/5; 25/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 25/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.777 total time=   4.1s\n",
      "[CV 2/5; 25/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 25/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.731 total time=   5.6s\n",
      "[CV 3/5; 25/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 25/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.745 total time=   2.8s\n",
      "[CV 4/5; 25/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 25/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.740 total time=   4.8s\n",
      "[CV 5/5; 25/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 25/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=sgd;, score=0.758 total time=   5.6s\n",
      "[CV 1/5; 26/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 26/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.780 total time=   2.7s\n",
      "[CV 2/5; 26/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 26/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.768 total time=   3.0s\n",
      "[CV 3/5; 26/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 26/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.755 total time=   4.1s\n",
      "[CV 4/5; 26/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 26/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.759 total time=   2.9s\n",
      "[CV 5/5; 26/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 26/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=constant, solver=adam;, score=0.761 total time=   4.2s\n",
      "[CV 1/5; 27/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 27/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.784 total time=   8.0s\n",
      "[CV 2/5; 27/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n",
      "[CV 2/5; 27/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.768 total time=   7.9s\n",
      "[CV 3/5; 27/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 27/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.773 total time=   8.3s\n",
      "[CV 4/5; 27/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n",
      "[CV 4/5; 27/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.199 total time=   3.4s\n",
      "[CV 5/5; 27/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 27/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=sgd;, score=0.759 total time=   7.9s\n",
      "[CV 1/5; 28/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 28/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.778 total time=   2.9s\n",
      "[CV 2/5; 28/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 28/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.753 total time=   4.2s\n",
      "[CV 3/5; 28/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 3/5; 28/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.756 total time=   3.2s\n",
      "[CV 4/5; 28/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 28/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.749 total time=   4.9s\n",
      "[CV 5/5; 28/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 28/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(10, 30, 10), learning_rate=adaptive, solver=adam;, score=0.761 total time=   3.6s\n",
      "[CV 1/5; 29/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 1/5; 29/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.673 total time=   2.9s\n",
      "[CV 2/5; 29/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 2/5; 29/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.737 total time=   3.6s\n",
      "[CV 3/5; 29/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 3/5; 29/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.744 total time=   1.1s\n",
      "[CV 4/5; 29/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 4/5; 29/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.742 total time=   4.7s\n",
      "[CV 5/5; 29/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd\n",
      "[CV 5/5; 29/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=sgd;, score=0.752 total time=   3.2s\n",
      "[CV 1/5; 30/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 1/5; 30/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.770 total time=   2.6s\n",
      "[CV 2/5; 30/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 2/5; 30/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.773 total time=   2.2s\n",
      "[CV 3/5; 30/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 3/5; 30/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.773 total time=   1.5s\n",
      "[CV 4/5; 30/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 4/5; 30/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.742 total time=   2.7s\n",
      "[CV 5/5; 30/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam\n",
      "[CV 5/5; 30/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=constant, solver=adam;, score=0.755 total time=   2.3s\n",
      "[CV 1/5; 31/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n",
      "[CV 1/5; 31/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.782 total time=   4.4s\n",
      "[CV 2/5; 31/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 31/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.770 total time=   4.7s\n",
      "[CV 3/5; 31/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n",
      "[CV 3/5; 31/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.758 total time=   4.0s\n",
      "[CV 4/5; 31/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n",
      "[CV 4/5; 31/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.750 total time=   4.4s\n",
      "[CV 5/5; 31/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Gatsby\\miniconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:679: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 31/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=sgd;, score=0.748 total time=   4.6s\n",
      "[CV 1/5; 32/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 1/5; 32/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.784 total time=   1.4s\n",
      "[CV 2/5; 32/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 2/5; 32/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.762 total time=   2.7s\n",
      "[CV 3/5; 32/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 3/5; 32/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.773 total time=   2.4s\n",
      "[CV 4/5; 32/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 4/5; 32/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.757 total time=   1.2s\n",
      "[CV 5/5; 32/32] START activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam\n",
      "[CV 5/5; 32/32] END activation=relu, alpha=0.05, hidden_layer_sizes=(20,), learning_rate=adaptive, solver=adam;, score=0.756 total time=   1.9s\n",
      "MLP Classifier GridSearchCV best score = 0.7711061646309254\n",
      "MLP Classifier GridSearchCV best parameters = {'activation': 'relu', 'alpha': 0.0001, 'hidden_layer_sizes': (10, 30, 10), 'learning_rate': 'adaptive', 'solver': 'adam'}\n",
      "MLP GridSearchCV deceased class f1-score = 0.6660924870809594\n",
      "MLP GridSearchCV hospitalized class f1-score = 0.8769386378961563\n",
      "MLP GridSearchCV nonhospitalized class f1-score = 0.7860696517412936\n",
      "MLP Classifier GridSearchCV accuracy score = 0.7820787349695273\n"
     ]
    }
   ],
   "source": [
    "# Takes about 12 minutes to run.\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "# x_train = train_data[['age', 'country', 'chronic_disease_binary', 'Case_Fatality_Ratio']]\n",
    "# y_train = train_data[['outcome_group']]\n",
    "mlp_gs = MLPClassifier()\n",
    "parameter_space = {\n",
    "    'hidden_layer_sizes': [(10,30,10),(20,)],\n",
    "    'activation': ['tanh', 'relu'],\n",
    "    'solver': ['sgd', 'adam'],\n",
    "    'alpha': [0.0001, 0.05],\n",
    "    'learning_rate': ['constant','adaptive'],\n",
    "}\n",
    "\n",
    "data = train_data.iloc[:, :4].values\n",
    "labels = train_data.iloc[:, 4].values.reshape(-1, 1)\n",
    "grid_search_cv = GridSearchCV(\n",
    "    estimator=mlp_gs,\n",
    "    param_grid=parameter_space,\n",
    "    scoring=\"f1_macro\",\n",
    "    cv=5,\n",
    "    verbose=10\n",
    ")\n",
    "grid_search_cv.fit(data,labels.ravel())\n",
    "\n",
    "print(\"MLP Classifier GridSearchCV best score = \" + str(grid_search_cv.best_score_))\n",
    "print(\"MLP Classifier GridSearchCV best parameters = \" + str(grid_search_cv.best_params_))\n",
    "predictions = grid_search_cv.predict(data)\n",
    "_, _, fscore, _ = precision_recall_fscore_support(predictions, labels)\n",
    "print(\"MLP GridSearchCV deceased class f1-score = \" + str(fscore[0]))\n",
    "print(\"MLP GridSearchCV hospitalized class f1-score = \" + str(fscore[1]))\n",
    "print(\"MLP GridSearchCV nonhospitalized class f1-score = \" + str(fscore[2]))\n",
    "accuracy = accuracy_score(predictions, labels)\n",
    "print(\"MLP Classifier GridSearchCV accuracy score = \" + str(accuracy))\n",
    "pd.DataFrame(grid_search_cv.cv_results_).to_csv(\"MLP Classifier_results.csv\")\n",
    "mlp_gs = grid_search_cv.best_estimator_\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving Hyper Parameter Results As Text Files"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom sklearn.model_selection import KFold\\ndata = train_data.iloc[:, :4].values\\nlabels = train_data.iloc[:, 4].values.ravel()\\nrf_model = RandomForestClassifier(random_state = 44)\\nkf = KFold(n_splits=k)\\nn_estimators = [50, 100, 150, 200]\\ncriterion = [\"gini\", \"entropy\"]\\nmax_features = [1, 2, 3, 4]\\ntext_file = open(\"randomforest_tuning.txt\", \"w\")\\nfor c in criterion:\\n    for n in n_estimators:\\n        for m in max_features:\\n            rf_model.set_params(criterion = c, n_estimators = n, max_features = m)\\n            deceased_f1score = 0\\n            mean_macro_f1score = 0\\n            accuracy = 0\\n            for i, (train_index, test_index) in enumerate(kf.split(data)):\\n                train_fold_data = np.take(data, train_index, 0)\\n                train_fold_labels = np.take(labels, train_index, 0)\\n                test_fold_data = np.take(data, test_index, 0)\\n                test_fold_labels = np.take(labels, test_index, 0)\\n                rf_model.fit(train_fold_data, train_fold_labels)\\n                predictions = rf_model.predict(test_fold_data)\\n                _, _, fscore, _ = precision_recall_fscore_support(predictions, test_fold_labels)\\n                deceased_f1score += fscore[0]\\n                mean_macro_f1score += ( fscore[0] + fscore[1] + fscore[2] ) / 3\\n                accuracy += accuracy_score(predictions, test_fold_labels)\\n            deceased_f1score = deceased_f1score / k\\n            mean_macro_f1score = mean_macro_f1score / k\\n            accuracy = accuracy / k\\n            output = \"\\ncriterion=\"+str(c)+\",n_estimators=\"+str(n)+\",max_features=\"+str(m)+\" -> mean_macro_f1_score=\"+str(mean_macro_f1score)+\",deceased_f1score=\"+str(deceased_f1score)+\",accuracy=\"+str(accuracy)\\n            print(output)\\n            n = text_file.write(output)\\ntext_file.close()\\n'"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# UNCOMMENT TO REPRODUCE randomforest_tuning.txt FILE\n",
    "\"\"\"\n",
    "from sklearn.model_selection import KFold\n",
    "data = train_data.iloc[:, :4].values\n",
    "labels = train_data.iloc[:, 4].values.ravel()\n",
    "rf_model = RandomForestClassifier(random_state = 44)\n",
    "kf = KFold(n_splits=k)\n",
    "n_estimators = [50, 100, 150, 200]\n",
    "criterion = [\"gini\", \"entropy\"]\n",
    "max_features = [1, 2, 3, 4]\n",
    "text_file = open(\"randomforest_tuning.txt\", \"w\")\n",
    "for c in criterion:\n",
    "    for n in n_estimators:\n",
    "        for m in max_features:\n",
    "            rf_model.set_params(criterion = c, n_estimators = n, max_features = m)\n",
    "            deceased_f1score = 0\n",
    "            mean_macro_f1score = 0\n",
    "            accuracy = 0\n",
    "            for i, (train_index, test_index) in enumerate(kf.split(data)):\n",
    "                train_fold_data = np.take(data, train_index, 0)\n",
    "                train_fold_labels = np.take(labels, train_index, 0)\n",
    "                test_fold_data = np.take(data, test_index, 0)\n",
    "                test_fold_labels = np.take(labels, test_index, 0)\n",
    "                rf_model.fit(train_fold_data, train_fold_labels)\n",
    "                predictions = rf_model.predict(test_fold_data)\n",
    "                _, _, fscore, _ = precision_recall_fscore_support(predictions, test_fold_labels)\n",
    "                deceased_f1score += fscore[0]\n",
    "                mean_macro_f1score += ( fscore[0] + fscore[1] + fscore[2] ) / 3\n",
    "                accuracy += accuracy_score(predictions, test_fold_labels)\n",
    "            deceased_f1score = deceased_f1score / k\n",
    "            mean_macro_f1score = mean_macro_f1score / k\n",
    "            accuracy = accuracy / k\n",
    "            output = \"\\ncriterion=\"+str(c)+\",n_estimators=\"+str(n)+\",max_features=\"+str(m)+\" -> mean_macro_f1_score=\"+str(mean_macro_f1score)+\",deceased_f1score=\"+str(deceased_f1score)+\",accuracy=\"+str(accuracy)\n",
    "            print(output)\n",
    "            n = text_file.write(output)\n",
    "text_file.close()\n",
    "\"\"\"\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom sklearn.model_selection import KFold\\nxgb_model_clone = xgb.XGBClassifier(random_state = 1)\\nkf = KFold(n_splits=k)\\nlearning_rate = [0.2, 0.3]\\nmax_depth = [6, 8, 10]\\nn_estimators = [150, 250]\\nobjective = \"multi:softmax\"\\nnum_class = 3\\nxgb_model_clone.set_params(objective = \"multi:softmax\", num_class = 3)\\ntext_file = open(\"xgboost_tuning.txt\", \"w\")\\nfor l in learning_rate:\\n    for n in n_estimators:\\n        for d in max_depth:\\n            xgb_model_clone.set_params(learning_rate = l, n_estimators = n, max_depth=d)\\n            deceased_f1score = 0\\n            mean_macro_f1score = 0\\n            accuracy = 0\\n            for i, (train_index, test_index) in enumerate(kf.split(data)):\\n                train_fold_data = np.take(data, train_index, 0)\\n                train_fold_labels = np.take(labels, train_index, 0)\\n                test_fold_data = np.take(data, test_index, 0)\\n                test_fold_labels = np.take(labels, test_index, 0)\\n                xgb_model_clone.fit(train_fold_data, train_fold_labels)\\n                predictions = xgb_model_clone.predict(test_fold_data)\\n                _, _, fscore, _ = precision_recall_fscore_support(predictions, test_fold_labels)\\n                deceased_f1score += fscore[0]\\n                mean_macro_f1score += ( fscore[0] + fscore[1] + fscore[2] ) / 3\\n                accuracy += accuracy_score(predictions, test_fold_labels)\\n            deceased_f1score = deceased_f1score / k\\n            mean_macro_f1score = mean_macro_f1score / k\\n            accuracy = accuracy / k\\n            output = \"\\nlearning_rate=\"+str(l)+\",n_estimators=\"+str(n)+\",max_depth=\"+str(d)+\" -> mean_macro_f1_score=\"+str(mean_macro_f1score)+\",deceased_f1score=\"+str(deceased_f1score)+\",accuracy=\"+str(accuracy)\\n            print(output)\\n            n = text_file.write(output)\\ntext_file.close()\\n'"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# UNCOMMENT TO REPRODUCE xgboost_tuning.txt FILE\n",
    "\"\"\"\n",
    "from sklearn.model_selection import KFold\n",
    "xgb_model_clone = xgb.XGBClassifier(random_state = 1)\n",
    "kf = KFold(n_splits=k)\n",
    "learning_rate = [0.2, 0.3]\n",
    "max_depth = [6, 8, 10]\n",
    "n_estimators = [150, 250]\n",
    "objective = \"multi:softmax\"\n",
    "num_class = 3\n",
    "xgb_model_clone.set_params(objective = \"multi:softmax\", num_class = 3)\n",
    "text_file = open(\"xgboost_tuning.txt\", \"w\")\n",
    "for l in learning_rate:\n",
    "    for n in n_estimators:\n",
    "        for d in max_depth:\n",
    "            xgb_model_clone.set_params(learning_rate = l, n_estimators = n, max_depth=d)\n",
    "            deceased_f1score = 0\n",
    "            mean_macro_f1score = 0\n",
    "            accuracy = 0\n",
    "            for i, (train_index, test_index) in enumerate(kf.split(data)):\n",
    "                train_fold_data = np.take(data, train_index, 0)\n",
    "                train_fold_labels = np.take(labels, train_index, 0)\n",
    "                test_fold_data = np.take(data, test_index, 0)\n",
    "                test_fold_labels = np.take(labels, test_index, 0)\n",
    "                xgb_model_clone.fit(train_fold_data, train_fold_labels)\n",
    "                predictions = xgb_model_clone.predict(test_fold_data)\n",
    "                _, _, fscore, _ = precision_recall_fscore_support(predictions, test_fold_labels)\n",
    "                deceased_f1score += fscore[0]\n",
    "                mean_macro_f1score += ( fscore[0] + fscore[1] + fscore[2] ) / 3\n",
    "                accuracy += accuracy_score(predictions, test_fold_labels)\n",
    "            deceased_f1score = deceased_f1score / k\n",
    "            mean_macro_f1score = mean_macro_f1score / k\n",
    "            accuracy = accuracy / k\n",
    "            output = \"\\nlearning_rate=\"+str(l)+\",n_estimators=\"+str(n)+\",max_depth=\"+str(d)+\" -> mean_macro_f1_score=\"+str(mean_macro_f1score)+\",deceased_f1score=\"+str(deceased_f1score)+\",accuracy=\"+str(accuracy)\n",
    "            print(output)\n",
    "            n = text_file.write(output)\n",
    "text_file.close()\n",
    "\"\"\"\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfrom sklearn.model_selection import KFold\\ndata = train_data.iloc[:, :4].values\\nlabels = train_data.iloc[:, 4].values.ravel()\\nmlp_gs = MLPClassifier()\\nhidden_layer_sizes = [(10,30,10),(20,)]\\nactivation =  [\\'tanh\\', \\'relu\\']\\nsolver =  [\\'sgd\\', \\'adam\\']\\nalpha =  [0.0001, 0.05]\\nlearning_rate =  [\\'constant\\',\\'adaptive\\']\\n\\nkf = KFold(n_splits=k)\\ntext_file = open(\"mlp_tuning.txt\", \"w\")\\nfor h in hidden_layer_sizes:\\n    for act in activation:\\n        for s in solver:\\n            for a in alpha:\\n                for l in learning_rate:\\n                    mlp_gs.set_params(hidden_layer_sizes = h, activation = act, solver = s, alpha = a, learning_rate = l)\\n                    deceased_f1score = 0\\n                    mean_macro_f1score = 0\\n                    accuracy = 0\\n                    for i, (train_index, test_index) in enumerate(kf.split(data)):\\n                        train_fold_data = np.take(data, train_index, 0)\\n                        train_fold_labels = np.take(labels, train_index, 0)\\n                        test_fold_data = np.take(data, test_index, 0)\\n                        test_fold_labels = np.take(labels, test_index, 0)\\n                        rf_model.fit(train_fold_data, train_fold_labels)\\n                        predictions = rf_model.predict(test_fold_data)\\n                        _, _, fscore, _ = precision_recall_fscore_support(predictions, test_fold_labels)\\n                        deceased_f1score += fscore[0]\\n                        mean_macro_f1score += ( fscore[0] + fscore[1] + fscore[2] ) / 3\\n                        accuracy += accuracy_score(predictions, test_fold_labels)\\n                    deceased_f1score = deceased_f1score / k\\n                    mean_macro_f1score = mean_macro_f1score / k\\n                    accuracy = accuracy / k\\n                    output = \"\\nhidden_layer_sizes=\"+str(h)+\",activation=\"+str(act)+\",solver=\"+str(s)+\",alpha=\"+str(a)+\",learning_rate=\"+str(l)+\" -> mean_macro_f1_score=\"+str(mean_macro_f1score)+\",deceased_f1score=\"+str(deceased_f1score)+\",accuracy=\"+str(accuracy)\\n                    print(output)\\n                    n = text_file.write(output)\\ntext_file.close()\\n'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# UNCOMMENT TO REPRODUCE mlps_tuning.txt FILE\n",
    "\"\"\"\n",
    "from sklearn.model_selection import KFold\n",
    "data = train_data.iloc[:, :4].values\n",
    "labels = train_data.iloc[:, 4].values.ravel()\n",
    "mlp_gs = MLPClassifier()\n",
    "hidden_layer_sizes = [(10,30,10),(20,)]\n",
    "activation =  ['tanh', 'relu']\n",
    "solver =  ['sgd', 'adam']\n",
    "alpha =  [0.0001, 0.05]\n",
    "learning_rate =  ['constant','adaptive']\n",
    "\n",
    "kf = KFold(n_splits=k)\n",
    "text_file = open(\"mlp_tuning.txt\", \"w\")\n",
    "for h in hidden_layer_sizes:\n",
    "    for act in activation:\n",
    "        for s in solver:\n",
    "            for a in alpha:\n",
    "                for l in learning_rate:\n",
    "                    mlp_gs.set_params(hidden_layer_sizes = h, activation = act, solver = s, alpha = a, learning_rate = l)\n",
    "                    deceased_f1score = 0\n",
    "                    mean_macro_f1score = 0\n",
    "                    accuracy = 0\n",
    "                    for i, (train_index, test_index) in enumerate(kf.split(data)):\n",
    "                        train_fold_data = np.take(data, train_index, 0)\n",
    "                        train_fold_labels = np.take(labels, train_index, 0)\n",
    "                        test_fold_data = np.take(data, test_index, 0)\n",
    "                        test_fold_labels = np.take(labels, test_index, 0)\n",
    "                        rf_model.fit(train_fold_data, train_fold_labels)\n",
    "                        predictions = rf_model.predict(test_fold_data)\n",
    "                        _, _, fscore, _ = precision_recall_fscore_support(predictions, test_fold_labels)\n",
    "                        deceased_f1score += fscore[0]\n",
    "                        mean_macro_f1score += ( fscore[0] + fscore[1] + fscore[2] ) / 3\n",
    "                        accuracy += accuracy_score(predictions, test_fold_labels)\n",
    "                    deceased_f1score = deceased_f1score / k\n",
    "                    mean_macro_f1score = mean_macro_f1score / k\n",
    "                    accuracy = accuracy / k\n",
    "                    output = \"\\nhidden_layer_sizes=\"+str(h)+\",activation=\"+str(act)+\",solver=\"+str(s)+\",alpha=\"+str(a)+\",learning_rate=\"+str(l)+\" -> mean_macro_f1_score=\"+str(mean_macro_f1score)+\",deceased_f1score=\"+str(deceased_f1score)+\",accuracy=\"+str(accuracy)\n",
    "                    print(output)\n",
    "                    n = text_file.write(output)\n",
    "text_file.close()\n",
    "\"\"\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.5 Check for overfitting"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XG Boost,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset F1-Score = 0.8209148980683397\n",
      "Validation Dataset F1-Score = 0.8166133988127054\n"
     ]
    }
   ],
   "source": [
    "# Checking for overfitting on XG Boost model by comparing results on train versus validation datasets.\n",
    "train_data_formatted = train_data.iloc[:, :4].values\n",
    "train_labels_truth = train_data.iloc[:, 4].values.reshape(-1, 1)\n",
    "train_labels_predicted = xgb_model.predict(train_data_formatted)\n",
    "train_data_score = f1_score(train_labels_predicted, train_labels_truth, average = \"macro\")\n",
    "\n",
    "validation_data_formatted = validation_data.iloc[:, :4].values\n",
    "validation_labels_truth = validation_data.iloc[:, 4].values.reshape(-1, 1)\n",
    "validation_labels_predicted = xgb_model.predict(validation_data_formatted)\n",
    "validation_data_score = f1_score(validation_labels_predicted, validation_labels_truth, average = \"macro\")\n",
    "\n",
    "print(\"Training Dataset F1-Score = \" + str(train_data_score))\n",
    "print(\"Validation Dataset F1-Score = \" + str(validation_data_score))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset F1-Score = 0.8217136663750333\n",
      "Validation Dataset F1-Score = 0.8163771105866132\n",
      "Validation dataset accuracy score = 0.8209520672047439\n"
     ]
    }
   ],
   "source": [
    "# Checking for overfitting on Random Forest model by comparing results on train versus validation datasets.\n",
    "train_data_formatted = train_data.iloc[:, :4].values\n",
    "train_labels_truth = train_data.iloc[:, 4].values.ravel()\n",
    "train_labels_predicted = rf_model.predict(train_data_formatted)\n",
    "train_data_score = f1_score(train_labels_predicted, train_labels_truth, average = \"macro\")\n",
    "\n",
    "validation_data_formatted = validation_data.iloc[:, :4].values\n",
    "validation_labels_truth = validation_data.iloc[:, 4].values.ravel()\n",
    "validation_labels_predicted = rf_model.predict(validation_data_formatted)\n",
    "validation_data_score = f1_score(validation_labels_predicted, validation_labels_truth, average = \"macro\")\n",
    "\n",
    "print(\"Training Dataset F1-Score = \" + str(train_data_score))\n",
    "print(\"Validation Dataset F1-Score = \" + str(validation_data_score))\n",
    "\n",
    "accuracy = accuracy_score(validation_labels_predicted, validation_labels_truth)\n",
    "print(\"Validation dataset accuracy score = \" + str(accuracy)) "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset F1-Score = 0.7763669255728032\n",
      "Validation Dataset F1-Score = 0.7701810376655058\n"
     ]
    }
   ],
   "source": [
    "train_data_formatted = train_data.iloc[:, :4].values\n",
    "train_labels_truth = train_data.iloc[:, 4].values.reshape(-1, 1)\n",
    "train_labels_predicted = mlp_gs.predict(train_data_formatted)\n",
    "train_data_score = f1_score(train_labels_predicted, train_labels_truth, average = \"macro\")\n",
    "\n",
    "validation_data_formatted = validation_data.iloc[:, :4].values\n",
    "validation_labels_truth = validation_data.iloc[:, 4].values.reshape(-1, 1)\n",
    "validation_labels_predicted = mlp_gs.predict(validation_data_formatted)\n",
    "validation_data_score = f1_score(validation_labels_predicted, validation_labels_truth, average = \"macro\")\n",
    "\n",
    "print(\"Training Dataset F1-Score = \" + str(train_data_score))\n",
    "print(\"Validation Dataset F1-Score = \" + str(validation_data_score))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.7 Prediction on test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "# USING XG BOOST FOR NOW BUT WE CAN SUB THIS OUT FOR BEST PERFORMING MODEL LATER\n",
    "testing_data = test_data.iloc[:, :4].values\n",
    "predicted_labels = xgb_model.predict(testing_data)\n",
    "# CHANGE MODEL NAME TO BEST PERFORMING MODEL LATER\n",
    "model_name = \"xgboost\"\n",
    "result_data_frame = pd.DataFrame(testing_data, columns=[\"age\", \"country\", \"chronic_disease_binary\", \"Case_Fatality_Ratio\"])\n",
    "\n",
    "# This function is from the TA\n",
    "def create_submission_file(y_preds, file_name):\n",
    "    with open(file_name, 'w') as csvfile:\n",
    "        wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)\n",
    "        wr.writerow([\"Id\", \"Prediction\"])\n",
    "        for i, pred in enumerate(y_preds):\n",
    "            wr.writerow([str(i), str(pred)])\n",
    "create_submission_file(predicted_labels, \"submission_\"+model_name+\".csv\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Depending on the final model selection, we can delete one of these\n",
    "\n",
    "import csv\n",
    "testing_data = test_data.iloc[:, :4].values\n",
    "predicted_labels = rf_model.predict(testing_data)\n",
    "\n",
    "model_name = \"random_forest\"\n",
    "result_data_frame = pd.DataFrame(testing_data, columns=[\"age\", \"country\", \"chronic_disease_binary\", \"Case_Fatality_Ratio\"])\n",
    "\n",
    "# This function is from the TA\n",
    "def create_submission_file(y_preds, file_name):\n",
    "    with open(file_name, 'w') as csvfile:\n",
    "        wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)\n",
    "        wr.writerow([\"Id\", \"Prediction\"])\n",
    "        for i, pred in enumerate(y_preds):\n",
    "            wr.writerow([str(i), str(pred)])\n",
    "create_submission_file(predicted_labels, \"submission_\"+model_name+\".csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5ebeb2db109b4e4030ca6b0eb886199afc2cd913864cd051a344632d0064a896"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}