example_usage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from KNN.modelling import KNN_data_collection as knn\n",
    "from KNN.modelling import generate_predictions as gp\n",
    "from KNN.assessment import model_metrics as mm\n",
    "from KNN.assessment import cross_validation as cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example 1: KNN classifier using Iris dataset**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Create model & perform train/test split*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created KNN classifier with k=8!\n"
     ]
    }
   ],
   "source": [
    "# Create classifier with k=8\n",
    "iris_classifier=knn.KNN('classifier',8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset successfully loaded!\n"
     ]
    }
   ],
   "source": [
    "# Load dataset into classifier and set response variable to \"Species\"\n",
    "iris_classifier.load_csv('datasets/Iris.csv','Species')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully completed train/test split!\n",
      "Training set: 120 samples\n",
      "Test set: 30 samples\n"
     ]
    }
   ],
   "source": [
    "# Perform a 80/20 train/test split\n",
    "iris_classifier.train_test_split(0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Generate predictions*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array('Iris-versicolor', dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate a prediction using \"generate_predictions\" module\n",
    "   # Generate prediction using fit on entire dataset\n",
    "gp.generate_prediction(iris_classifier,[6.3,2.3,4.4,1.3],'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Iris-versicolor', 'Iris-virginica', 'Iris-setosa'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate multiple predictions\n",
    "   # Generate prediction using fit on training set only\n",
    "to_predict=np.array([[6.3,2.3,4.4,1.3],[6.8,3.2,5.9,2.3],[4.3,3,1.1,0.1]])\n",
    "gp.generate_predictions(iris_classifier,to_predict,'all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Assess model performance for these predictions*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6666666666666666"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute model accuracy for these 3 hypothetical predictions\n",
    "mm.model_accuracy(['Iris-virginica', 'Iris-virginica', 'Iris-setosa'],gp.generate_predictions(iris_classifier,to_predict,'all'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3333333333333333"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute model misclassification rate for these 3 hypothetical predictions\n",
    "mm.model_misclassification(['Iris-virginica', 'Iris-virginica', 'Iris-setosa'],gp.generate_predictions(iris_classifier,to_predict,'all'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check number correct\n",
    "mm.model_num_correct(['Iris-virginica', 'Iris-virginica', 'Iris-setosa'],gp.generate_predictions(iris_classifier,to_predict,'all'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check number incorrect\n",
    "mm.model_num_incorrect(['Iris-virginica', 'Iris-virginica', 'Iris-setosa'],gp.generate_predictions(iris_classifier,to_predict,'all'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Assess model performance on test set*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.8333333333333334\n",
      "Test misclassification rate: 0.16666666666666666\n",
      "Num correct predictions: 25\n",
      "Num incorrect predictions: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ericphillips/ubc_mds/data_533/knn/KNN/modelling/generate_predictions.py:27: UserWarning: Warning: A tie has occurred (top two classes in K nearest neighbors have the same number of occurances). Classification depends on the order of the training data.\n",
      "  warnings.warn('Warning: A tie has occurred (top two classes in K nearest neighbors have the same number of occurances). Classification depends on the order of the training data.')\n"
     ]
    }
   ],
   "source": [
    "mm.assessment_metrics(iris_classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Above warning indicates that a tie occurred while generating test set predictions\n",
    "   # This is explained by our choice of an even number for k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Tune k hyperparameter using 6-fold cross validation*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created CV instance for KNN classifier with 6 folds!\n"
     ]
    }
   ],
   "source": [
    "# Create CV instance for 6-fold CV\n",
    "iris_cv=cv.CvKNN('classifier',6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset successfully loaded!\n",
      "Successfully completed train/test split!\n",
      "Training set: 120 samples\n",
      "Test set: 30 samples\n"
     ]
    }
   ],
   "source": [
    "# Load dataset into CV instance and set response variable to \"Species\"\n",
    "iris_cv.load_csv('datasets/Iris.csv','Species')\n",
    "# Perform 80/20 train/test split\n",
    "iris_cv.train_test_split(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully performed 6 CV!\n"
     ]
    }
   ],
   "source": [
    "# Perform 6-fold CV for k values 1 through 10\n",
    "iris_cv.perform_cv(np.arange(1,11))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k=1: 0.041666666666666664\n",
      "k=2: 0.05833333333333334\n",
      "k=3: 0.05000000000000001\n",
      "k=4: 0.041666666666666664\n",
      "k=5: 0.03333333333333333\n",
      "k=6: 0.041666666666666664\n",
      "k=7: 0.05000000000000001\n",
      "k=8: 0.05000000000000001\n",
      "k=9: 0.075\n",
      "k=10: 0.08333333333333333\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0p0lEQVR4nO3deXhU5dnH8e+dFQiENez7LouABAhatYqoKGq1KIsIytpW69r21b5tX2t3W9FatbKvsqi1LVA3xF1JIBETdgh7SCAhQFhDlrnfP+bQxnQSBsjkzGTuz3XNlZlzzpzzyyhz5zzPOc8jqooxxhhTXoTbAYwxxgQnKxDGGGN8sgJhjDHGJysQxhhjfLICYYwxxicrEMYYY3yyAmFqFBH5tYgcFpGDfmy7R0RuqGDdt0Ukq+oTBg8RuV9EPnc7hwleViBM0BORUSKyRUROichOEbm6gu3aAk8APVS1eYAziYg8LCIbnVxZIvKGiPQWkSdF5FMf72kiIkUi0svHuqdFpFhETorIMRH5UkQGB/J38JFBRaRzdR7TBDcrECaoichQ4A/AA0A94BpgVwWbtwXyVTW3GqL9GXgEeBhoBHQF/gHcCiwCrhSRDuXeMwrYoKobK9jnMlWtCzQBPgLeCEBuY/xmBcIEu18Cz6hqsqp6VPWAqh4ov5HTVLQKaOn8FT7PWX67iGxy/ir/WEQu83UQEaktIvNE5KiIbAYGVBRIRLoADwKjVfVDVT2rqqdV9TVV/b2qZgEfAveVe+s4YMH5fmFVLQFeA1qJSIJzzPoiMltEckTkgNOUFums6ywin4hIgdO8tsxZ3t45K4gqk/1jEZnk43c6d8aT7nx+I50znpXOZ3dERD4TEfvOCCP2H9sELecLMBFIEJFMpxnnJRGpXX5bVf0AGAZkq2pdVb1fRLoCS4BHgQTgbWCFiMT4ONz/AZ2cx03A+EqiDQGyVHVtJdvMp0yBEJFuQF9gcSXvObdtDN5ikg8cdRbPA0qAzkA/4Ebg3Bf9r4D3gYZAa+Av5ztGeap6jfO0j/P5LcPbXJeF97NrBvwUsLF5wogVCBPMmgHRwAjgarxfsP2An/n5/pHAv1R1laoWA38CagNX+tj2HuA3qnpEVfcDL1ay38ZAznmO/XegmYicO9Y44B1VzavkPfeIyDHgDDAZGKGqJSLSDLgFeFRVTzlNaM/jbbICKAbaAS1VtVBVq6rjuRhoAbRT1WJV/Uxt8LawYgXCBLMzzs+/qGqOqh4GpuH9svRHS2DvuReq6gH2A60q2HZ/mdd7fWxzTj7eL84KqeppvH0I40REgHs5f/PS66raAG9h3Aj0d5a3w1soc5zmnmPAdKCps/4ngABrnea0Cec5jr/+CGQC74vILhF5sor2a0KEFQgTtFT1KN4mjrJ/tV7IX7DZeL9cAe+VR0Ab4L/6MPCeEbQp87ptJftdDbQWkcTzHH8+3jOToXg72Ff4kRmnEE4BnhaRFngL11mgiao2cB7xqtrT2f6gqk5W1ZbAVOAV52qkU84u65TZvd9Xd6nqCVV9QlU7ArcDj4vIEH/fb0KfFQgT7OYCPxSRpiLSEHgMWOnne18HbhWRISISjbdN/SzwZQXbPiUiDUWkNfDDinaqqjuAV4Alzv0SMSJSy7kct+xf2Z8Bx4AZwFJVLfIzN6q6DXgP+Imq5uDtY3hOROJFJEJEOonItQAicreTGbx9Fgp4nOasA8BYEYl0ziw6VXLYQ0DHcy9EZLjTAS5AAVAKePz9HUzoswJhgt2vgHXAdmALsB74jT9vdL5kx+LttD0M3AbcVsEX9S/xNivtxvtlvPA8u38YeAl4GW8R2AncSZmzBKe9fgHes5jzXr3kwx+BKSLSFG8fRgywGW8ReJP/NHMNAFJE5CSwHHhEVc9dCjwZ+DHeZrGe+C6O5zwNzHease4BugAfACeBNcArqvrRRfweJkSJ9TkZY4zxxc4gjDHG+GQFwhhjjE9WIIwxxvhkBcIYY4xPUeffJDQ0adJE27dv73YMY4wJKWlpaYdVNcHXuhpTINq3b09qaqrbMYwxJqSISIWjBlgTkzHGGJ+sQBhjjPHJCoQxxhifrEAYY4zxyQqEMcYYn6xAGGOM8ckKhDHGGJ+sQBhjTAhblLyXz3ccDsi+rUAYY0yImv35bn72j40sS91//o0vQo25k9oYY8LJXz/eyR/e3cqwXs157u4+ATmGFQhjjAkxL67ewbRV27mtT0uev6cPUZGBaQyyAmGMMSFCVZm2ajt/+TCTu65oxR9H9CEyQgJ2PCsQxhgTAlSV37+zlemf7mLUgDb89s7eRASwOIAVCGOMCXqqyjMrNzP3iz3cl9SOX97eM+DFAaxAGGNMUPN4lJ//cyOvpexjwlUd+PnwyxAJfHEAKxDGGBO0Sj3KU29l8HpqFt+7thP/c3O3aisOYAXCGGOCUkmph5+8mcFb6w/w8JAuPHZDl2otDmAFwhhjgk5xqYfHln3NyowcfnRjVx66vosrOQJ6J7WI3Cwi20QkU0Se9LE+VkSWOetTRKS9szxaROaLyAYR2SIiTwUypzHGBIuiEg8PLf6KlRk5PDWsu2vFAQJYIEQkEngZGAb0AEaLSI9ym00EjqpqZ+B54A/O8ruBWFXtDfQHpp4rHsYYU1OdLSnl+4vSeG/TIX4xvAdTr+3kap5AnkEMBDJVdZeqFgFLgTvKbXMHMN95/iYwRLyNbArEiUgUUBsoAo4HMKsxxriqsLiUyQvSWL01l19/pxcTvtXB7UgBLRCtgLIjSGU5y3xuo6olQAHQGG+xOAXkAPuAP6nqkfIHEJEpIpIqIql5eXlV/xsYY0w1OF1UwoR56/hsRx7Pfvdyxia1czsSELyjuQ4ESoGWQAfgCRHpWH4jVZ2hqomqmpiQkFDdGY0x5pKdPFvC/XPWkbwrn2n39OGeAW3cjvRvgSwQB4Cyv2lrZ5nPbZzmpPpAPjAGeFdVi1U1F/gCSAxgVmOMqXbHC4sZNzuFtH1H+fOoftzZr7Xbkb4hkAViHdBFRDqISAwwClhebpvlwHjn+QjgQ1VVvM1K1wOISByQBGwNYFZjjKlWBaeLGTsrhQ0HCnh5TD9u69PS7Uj/JWD3QahqiYg8BLwHRAJzVHWTiDwDpKrqcmA2sFBEMoEjeIsIeK9+misimwAB5qpqRqCyGmNMdTpyqoixs1LIzD3Jq2P7M+SyZm5H8km8f7CHvsTERE1NTXU7hjHGVCrvxFnGzkphT/4pZoxL5Nqu7vafikiaqvpswrc7qY0xppocOl7ImJnJZB8rZO79A7iycxO3I1XKCoQxxlSD7GNnGDMzmbwTZ5n3wAAGdWzsdqTzsgJhjDEBtv/IacbMSubYqWIWTBxE/3YN3Y7kFysQxhgTQHvzTzFmZgonCotZNGkQfdo0cDuS36xAGGNMgOzMO8mYmckUlXhYPDmJXq3qux3pgliBMMaYANh+6ARjZqYAypIpSXRvHu92pAtmBcIYY6rY5uzjjJ2dQlSEsHhyEp2b1nM70kWxAmGMMVVo44ECxs5OoXZ0JIsnJ9GhSZzbkS6aFQhjjKki6/cdZdyctcTXimbplCTaNKrjdqRLYgXCGGOqwLo9R3hg7joaxcWwZEoSrRrUdjvSJQvW4b6NMSZkrNmZz/g5a2laL5bXpw6uEcUBrEAYY8wl+WxHHg/MW0urBrVZOjWJ5vVruR2pylgTkzHGXKSPtuYydVEaHZvE8dqkQTSuG+t2pCplBcIYYy7CR9tymbIwlW7N67FwwiAaxsW4HanKWYEwxpiL8Oy722jbqA6vTUqifu1ot+MEhPVBGGPMBcrMPcmWnOPcO6hdjS0OYAXCGGMu2MqMbETg1stbuB0loKxAGGPMBVBVVqRnM7B9I5rF15wrlnyxAmGMMRdgS84Jduad4rY+Ld2OEnBWIIwx5gKszMgmMkIY1qu521ECzgqEMcb4SVVZkZHNlZ0a17h7HnyxAmGMMX7KyCpg/5EzYdG8BFYgjDHGbyvSs4mOFG7qUfObl8AKhDHG+MXjUVZm5HBt1wTq16m59z6UZQXCGGP8kLbvKAePFzL88vBoXgIrEMYY45cV6dnERkVwQ49mbkepNlYgjDHmPEpKPby9IYchlzWlbmz4DGFnBcIYY84jZfcRDp8sCqvmJfCzQIhIbRHpFugwxhgTjFakZxMXE8l13Zq6HaVanbdAiMhtwNfAu87rviKyPMC5jDEmKBSVeHh300Fu6NGM2jGRbsepVv6cQTwNDASOAajq10CHgCUyxpgg8kXmYY6dLua2MGteAv8KRLGqFpRbpv7sXERuFpFtIpIpIk/6WB8rIsuc9Ski0t5Zfq+IfF3m4RGRvv4c0xhjqtKKjGzia0Vxddcmbkepdv4UiE0iMgaIFJEuIvIX4MvzvUlEIoGXgWFAD2C0iPQot9lE4KiqdgaeB/4AoKqvqWpfVe0L3Afsds5cjDGm2hQWl/L+pkPc1LM5sVHh1bwE/hWIHwI9gbPAYqAAeMSP9w0EMlV1l6oWAUuBO8ptcwcw33n+JjBERKTcNqOd9xpjTLX6eFseJ8+WhM3YS+X5UyBuVdX/VdUBzuNnwO1+vK8VsL/M6yxnmc9tVLUEb/FpXG6bkcASXwcQkSkikioiqXl5eX5EMsYY/63MyKZRXAxXdir/tRQe/CkQT/m5rMqJyCDgtKpu9LVeVWeoaqKqJiYkJFRHJGNMmDhdVMLqLbkM69WcqMjwvGWswlsCRWQYcAvQSkReLLMqHijxY98HgDZlXrd2lvnaJktEooD6QH6Z9aOo4OzBGGMCafWWXM4Ul4Zt8xJUfgaRDaQChUBamcdy4CY/9r0O6CIiHUQkBu+Xffn7J5YD453nI4APVVUBRCQCuAfrfzDGuGBFejZN68UyoH0jt6O4psIzCFVNB9JFZLGqFl/ojlW1REQeAt4DIoE5qrpJRJ4BUlV1OTAbWCgimcARvEXknGuA/aq660KPbYwxl+J4YTEfb8/j3kFtiYwof91M+PBn1Kn2IvI7vJeq1jq3UFU7nu+Nqvo28Ha5Zb8o87wQuLuC934MJPmRzxhjqtSqTYcoKvGE3dhL5fnT8zIX+CvefofrgAXAokCGMsYYN63IyKZVg9pc0baB21Fc5U+BqK2qqwFR1b2q+jRwa2BjGWOMO46eKuLzHYcZ3qcF/31bVnjxp4nprNNhvMPpUzgA1A1sLGOMcce7mw5S4tGwHHupPH/OIB4B6gAPA/3xDn0xLpChjDHGLSszsunQJI6eLePdjuK68xYIVV2nqidVNUtVH8Dbqdw58NGMMaZ65Z4oZM3OfIZfbs1LUEmBEJF4EXlKRF4SkRvF6yEgE+/9CcYYU6O8s+EgHiWsb44rq7I+iIXAUWANMAn4KSDAnTayqjGmJlqZkU3XZnXp2qye21GCQmUFoqOq9gYQkVlADtDWuXfBGGNqlOxjZ1i35yhPDO3qdpSgUVkfxL/vnlbVUiDLioMxpqZ6e0MOAMOteenfKjuD6CMix53nAtR2Xgugqmpd/MaYGmNFeja9WsXToUmc21GCRmVjMYXf9EnGmLC0N/8U6VkFPDWsu9tRgkp4DnJujDFlrMzwNi/denkLl5MEFysQxpiwtyI9myvaNqB1wzpuRwkqViCMMWEtM/cEWw+esHsffLACYYwJayvScxCBW3pb81J55y0QInKXiOwQkQIROS4iJ8pc3WSMMSFLVVmRkc2gDo1oFl/r/G8IM/6cQTwL3K6q9VU1XlXr2SWuxpiaYEvOCXblnQr7iYEq4k+BOKSqWwKexBhjqtmKjGwiI4RhvZq7HSUo+TMfRKqILAP+AZw9t1BV3wpUKGOMCTRVZWVGNld1bkLjurFuxwlK/hSIeOA0cGOZZQpYgTDGhKz0rAL2HznDD6/v4naUoHXeAuHMAWGMMTXKyvRsoiOFm3pa81JF/LmKqbWI/F1Ecp3H30SkdXWEM8aYQPB4lJUZOVzbNYH6taPdjhO0/OmkngssB1o6jxXOMmOMCUmpe49y8Hih3Rx3Hv4UiARVnauqJc5jHpAQ4FzGGBMwKzOyqRUdwQ2XNXM7SlDzp0Dki8hYEYl0HmOB/EAHM8aYQCgp9fD2hhyu796UuFh/rtMJX/4UiAl456A+iHdWuRGAdVwbY0JSyu4jHD5ZxG12c9x5+XMV017g9mrIYowxAbciPZu4mEiu697U7ShBr8ICISI/UdVnReQveO97+AZVfTigyYwxpooVlXh4Z+NBhvZoRq1omxPtfCo7gzg3vEZqdQQxxphA+yLzMAVniu3qJT9VNuXoCufpaVV9o+w6Ebk7oKmMMSYAVqRnE18riqu72IWY/vCnk/opP5f9FxG5WUS2iUimiDzpY32siCxz1qeISPsy6y4XkTUisklENoiIjcVrjLlohcWlvL/5EDf3ak5MlE2F44/K+iCGAbcArUTkxTKr4oGS8+1YRCKBl4GhQBawTkSWq+rmMptNBI6qamcRGQX8ARgpIlHAIuA+VU0XkcZA8QX+bsYY828fb8vj5NkSG9r7AlRWRrPx9j8UAmllHsuBm/zY90AgU1V3qWoRsBS4o9w2dwDznedvAkNERPAODJihqukAqpqvqqX+/UrGGPPfVmRk0yguhis7NXY7SsiorA8iHUgXkcWqejF/vbcC9pd5nQUMqmgbVS0RkQKgMdAVUBF5D+9d20tV9dnyBxCRKcAUgLZt215ERGNMODhdVMKHW3L5bv9WREVa85K//Pmk2ovImyKyWUR2nXsEOFcU8C3gXufnnSIypPxGqjpDVRNVNTEhwTqdjDG+fbAllzPFpda8dIH8Hazvr3j7Ha4DFuDtHzifA0CbMq9bO8t8buP0O9THO4xHFvCpqh5W1dPA28AVfhzTGGP+y8r0bJrFxzKgfSO3o4QUfwpEbVVdDYiq7lXVp4Fb/XjfOqCLiHQQkRhgFN7+i7KWA+Od5yOAD1VVgfeA3iJSxykc1wKbMcaYC3S8sJiPt+VxS+8WREaI23FCij8jVZ0VkQhgh4g8hPev/rrne5PTp/AQ3i/7SGCOqm4SkWeAVFVdDswGFopIJnAEbxFBVY+KyDS8RUaBt1X1Xxfx+xljwtz7mw5RVOqxm+Mugj8F4hGgDvAw8Cu8zUzjK32HQ1Xfxts8VHbZL8o8LwR83nSnqovwrynLGGMqtDIjm1YNatOvTQO3o4Sc8zYxqeo6VT2pqlmq+oCqfldVk6sjXDjZl3+a3/xrMwVn7HYPY6rK0VNFfL7jMMP7tMB7Bb25EP5MObpKRBqUed3QufzUVKFfrtjEzM92M3ZWCsdOF7kdx5ga4d1NBynxqA3tfZH86aRuoqrHzr1Q1aOAjZNbhdbvO8rqrbnc2KMZ2w6dYPTMFPJPnnU7ljEhb0V6Nh2axNGzZbzbUUKSPwXCIyL/vgtNRNrhY/hvc/GmrdpOo7gYnh/Zl9njE9l9+CSjZyaTe6LQ7WjGhKzcE4Uk78rntsuteeli+VMg/hf4XEQWisgi4FP8HKzPnN/a3Uf4bMdhvn9tJ+JivaNMzr1/IFlHzzBqRjIHC6xIGHMx3tlwEI9iVy9dAn86qd/Fe5PaMrzjKfVXVeuDqAKqyp/e30ZCvVjGJrX79/LBnRqzYMJAco+fZeSMNRw4dsbFlMaEphXp2XRrVo8uzeq5HSVkVVggRKS78/MKoC3ewfuygbbOMnOJvsjMZ+3uIzx0XWdqx3xzdqvE9o1YOHEgR04VMXL6GvYfOe1SSmNCT/axM6TuPcptfVq4HSWkVXYG8bjz8zkfjz8FOFeNp6o8t2obLevXYtTANj636de2IYsnJXHybAkjp69hz+FT1ZzSmND0r4wcABt76RJVViBWOT8nqup15R7XV0e4muyjbbms33eMHw7pQmxUxXPj9m5dn8WTkigs8XDP9DVk5p6sxpTGhKaVGdn0blWf9k3i3I4S0iorEOc6ot+sjiDhRFWZtmo7bRvVYUT/1ufdvkfLeJZOScKjMGrGGrYdPFENKY0JTXvzT5GeVWDNS1WgsgKRLyLvAx1EZHn5R3UFrIne23SIjQeO8/CQLkT7OTZ912b1WDY1icgIYdSMNWzKLghwSmNC00qneelWa166ZJV9O90K/AI4jO9+CHMRPB7l+VXb6ZgQx3f6Xtj/wJ0S6rJsymBqR0cyZmYKGVnHAhPSmBC2Ij2b/u0a0qpBbbejhLwKC4SqFjljLl2pqp+Uf1Rjxhpl5YYcth06waM3dL2oma3aN4lj2dTBxNeO4t6ZKXy172gAUhoTmjJzT7D14AmGX27NS1WhsstcX3CezrEmpqpRUurhhVXb6dasHsN7X/z/wG0a1WHZlME0rhvDfbNSWLv7SBWmNCZ0rUjPQQRuvYR/X+Y/Khvue6Hz0y5prSL/+DqbXYdP8erY/kRc4sQlLRvUZtnUwYyZmcz4OWuZPT6RKzs3qaKkxoQeVWVFRjZJHRrTNL6W23FqhMqamNKcn2WblTKAo9bEdOGKSz38efV2erWK56aezapkn83ia7F0ymDaNqrDA/PW8cn2vCrZrzGhaHPOcXblnWK4Xb1UZfwZ7vtjEYkXkUbAV8BMZ7Y3cwHeSM1i/5EzPDG0W5UOHJZQL5YlU5LolFCXyfNTWb3lUJXt25hQsjIjh8gIYVgvKxBVxZ9e0vqqehy4C1igqoOAGwIbq2Y5W1LKSx/uoF/bBny7W0KV779RXAyLJw+ie4t6fG9RGu9uPFjlxzAmmKkqK9KzuapzExrFxbgdp8bwp0BEiUgL4B5gZYDz1EhL1+4nu6CQH91YtWcPZTWoE8OiSYPo3ao+Dy7+ipUZ2QE5jjHBKD2rgKyjZ7jNrl6qUv4UiGeA94BMVV0nIh2BHYGNVXOcKSrlpY8yGdShEVd2ahzQY8XXimbBxEH0b9uQh5es5+/rswJ6PGOCxYr0bGIiI7ixZ3O3o9Qo/gz3/YaqXq6qP3Be71LV7wY+Ws2wKHkveSfO8kQAzx7KqhsbxbwJA0jq2JjHX0/n9XX7A35MY9zk8Sj/ysjhmq4J1K8d7XacGsWfTupnnU7qaBFZLSJ5IjK2OsKFupNnS/jrJzu5uksTBnZoVG3HrRMTxZz7B3B1lwR+8rcMXkvZW23HNqa6pe49ysHjhTb2UgD408R0o9NJPRzYA3QGfhzIUDXF/C/3cORUEU/c2K3aj10rOpIZ9/VnSPem/O/fNzLvi93VnsGY6rAiPZta0RHccFnVXD5u/sOvTmrn563AG6pqo8T5oeBMMdM/2ckNlzWlb5sGrmSoFR3JX8f256aezXh6xWZmfLrTlRzGBEpJqYd3NuYwpHsz4mIru+/XXAx/CsRKEdkK9AdWi0gCYBMln8fsz3dzvLCEx4Z2dTVHTFQEL425guGXt+C3b2/lpQ/t+gJTcyTvOsLhk0XWvBQg5y25qvqkiDwLFKhqqYicAu4IfLTQdfRUEXM+382wXs3p2bK+23GIjozghZF9iYmM4E/vb6eoVHnshi7V0mluTCCtzMgmLiaSb3dr6naUGsnfc7KWwA0iUnaAkwUByFMjTP90F6eK3D97KCsqMoI/3t2HqEjhxdU7KC718JObqufKKmMCoajEwzsbD3Jjz+bUiq54VkZz8c5bIETk/4BvAz2At4FhwOdYgfAp78RZ5n+5h9v7tKRrs3pux/mGyAjh93ddTnRkBH/9eCdFJR5+dutlViRMSPo8M4+CM8U2tHcA+XMGMQLoA6xX1QdEpBmwKLCxQtern+ykqNTDI0O6uB3Fp4gI4dff6UV0ZASzP99NcamHp2/recmjyxpT3Vam5xBfK4qru1T98DXGy58CcUZVPSJSIiLxQC7QJsC5QtLBgkIWJu/lrn6t6JhQ1+04FRIR/u+2HsRGRTD9010UlXj47Z29rUiYkFFYXMr7mw9xa+8WxERd+MRbxj/+FIhUEWkAzATSgJPAmkCGClUvf5SJx6M8HKRnD2WJCE8O6050ZAQvfZRJcany7IjLibQiYULAx9vyOHm2xIb2DjB/rmL6gfP0VRF5F4hX1Qx/di4iNwN/BiKBWar6+3LrY/H2ZfQH8oGRqrpHRNoDW4BtzqbJqvo9f47plqyjp1m6bh8jB7ShTaM6bsfxi4jwo5u6ERMVwbRV2yku9TDtnj4XNRWqMdVpRUY2jeNiGNwxsOObhbsKC4SIXFHZOlX9qrIdi0gk8DIwFMgC1onIclXdXGaziXgnIOosIqOAPwAjnXU7VbWvf7+G+/6yOhMR4aHrO7sd5YI9PKQL0ZER/OHdrZR4PPx5VD+irUiYIHXqbAmrtxxiRP/W9sdMgFV2BvFcJesUuP48+x6IdwTYXQAishTv/RNlC8QdwNPO8zeBlyQEL6nZc/gUb36VxbjB7WhRv7bbcS7K97/diehI4df/2kJRyVe8fG8/YqPs0kHzH1sPHmfa+9s5U1zqao7jhSUUFnu47fKWruYIBxUWCFW97hL33QooO5RoFjCoom1UtURECoBz54wdRGQ9cBz4map+Vv4AIjIFmALQtm3bS4x78f68egfRkcL3v93JtQxVYdLVHYmNiuDn/9zE1IVpvDq2v11fbgDYeKCA+2anANC+SZyrWSIEbu/TkgHtq28AzHDlz30QDwKvqeox53VDYLSqvhLAXDlAW1XNF5H+wD9EpKczaOC/qeoMYAZAYmKiBjBPhTJzT/CPrw8w5eqONK0X+hOl3ze4PVGREfz07xuYND+VmeMSqR1jRSKcfb3/GONmp1CvVjSLJw+iXWN3C4SpPv404E0+VxwAVPUoMNmP9x3gm5fDtnaW+dxGRKKA+kC+qp5V1XzneGnATiB4bksu4/kPdlAnOpKp14b22UNZowe25Y8j+vDlzsPcP3ctp86WuB3JuCRt7xHGzkqhfp1olk1NsuIQZvwpEJFl+wWczmd/Jn1dB3QRkQ4iEgOMApaX22Y5MN55PgL4UFVVRBKc4+DMYNcF2OXHMavV5uzj/Csjhwnf6lDj5sEd0b81z4/sS+reo4yfs5YThcVuRzLVLGVXPvfNXktCvVhenzqY1g1D4+o8U3X8KRDvAstEZIiIDAGWOMsqpaolwEN4pyvdAryuqptE5BkRud3ZbDbQWEQygceBJ53l1wAZIvI13s7r76nqkQv4varF8x9sJ75WFJOu7uh2lIC4o28rXhrdj6/3H2Ps7LUUnLYiES6+yDzM+LlraVG/FsumJIXsxRfm0ohq5U33IhKBtyP4BmfRKrz3NLh7KUM5iYmJmpqaWm3HS99/jDte/oInhnblhyFwY9ylWLX5ED94LY2uzeqxaOIgGtawsyXzTR9vy2XqwjTaN45j0aRBJNSLdTuSCSARSVPVRF/r/JmT2qOqr6rqCLyFYk2wFQc3TFu1nYZ1onngWx3cjhJwQ3s0Y8a4RHbknmT0zGQOnzzrdiQTIB9sPsSUBWl0SqjLkilJVhzCnD9zUn/szEndCO9QGzNF5PnARwteqXuO8Mn2PL53bSfqhsksVtd1a8qc8QPYk3+K0TOSyT1uc0bVNO9syOF7i9K4rEU9lkxOqnH9aubC+dMHUd+5vPQuYIGqDgKGBDZWcHvu/e00qRvLuMHt3Y5Srb7VpQnzHhjIgWNnGDUjmYMFViRqiuXp2Ty0ZD192jRg4aRB1K8T7XYkEwT8mpNaRFoA9wArA5wn6H2ZeZg1u/J58LpOYXl/QFLHxiyYMJDcE2e5Z/oaso6edjuSuUR/S8vi0aXr6d+uIfMnDCS+lhUH4+VPgXgG75VImaq6zrnsNCwnNlZVnlu1nRb1azF6oHt3brstsX0jFk4cyNHTRYycnsy+fCsSoWrZun386M10kjo2Zt4DA8KmydT4x59O6jdU9fJzo7qq6i5V/W7gowWfT7bnkbb3KA9e1znsh6Do17YhSyYncaqohJEz1rD78Cm3I5kLtHDNHv7nbxu4pksCc+4fQJ0YKw7mmyosECLyE+fnX0TkxfKP6osYHFSVaau207phbe5JtPmSAHq1qs+SyUkUlXi4Z/oaMnNPuB3J+Gn257v5+T83ccNlTZkxzsbcMr5VdgaxxfmZivfqpfKPsLJq8yEysgp4eEgXm8GqjMtaxLN0ShKqMHJ6MlsPHj//m4yrXv1kJ79auZlhvZrzyr39bdReU6Hz3igXKgJ5o5zHo9zy4mecLfGw6rFrbAx6H3bmnWTMzGSKSjwsnDiIXq3qux3J+PDi6h1MW7Wd2/q05HmbHMpwkTfKicjyyh6Bixt83t6Yw9aDJ3j0hi72D6oCnRLq8vrUwdSJiWLMzGTS9x9zO5IpQ1V57v1tTFu1nbv6teKFkX3t/2VzXpX1Sg3GO1fDEiAFCLmJfKpCqUd54YMddGlal+E2QUml2jWOY9nUJEbPTGbsrBTmTRhA/3Y2Zr/bVJXfv7OV6Z/uYmRiG357V2+be9z4pbI/IZoDPwV64Z1XeihwWFU/UdVPqiNcMFiefoDM3JM8PrSr/aPyQ+uGdXh96mCa1IvlvtlrSdmV73aksKaqPLNyM9M/3cXYpLb8zoqDuQAVFghVLVXVd1V1PJAEZAIfi8hD1ZbOZcWlHl74YAc9WsRzU8/mbscJGS3q13ZGAK3F+Llr+SLzsNuRwpLHo/z8nxuZ+8UeJlzVgV/d0YsIKw7mAlTaCCkisSJyF7AIeBB4Efh7dQQLBm99lcXe/NM8cWNX+4d1gZrG12LplMG0axTHhHnr+HhbrtuRwkqpR3nqrQ0sSt7H1Gs78vPhlxGC070bl1XWSb0AWANcAfxSVQeo6q9UtfyscDXS2ZJSXlydSd82Dbi+e1O344SkhHqxLJmSRKeEukxZkMYHmw+5HSkslJR6+PEb6SxL3c/D13fmyZu7W3EwF6WyM4ixeGdyewT4UkSOO48TIlLjL3Z/fd1+Dhw7w+NDu9o/rkvQKC6GJZOTuKxFPb63KI13N+a4HalGKy718Oiyr3lr/QGeGNqVx2/sZv//motWWR9EhKrWcx7xZR71VDW+OkNWt8LiUl76KJOB7RtxdZcmbscJefXrRLNw0iAub12fBxevZ3l6ttuRaqSiEg8/XLyelRk5PDWse42fyMoEnl0I7cOi5L0cOn6Wx2+0s4eqEl8rmgUTB9G/XUMeXbqet77KcjtSjXK2pJQfvJbGu5sO8ovhPZh6bSe3I5kawApEOaeLSnj1k51c1bkxSR0bux2nRqkbG8W8BwaQ1LExT7yRzuvr9rsdqUYoLC5l8oI0PtiSy6+/04sJYTDLoakeViDKmf/lXg6fLOLxod3cjlIj1YmJYs79A7imSwI/+VsGC5P3uh0ppJ0uKmHCvHV8tiOPZ797OWOT2rkdydQgViDKOFFYzPRPd3JdtwT6t2vodpwaq1Z0JDPG9eeGy5ry839sZM7nu92OFJJOni3h/jnrSN6Vz3N39+GeATbKsKlaViDKmPP5Ho6dLrazh2oQGxXJK/f25+aezb13+n6y0+1IIeV4YTHjZqeQtu8ofx7Vj7uuaO12JFMDWYFwHDtdxKzPdnFTz2b0bm0jkVaHmKgI/jKmH7f1acnv3tnKX1aH5USFF6zgdDH3zUphw4ECXnY+P2MCwaaQcsz8bBcni0p4bGhXt6OElejICF4Y2ZfoCOG5VdspLvXwmN17UqEjp4oYOyuFzNyTvDq2P0Mua+Z2JFODWYEA8k+eZe4Xexh+eUu6N6/Rt3gEpcgI4Y939yE6MoIXP8zkbKnH7v71Ie/EWcbOSmFP/ilmjk/k2q4JbkcyNZwVCLwzbBUWl/LoDXZjkVsiI4Tf3dWb6Chh+ie7KCrx8IvhPaxIOA4dL2TMzGQOHDvDnPsHcFVnu4HTBF7YF4jc44UsWLOXO/u1plNCXbfjhLWICOFXd/QiOjKCuV/sobjUwzO32wikOQVnGDMzhdzjhcx/YCCD7P4cU03CvkAcPV1Mz5bxPGLDEgQFEeEXw3sQExXB9E92UVKq/PbO3mFbJPYfOc2YWckcO1XMgokDbQImU63CvkB0a16Pt35wldsxTBkiwpM3dyfW6ZMoKvXwxxF9wm6im735pxgzM4UThcUsmjSIPm0auB3JhJmwLxAmOIkIj9/YjajICKat2k5xqTLtHm9HdjjYmXeSMTOTKSrxsHhyEr1a2aXXpvoF9F+biNwsIttEJFNEnvSxPlZEljnrU0Skfbn1bUXkpIj8KJA5TfB6eEgXnhzWnRXp2Ty8ZD1FJR63IwXcjkMnGDk9mVKPsmSKFQfjnoAVCBGJBF4GhgE9gNEi0qPcZhOBo6raGXge+EO59dOAdwKV0YSG713biZ8P78E7Gw/yg9fSOFtS6nakgNmSc5xRM5KJEFg6JckuuzauCuQZxEAgU1V3qWoRsBS4o9w2dwDznedvAkPEua5RRL4D7AY2BTCjCRETv9WBX32nFx9syWXKgjQKi2tekdh4oIDRM5OJiYpg2dTBdG5az+1IJswFskC0AsqO55zlLPO5jaqWAAVAYxGpC/wP8MsA5jMh5r6kdvzhu735dEceE+ev43RRiduRqsz6fUcZPTOZuJgoXp86mA5N4tyOZEzQjsX0NPC8qp6sbCMRmSIiqSKSmpeXVz3JjKtGDmjLc3f3Yc3OfO6fu46TZ0O/SKTuOcJ9s9fSsE4My6Ym0aZRHbcjGQMEtkAcAMqOP9zaWeZzGxGJAuoD+cAg4FkR2QM8CvxURB4qfwBVnaGqiaqamJBgww6Ei7uuaM0Lo/qRtvco42ancLyw2O1IF23NznzGzVlL03qxvD51MK0bWnEwwSOQBWId0EVEOohIDDAKWF5um+XAeOf5COBD9bpaVduranvgBeC3qvpSALOaEHN7n5a8NLofGVkF3DcrhYLToVckPt9xmAfmraVVg9osnZpE8/q13I5kzDcErEA4fQoPAe8BW4DXVXWTiDwjIrc7m83G2+eQCTwO/NelsMZUZFjvFrw6tj9bck4wZlYyR04VuR3Jbx9tzWXC/HW0bxzH0ilJNK1nxcEEH1FVtzNUicTERE1NTXU7hnHBx9tymbowjQ5N4lg0aRBN6sa6HalS7286yIOLv6Jb83osnDCIhnExbkcyYUxE0lQ10de6YO2kNsZv3+7WlDn3D2BP/ilGzUgm93ih25Eq9PaGHH7w2lf0aFmf1yYlWXEwQc0KhKkRrurchHkPDCT72BlGzkgmp+CM25H+yz+/PsAPl6ynb5sGLJo4kPq1o92OZEylrECYGiOpY2MWThzI4RNnGTk9mayjp92O9G9vpmXx6LKvGdC+IfMnDKReLSsOJvhZgTA1Sv92jVg0aRDHThcxcnoye/NPuR2JJWv38eM307mqUxPm3j+QuFgbI9OEBisQpsbp06YBiycncbqohJHTk9mZV+n9lgG1YM0ennprA9d2TWDW+ERqx0S6lsWYC2UFwtRIvVrVZ8mUJIpLPYyakcyOQyeqPcOsz3bxi39uYmiPZky/rz+1oq04mNBiBcLUWN2bx7N0ShIAo2YksyXneLUd+5WPM/n1v7ZwS+/mvHLvFcRGWXEwoccKhKnRujSrx7IpSURHRjB6ZjIbDxQE/Jh//mAHz767jTv6tuTFUf3CZpIjU/PY/7mmxuuYUJfXpw4mLiaKMTOT+Xr/sYAcR1X503vbeP6D7Xz3itZMu6cvUVYcTAiz/3tNWGjbuA7LpibRoE4MY2elkLrnSJXuX1X53TtbeemjTEYPbMMfR1wednNom5rHCoQJG60b1uH1qYNpWi+WcXPWkrwrv0r2q6r8csVmZny6i3GD2/Gb7/QmwoqDqQGsQJiw0rx+LZZOSaJVg9rcP3ctn+84fEn783iUn/1jI/O+3MOkb3Xgl7f3tOJgagwrECbsNI2vxZIpSbRvHMeE+ev4aFvuRe2n1KM8+VYGr6Xs4/vf7sT/3noZzoy5xtQIViBMWGpSN5Ylk5Po2qwuUxeksWrzoQt6f0mphx+9kc7rqVk8MqQLP7mpmxUHU+NYgTBhq2FcDK9NSuKylvF8f1Ea72zI8et9xaUeHln2NX9ff4Af39SNx4Z2teJgaiQrECas1a8dzaKJA+nTpgEPLVnPP78uPyvuNxWVeHho8Vf8KyOH/73lMh68rnM1JTWm+lmBMGGvXq1oFkwYSGK7hjy27GveTMvyuV1hcSnfX5TGe5sO8fRtPZh8TcdqTmpM9bICYQwQFxvFvAcGcmWnJvz4zXSWrt33jfWFxaVMXpDK6q25/ObOXtx/VQeXkhpTfaxAGOOoHRPJrPGJXNs1gSff2sCCNXsAOF1UwgNz1/F55mGeHXE59w5q525QY6qJDUxvTBm1oiOZfl9/Hlq8nl/8cxMnCkv4ZFseqXuPMO2ePtzZr7XbEY2pNnYGYUw5sVGRvHLvFdzSuzl/fG8bafuO8uLoflYcTNixMwhjfIiOjODFUf3o0jSTvm0bcF23pm5HMqbaWYEwpgJRkRE8NrSr2zGMcY01MRljjPHJCoQxxhifrEAYY4zxyQqEMcYYn6xAGGOM8ckKhDHGGJ+sQBhjjPHJCoQxxhifRFXdzlAlRCQP2Ot2jkvUBLi0SZJrFvs8vsk+j/+wz+KbLuXzaKeqCb5W1JgCUROISKqqJrqdI1jY5/FN9nn8h30W3xSoz8OamIwxxvhkBcIYY4xPViCCywy3AwQZ+zy+yT6P/7DP4psC8nlYH4Qxxhif7AzCGGOMT1YgjDHG+GQFIgiISBsR+UhENovIJhF5xO1MbhORSBFZLyIr3c7iNhFpICJvishWEdkiIoPdzuQmEXnM+XeyUUSWiEgttzNVJxGZIyK5IrKxzLJGIrJKRHY4PxtWxbGsQASHEuAJVe0BJAEPikgPlzO57RFgi9shgsSfgXdVtTvQhzD+XESkFfAwkKiqvYBIYJS7qardPODmcsueBFarahdgtfP6klmBCAKqmqOqXznPT+D9Amjlbir3iEhr4FZglttZ3CYi9YFrgNkAqlqkqsdcDeW+KKC2iEQBdYBsl/NUK1X9FDhSbvEdwHzn+XzgO1VxLCsQQUZE2gP9gBSXo7jpBeAngMflHMGgA5AHzHWa3GaJSJzbodyiqgeAPwH7gBygQFXfdzdVUGimqjnO84NAs6rYqRWIICIidYG/AY+q6nG387hBRIYDuaqa5naWIBEFXAH8VVX7AaeoouaDUOS0rd+Bt3C2BOJEZKy7qYKLeu9dqJL7F6xABAkRicZbHF5T1bfczuOiq4DbRWQPsBS4XkQWuRvJVVlAlqqeO6N8E2/BCFc3ALtVNU9Vi4G3gCtdzhQMDolICwDnZ25V7NQKRBAQEcHbxrxFVae5ncdNqvqUqrZW1fZ4Ox8/VNWw/QtRVQ8C+0Wkm7NoCLDZxUhu2wckiUgd59/NEMK4076M5cB45/l44J9VsVMrEMHhKuA+vH8tf+08bnE7lAkaPwReE5EMoC/wW3fjuMc5k3oT+ArYgPc7LKyG3RCRJcAaoJuIZInIROD3wFAR2YH3LOv3VXIsG2rDGGOML3YGYYwxxicrEMYYY3yyAmGMMcYnKxDGGGN8sgJhjDHGJysQxlRARNqXHTGzgm12lblH4dyyF0Tkfyp5zx4RaVJVOY0JFCsQxlyapZQZTVREIoARznJjQpoVCGP8ICIdncHyBpRbtQQYWeb1NcBeVd0rIv8QkTRn7oIpPvb5jTMUEfmRiDztPO8kIu867/9MRLoH4NcyplJRbgcwJtg5TUhLgftVNb3sOlXdICIeEenjrBuFt2gATFDVIyJSG1gnIn9T1Xw/DzsD+J6q7hCRQcArwPVV8xsZ4x8rEMZULgHvuDZ3qWpFYyAtAUaJyCa84/D/n7P8YRG503neBugCnLdAOKP6Xgm84R1uCIDYi0pvzCWwAmFM5QrwDhD3LSoeJG8p8D7wCZChqodE5Nt4x8QZrKqnReRjoPzUmCV8s5n33PoI4Jiq9q2C/MZcNOuDMKZyRcCdwDgRGeNrA1XdCRzGO0Daueal+sBRpzh0xzuVbHmHgKYi0lhEYoHhzv6OA7tF5G7wjvYrIn2q8pcyxh9WIIw5D1U9hffL+zERub2CzZYA3fHOTwDwLhAlIlvwFo5kH/stBp4B1gKrgK1lVt8LTBSRdGAT3klyjKlWNpqrMcYYn+wMwhhjjE9WIIwxxvhkBcIYY4xPViCMMcb4ZAXCGGOMT1YgjDHG+GQFwhhjjE//D+IBdQm5lnMjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Examine CV results\n",
    "iris_cv.get_cv_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best k: 5\n",
      "CV Misclassification Rate: 0.03333333333333333\n"
     ]
    }
   ],
   "source": [
    "# Get best K\n",
    "   # This sets \"k\" instance attribute to allow for assessment on test set using tuned k\n",
    "iris_cv.get_best_k()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.8666666666666667\n",
      "Test misclassification rate: 0.13333333333333333\n",
      "Num correct predictions: 26\n",
      "Num incorrect predictions: 4\n"
     ]
    }
   ],
   "source": [
    "# Examine classifier performance on test set using tuned k hyperparameter\n",
    "mm.assessment_metrics(iris_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6-fold CV indicated we should change our k hyperparameter from 8 to 5\n",
    "   # Improved model test accuracy by 3.33%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example 2: KNN regressor using auto_mpg dataset**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Create model and perform train/test split*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created KNN regressor with k=7!\n"
     ]
    }
   ],
   "source": [
    "# Create classifier with k=7\n",
    "mpg_regressor=knn.KNN('regressor',7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset successfully loaded!\n"
     ]
    }
   ],
   "source": [
    "# Load dataset into regressor and set response variable to 'mpg'\n",
    "mpg_regressor.load_csv('datasets/auto_mpg.csv','mpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully completed train/test split!\n",
      "Training set: 294 samples\n",
      "Test set: 98 samples\n"
     ]
    }
   ],
   "source": [
    "# Perform 75/25 train/test split\n",
    "mpg_regressor.train_test_split(0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Generate predictions*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18.957142857142856"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate single prediction from fit on full set\n",
    "gp.generate_prediction(mpg_regressor,[6,250,100,3282,15],'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([18.64285714, 17.98571429, 31.18571429, 29.94285714])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate multiple predictions from fit on training set\n",
    "mpg_to_predict=np.array([[6,250,100,3282,15],[6,250,88,3139,14.5],\n",
    "                        [4,122,86,2220,14],[4,116,90,2123,14]])\n",
    "gp.generate_predictions(mpg_regressor,mpg_to_predict,'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assess model performance on these 4 values\n",
    "actual_mpg=[19,18,23,28]\n",
    "predicted_mpg=gp.generate_predictions(mpg_regressor,mpg_to_predict,'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root mean squared error\n",
      "4.210355310034379\n",
      "Mean squared error\n",
      "17.727091836734697\n",
      "Mean absolute percent error\n",
      "11.121975487372675\n",
      "Mean absolute error\n",
      "2.625\n"
     ]
    }
   ],
   "source": [
    "# Compute RMSE, MSE, MAPE, MAE\n",
    "print('Root mean squared error')\n",
    "print(mm.model_rmse(actual_mpg,predicted_mpg))\n",
    "print('Mean squared error')\n",
    "print(mm.model_mse(actual_mpg,predicted_mpg))\n",
    "print('Mean absolute percent error')\n",
    "print(mm.model_mape(actual_mpg,predicted_mpg))\n",
    "print('Mean absolute error')\n",
    "print(mm.model_mae(actual_mpg,predicted_mpg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Assess model performance on test set*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test MSE: 20.935974593919205\n",
      "Test RMSE: 4.575584617720364\n",
      "Test MAE: 3.3689504373177845\n",
      "Test MAPE: 14.555844306759363%\n"
     ]
    }
   ],
   "source": [
    "mm.assessment_metrics(mpg_regressor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Tune k hyperparameter using 8-fold cross validation*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created CV instance for KNN regressor with 8 folds!\n"
     ]
    }
   ],
   "source": [
    "# Create CV instance for 8-fold CV\n",
    "mpg_cv=cv.CvKNN('regressor',8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset successfully loaded!\n",
      "Successfully completed train/test split!\n",
      "Training set: 314 samples\n",
      "Test set: 78 samples\n"
     ]
    }
   ],
   "source": [
    "# Load dataset into CV instance and set response variable to \"mpg\"\n",
    "mpg_cv.load_csv('datasets/auto_mpg.csv','mpg')\n",
    "# Perform 75/25 train/test split\n",
    "mpg_cv.train_test_split(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully performed 8 CV!\n"
     ]
    }
   ],
   "source": [
    "# Perform 8-fold CV using k values 2-12\n",
    "mpg_cv.perform_cv(np.arange(2,13))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k=2: 23.49831730769231\n",
      "k=3: 22.150797720797723\n",
      "k=4: 22.0321875\n",
      "k=5: 20.7240782051282\n",
      "k=6: 20.700193198005696\n",
      "k=7: 20.746401753008897\n",
      "k=8: 21.00656650641026\n",
      "k=9: 20.92890115542893\n",
      "k=10: 20.67497884615385\n",
      "k=11: 20.359530356007628\n",
      "k=12: 20.526462784900286\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtCElEQVR4nO3deZwU9Z3/8ddnTmBmYBiYgZkeTkEQEZiRiHciJqgodw6jMWaziY/satTExFy7m72ycaPrmsRkE6PZJIbo5idoxAMhKvGIooDch9zXDMwAAsM15+f3Rxc6Qs8BdHfN8X4+Hv2Yquqq6k8l2O+ub33rW+buiIiInCgl7AJERKRtUkCIiEhMCggREYlJASEiIjEpIEREJCYFhIiIxKSAkA7FzP7OzHab2SEz69XCugvM7EtNvDfQzNzM0hJTafjM7GNmtiPsOqTtUkBImxZ8UT9nZu+Z2S4ze7CpL20zSwfuBya4e7a7701wbTeY2aIgjMrN7Hkzu9TMrjezLWZmJ6yfZmYVZnZdjH19wczqg30dNLNlsdZLpKDmjyfzM6VtU0BIW/dzoAIoBMYAHwX+vol1+wBdgFWJLsrMvg48APxH8Ln9idY6BXgKyCVaa2NXAw7MbWK3b7h7drDtz4HHzSw3roWLnAIFhLR1g4A/uvsxd99F9Mv13BNXMrOzgXXB7H4zeylYfrGZvW1mB4K/F8f6EDNLNbP7zGyPmW0Crm2qIDPrAfwrcKu7z3b3w+5e6+5z3P2b7n4M+CPw+RM2/TzwB3eva+6A3b0BeBTIAoYGn5kZ1LctaEL7hZl1Dd7rbWbPmNl+M9tnZq+aWUrwnpvZkEa1/8bM/j3GMT1KNOTmBGcxd5tZFzP7vZntDfb9tpn1aa526VgUENLWPQBcb2bdzCwCXEOMX+Du/i4fBEeuu483szzgWeAnQC+izU/PNnFt4svAdUAJMBb4ZDM1XUT0TOXJZtb5LfDJRl/iPYBJwfJmmVkq8DdALbA1WHwPcDbRs6ghQAT4p+C9u4AdQD7Rs5nvEj1TaTV3vwnYBkwKmud+BNwM9AD6Ef3f7yvA0VPZr7RvCghp614h+sV/kOiX4CKiTTitcS2w3t0fdfc6d38MWEv0i/pEnwYecPft7r4P+GEz++0F7GnuTMDdXwd2A9Ma7f9dd1/azH4vNLP9wDHgPuBz7l4RXMu4Bfiau+9z9yqiTVvXB9vVEm2CGxCcybzq8RlkrZbosQ5x93p3X+zuB+OwX2knFBDSZgXNJHOB2USbW3oDPYH/bOUuivjgF/hxW4n++o617vYT1mvKXqB3K3o4/Y4PmpluCuab86a75xI9xqeBy4Ll+UA3YHHQ1LOf6P8u+cH79wIbgHlmtsnMvt3C57TWo8ALRK+FlJnZj4KOANJJKCCkLcsj2i7+oLtXB72S/heY2Mrty4ABJyzrD+yMsW450aaUxus15Q2gGpjawuc/ClxpZhcBFwIzW1gfAHc/BPwdcJOZlQB7iDbtnOvuucGrR3BBG3evcve73H0wMBn4upldGezuCNFwOa5vcx99Qh217v4v7j4CuJhoE9yJ11WkA1NASJvl7nuAzcDfBV1Ec4m2iy9v5S6eA84OuqOmmdlngBHAMzHW/SNwu5kVm1lPoMlf4e5+gGj7/8/MbGpwfSTdzK4xsx81Wm8L8BrwGDA/uMjeKkEz18PAPwUXrX8F/LeZFQCYWcTMrgqmrzOzIUFT1AGgHmgIdrUUuCG4CH81J/esamw3MPj4jJldYWbnBddEDhJtcmpoamPpeBQQ0tZNJ9o9tJJoM0ot8LXWbBiccVxH9CLuXuBu4LogeE70K6LNKcuAJUSbtZrb938BXwf+IahtO3AbJ18f+S3Rs5iWmpdieQCYaGajgG8RPf43zewg8GdgWLDe0GD+ENGzm5+7+8vBe3cQveayH7gxRn2N/RD4h6AZ6xtEzzaeIBoOa4C/ED0rkk7C9MAgERGJRWcQIiISkwJCRERiUkCIiEhMCggREYmpQw1l3Lt3bx84cGDYZYiItBuLFy/e4+75sd7rUAExcOBAFi1aFHYZIiLthpk1OWqAmphERCQmBYSIiMSkgBARkZgUECIiEpMCQkREYkpYQJhZPzN72cxWm9kqM7sjWP5vZrbczJaa2TwzK2pi+/pgnaVm9nSi6hQRkdgS2c21DrjL3ZeYWQ7Rh53MB+51938EMLPbiQ6b/JUY2x919zEJrE9ERJqRsDMIdy939yXBdBXR4YIjJzyyMItTfHZuvB2rreehVzby1w2xRoAWEem8knINwswGEn0Y/MJg/gdmtp3o+PT/1MRmXcxskZm9aWZTm9n3LcF6iyorK0+5tvTUFH716mZ++8aWU95WRKQjS3hAmFk2MAu48/jZg7t/z937EX0E421NbDrA3ccCNwAPmNlZsVZy94fcfay7j83Pj3m3eLNSU4zJo4t4eW0l+4/UnPL2IiIdVUIDInjA+SxgprvHekLXTGBGrG3dfWfwdxOwgOgZSEJMK4lQU9/Acyta/URIEZEOL5G9mAx4BFjj7vc3Wj600WpTgLUxtu1pZpnBdG/gEmB1omo9t6g7QwqyeeqdWM+yFxHpnBJ5BnEJcBMwvlF31YnAPWa20syWAxOIPjMXMxtrZg8H254DLDKzZcDLwD3unrCAMDOmlUR4a8s+drx3JFEfIyLSriSsm6u7vwZYjLeea2L9RcCXgum/AuclqrZYJo8u4t4X1vGnpWXcesWQZH60iEibpDupA/3yunHBwDyefGcn7qH2vBURaRMUEI1MKSliQ8UhVpUdbHllEZEOTgHRyLXnFZKearpYLSKCAuJDcrtlcMWwAp5eVkZ9g5qZRKRzU0CcYFpJhIqqat7YuDfsUkREQqWAOMEVwwvI6ZLGk2pmEpFOTgFxgi7pqUwcWcjcleUcrakPuxwRkdAoIGKYWhLhcE0989fsDrsUEZHQKCBiGDcoj8IeXdSbSUQ6NQVEDCkpxpQxEV55t5K9h6rDLkdEJBQKiCZMK4lQ1+A8u6I87FJEREKhgGjCsL45DO+bo95MItJpKSCaMa0kwjvb9rNlz+GwSxERSToFRDMmjynCDJ5aqrMIEel8FBDNKOzRlYsG9+IpjfAqIp2QAqIFU0sibNl7hGU7DoRdiohIUikgWnD1yL5kpKXonggR6XQUEC3o3iWdT5zThznLyqitbwi7HBGRpFFAtMLUkgh7D9fw2vo9YZciIpI0CohW+OjZ+eR2S9c9ESLSqSggWiEjLYXrRhUyb/UuDlXXhV2OiEhSKCBaaeqYCMdqG5i3alfYpYiIJEXCAsLM+pnZy2a22sxWmdkdwfJ/M7PlZrbUzOaZWVET299sZuuD182JqrO1zh/Qk+KeXdXMJCKdRiLPIOqAu9x9BHAhcKuZjQDudfdR7j4GeAb4pxM3NLM84PvAOOAC4Ptm1jOBtbbIzJhWEuH1DXuoOHgszFJERJIiYQHh7uXuviSYrgLWABF3P9hotSwg1i3KVwHz3X2fu78HzAeuTlStrTVlTIQGh6eXlYVdiohIwiXlGoSZDQRKgIXB/A/MbDtwIzHOIIAIsL3R/I5gWaiGFGQzqriHxmYSkU4h4QFhZtnALODO42cP7v49d+8HzARuO8P932Jmi8xsUWVl5ZkX3IKpYyKs3HmQDRVVCf8sEZEwJTQgzCydaDjMdPfZMVaZCcyIsXwn0K/RfHGw7CTu/pC7j3X3sfn5+WdacouuG11IisFT76iZSUQ6tkT2YjLgEWCNu9/faPnQRqtNAdbG2PwFYIKZ9QwuTk8IloWuIKcLlw7N56mlO2lo0AivItJxJfIM4hLgJmB80KV1qZlNBO4xs5VmtpzoF//x7q9jzexhAHffB/wb8Hbw+tdgWZswraSIHe8dZfG298IuRUQkYdIStWN3fw2wGG8918T6i4AvNZr/NfDrxFR3ZiaM6EvX9JU8+c5OPjIwL+xyREQSQndSn4aszDSuOrcPzy4vp6ZOI7yKSMekgDhNU0oiHDhay4J1FWGXIiKSEAqI03TZkN70ysrQPREi0mEpIE5TWmoKk0YX8ec1FRw4Wht2OSIicaeAOAPTSiLU1DUwd2V52KWIiMSdAuIMjCruweDeWRrhVUQ6JAXEGTAzpoyJsHDzPsr2Hw27HBGRuFJAnKGpJUW4RngVkQ5IAXGGBvTKorR/Lk+pmUlEOhgFRBxMK4mwdlcVa8oPtryyiEg7oYCIg2tHFZGWYjqLEJEORQERB3lZGXz07Hz+tLSMeo3wKiIdhAIiTqaWRNh18BgLN+8NuxQRkbhQQMTJx8/pQ3ZmmpqZRKTDUEDESdeMVK4e2ZfnV+ziWG192OWIiJwxBUQcTSuJUFVdx4trNMKriLR/Cog4unBwLwpyMjX0hoh0CAqIOEpNMaaMKeIv71bw3uGasMsRETkjCog4m1oSobbeeXaFRngVkfZNARFnIwq7c3afbPVmEpF2TwERZ2bG1JIIi7a+x7a9R8IuR0TktCkgEmDKmAgAf9LjSEWkHVNAJEAktysXDMrjyaU7cdfQGyLSPiUsIMysn5m9bGarzWyVmd0RLL/XzNaa2XIze9LMcpvYfouZrTCzpWa2KFF1Jsq0kgibKg+zcqdGeBWR9imRZxB1wF3uPgK4ELjVzEYA84GR7j4KeBf4TjP7uMLdx7j72ATWmRATRxaSkZqieyJEpN1KWEC4e7m7Lwmmq4A1QMTd57l7XbDam0BxomoIU49u6YwfXsDTy8qoq28IuxwRkVOWlGsQZjYQKAEWnvDWF4Hnm9jMgXlmttjMbmlm37eY2SIzW1RZWRmXeuNlakmEPYeqeX2jRngVkfYn4QFhZtnALOBOdz/YaPn3iDZDzWxi00vdvRS4hmjz1OWxVnL3h9x9rLuPzc/Pj3P1Z+aK4fl076IRXkWkfUpoQJhZOtFwmOnusxst/wJwHXCjN9HNx913Bn8rgCeBCxJZayJkpqVy7ahCXli1iyM1dS1vICLShiSyF5MBjwBr3P3+RsuvBu4GJrt7zDvJzCzLzHKOTwMTgJWJqjWRpo6JcKSmnvmrd4ddiojIKUnkGcQlwE3A+KCr6lIzmwg8COQA84NlvwAwsyIzey7Ytg/wmpktA94CnnX3uQmsNWE+MjCPSG5X9WYSkXYnLVE7dvfXAIvx1nMxluHuZcDEYHoTMDpRtSVTSjDC6y9f2URlVTX5OZlhlyQi0iq6kzoJppZEqG9wnlleFnYpIiKtpoBIgrP75DCisLt6M4lIu6KASJJpJRGW7TjApspDYZciItIqCogkmTymCDN4aqmamUSkfVBAJEmf7l245KzePPWORngVkfZBAZFEU8YUsW3fEZZs2x92KSIiLVJAJNHVI/uSmZaii9Ui0i4oIJIop0s6nxjRh2eWl1GrEV5FpI1TQCTZtJII7x2p5ZV329bIsyIiJ1JAJNnlZ+fTs1u6ht4QkTZPAZFk6akpXDeqiPmrd1N1rDbsckREmqSACMHUkgjVdQ3MXbkr7FJERJqkgAhBaf9c+ud146mlamYSkbZLARECM2NqSYS/btzL7oPHwi5HRCQmBURIpo4pwh2e1tAbItJGJex5ENK8wfnZjO6Xy6NvbqV3TgbjBvWiKLdr2GWJiLxPARGi28cP4c7/W8rX/m8ZAP3yujJuUC/GDcpj3KBe9MvrSvTJrSIiyWcdaeC4sWPH+qJFi8Iu45TUNzhryg+ycPM+Fm7ay1tb9rH/SLT7a2GPLtGwGBwNjUG9sxQYIhJXZrbY3cfGfE8B0bY0NDjrKw6xcPNeFm7ax8LNe9lzqAaA/JxMLhiUx4VBaAwtyFZgiMgZUUC0Y+7OxsrDLNy8l7c272Phpn3sCno+5WVlcMHAPMYNzuOCQXmc07c7KSkKDBFpveYCQtcg2jgzY0hBNkMKsrlx3ADcnW37jrBw0z7eDM4y5q6K3nDXvUsaFwTXL8YNzmNEYXfSUtVRTUROjwKinTEzBvTKYkCvLD79kX4A7HjvyPtnFws37+XPayoAyM5M4/wBPRk3OBoao4p7kK7AEJFWaraJycw+5+6/D6YvcffXG713m7s/2My2/YDfAX0ABx5y9x+b2b3AJKAG2Aj8jbvvj7H91cCPgVTgYXe/p6WD6YhNTKdj98FjvLkpaJLavI8NFdHnYHdNT40GxqA8LjqrF+cP6KlrGCKd3GlfgzCzJe5eeuJ0rPkY2xYChe6+xMxygMXAVKAYeMnd68zsPwHc/VsnbJsKvAt8AtgBvA181t1XN3egCojY9hyqDs4w9rJw8z7W7qoC4AfTRnLjuAEhVyciYTqTaxDWxHSs+Q9x93KgPJiuMrM1QMTd5zVa7U3gkzE2vwDY4O6bAMzscWAK0GxASGy9szOZeF4hE88rBGD/kRpufHghM9/cpoAQkSa11CDtTUzHmm+SmQ0ESoCFJ7z1ReD5GJtEgO2N5ncEyyQOcrtl8Knzi1ldfpA15QfDLkdE2qiWAmK4mS03sxWNpo/PD2vNB5hZNjALuNPdDzZa/j2gDph5mrUf388tZrbIzBZVVuopba01eUyEtBRj9pIdYZciIm1US01M55zJzs0snWg4zHT32Y2WfwG4DrjSY18E2Qn0azRfHCw7ibs/BDwE0WsQZ1JvZ5KXlcEVwwt4amkZ37p6uLrDishJmv1WcPetjV/AIaAU6B3MN8mi3WMeAda4+/2Nll8N3A1MdvcjTWz+NjDUzAaZWQZwPfB0q49KWmVGaTGVVdW8umFP2KWISBvUbECY2TNmNjKYLgRWEr1u8KiZ3dnCvi8BbgLGm9nS4DUReBDIAeYHy34R7L/IzJ4DcPc64DbgBWAN8Ed3X3W6BymxXTE8n9xu6cxarGYmETlZS01Mg9x9ZTD9N8B8d/980G31deCBpjZ099eI3dPpuSbWLwMmNpp/rql1JT4y01KZPLqIx9/ezoGjtfTomh52SSLShrTU8FzbaPpKgi9sd68CGhJVlCTPjNJiauoaeG5FediliEgb01JAbDezr5rZNKLXHuYCmFlXQD83O4BRxT04Kz9LvZlE5CQtBcTfAucCXwA+02hIjAuB/01cWZIsZsaM84t5e8t7bN17OOxyRKQNaakXU4W7f8XdpzS+A9rdX3b3+xJfniTDtJIIZjB7ScyexCLSSTV7kdrMmu1a6u6T41uOhKGwR1cuOas3s9/ZwR1XDtUzJUQEaLkX00VEh7x4jOgwGfrm6KCml0b4+h+X8faWfYwb3CvsckSkDWjpGkRf4LvASKJDb38C2OPuf3H3vyS6OEmeq0f2JSsjVc1MIvK+lq5B1Lv7XHe/meiF6Q3AAjO7LSnVSdJ0y0jjmvMKeXZFOUdr6sMuR0TagBYH4DGzTDObDvweuBX4CfBkoguT5JteGuFQdR3zVu8KuxQRaQNaukj9O6LNS88B/9LormrpgC4c1ItIbldmLdnJlDEaXV2ks2vpDOJzwFDgDuCvZnYweFWZmR4k0MGkpBjTSiK8tr6S3QePhV2OiISspWsQKe6eE7y6N3rluHv3ZBUpyTO9NEKDw1Pv6GK1SGenhwDIhwzOz6a0fy6zluygueeVi0jHp4CQk0wvLebd3YdYVaZWRJHOTAEhJ5k0qoiM1BSe0HMiRDo1BYScpEe3dD4+ooCnl5VRW69R3UU6KwWExDSjtJh9h2tYsK4y7FJEJCQKCInp8rPz6ZWVoedEiHRiCgiJKT01hSljIry4poL9R2rCLkdEQqCAkCbNOD9CTX0Dc5aVhV2KiIRAASFNGlHYneF9c5ilEV5FOiUFhDTJzJhRWszS7fvZWHko7HJEJMkUENKsKWOKSDF0sVqkE0pYQJhZPzN72cxWm9kqM7sjWP6pYL7BzMY2s/0WM1thZkvNbFGi6pTmFXTvwuVn5/Pkkp00NGjoDZHOJJFnEHXAXe4+gujDhm41sxHASmA68Eor9nGFu49x9yaDRBJvemkxZQeO8eamvWGXIiJJlLCAcPdyd18STFcBa4CIu69x93WJ+lyJvwkj+pDTJY0n1Mwk0qkk5RqEmQ0ESoCFp7CZA/PMbLGZ3dLMvm8xs0VmtqiyUnf9JkKX9FSuG1XI3JW7OFxdF3Y5IpIkCQ8IM8sGZgF3uvupDA96qbuXAtcQbZ66PNZK7v6Qu49197H5+flxqFhimV5azJGaeuau1ONIRTqLhAaEmaUTDYeZ7j77VLZ1953B3wqiz8C+IP4VSmuNHdCT/nndmKVmJpFOI5G9mAx4BFjj7vef4rZZZpZzfBqYQPTitoTEzJheGuGNTXvZuf9o2OWISBIk8gziEuAmYHzQVXWpmU00s2lmtgO4CHjWzF4AMLMiM3su2LYP8JqZLQPeAp5197kJrFVaYXpJMa7HkYp0GmmJ2rG7vwZYE28/GWP9MmBiML0JGJ2o2uT09O/VjQsG5jFryQ7+/mNnET1JFJGOSndSyymZcX6ETZWHWbp9f9iliEiCKSDklEw8r5DMtBRdrBbpBBQQckpyuqRz1bl9mbOsnOq6+rDLEZEEUkDIKZtxfjEHjtby0pqKsEsRkQRSQMgpu3RIbwpyMvWcCJEOTgEhpyw1xZhWEmHBugr2HqoOuxwRSRAFhJyW6aXF1DU4T+txpCIdlgJCTsuwvjmMjHRXbyaRDkwBIadtRmkxK3ceZN2uqrBLEZEEUEDIaZs8uoi0FNPjSEU6KAWEnLZe2Zl8bFgBT76zk7r6hrDLEZE4U0DIGZlRGqGiqprXN+pxpCIdjQJCzsj4cwro0TWdWYvVzCTS0Sgg5IxkpqUyeXQRL6zaRdWx2rDLEZE4UkDIGZteGqG6roHnVpSHXYqIxJECQs7YmH65DM7PYtZiDb0h0pEoIOSMmRkzSot5a8s+tu09EnY5IhInCgiJi6klEcxg9ju6WC3SUSggJC4iuV25aHAvZi/ZibuHXY6IxIECQuJmRmkx2/YdYdHW98IuRUTiQAEhcXP1yL50y0jV0BsiHYQCQuImKzONq0f25Zll5Ryr1eNIRdq7hAWEmfUzs5fNbLWZrTKzO4LlnwrmG8xsbDPbX21m68xsg5l9O1F1Snx9srSYquo65q3eHXYpInKGEnkGUQfc5e4jgAuBW81sBLASmA680tSGZpYK/Ay4BhgBfDbYVtq4Cwf3oqhHFzUziXQAaYnasbuXA+XBdJWZrQEi7j4fon3nm3EBsMHdNwXrPg5MAVYnql6Jj5QUY1pphP9ZsJGKg8co6N4l7JI6reU79vPsinKKenRlSEE2QwqyKcjJbOm/PZH3JSwgGjOzgUAJsLCVm0SA7Y3mdwDj4lyWJMj00mJ+9vJG/rS0jC9fPjjscjqdLXsOc++8dTy7vBwzaNzrOKdLWjQs8rPfD40hBdkU9+xGaoqCQz4s4QFhZtnALOBOdz+YgP3fAtwC0L9//3jvXk7DWfnZjOmXy6wlO/jSZYP0izVJKqqO8dMXN/DYW9tIT03h9vFD+PLlgzlSU8/63YfYUFHFhspDbKg4xMvrKvl/jUbgzUxLYVDvLIb2yflQeAzs3Y3MtNQQj0rClNCAMLN0ouEw091nn8KmO4F+jeaLg2UncfeHgIcAxo4dqzu02ogZ5xfzj0+tZHX5Qc4t6hF2OR1a1bFafvXqZh5+dRM1dQ1cf0E/br9yKAU50ea9nC7p9OnehUuH9v7QdvuP1LCx8lAQHofYUHmId7a9x5xlZe+vk5pi9M/rxln52Qzt88GZx1kF2WRnJqUBQkKUsP+HLfqz8RFgjbvff4qbvw0MNbNBRIPheuCGOJcoCTRpVCH/Nmc1sxbvVEAkSHVdPX9YuI2fvrSBfYdruHZUId+YMIxBvbNatX1utwzOH5DH+QPyPrT8aE09GysPnRQeC9ZVUNfwwW+wwh5domFxQnj0ys6M63FKeBL5E+AS4CZghZktDZZ9F8gEfgrkA8+a2VJ3v8rMioCH3X2iu9eZ2W3AC0Aq8Gt3X5XAWiXOcrtlcOU5BTy9bCffmTic9FTdchMvDQ3OnOVl3DdvHdv3HeWiwb349jXDGd0vNy7775qRyshID0ZGPhzstfUNbN17hA0Vx8Mj2mT1f29v52ij+156dktnSEE2V53bl89dOIAu6Wqiaq+sI42bM3bsWF+0aFHYZUhg/urdfPl3i3jk5rFceU6fsMtp99ydV9bv4T+fX8vq8oOMKOzOt64ZzuVDe4d6naehwSk7cDR6phGEx6qygyzfcYC+3btw+5VD+dTYYv1IaKPMbLG7x7wnTY2IkjAfG5ZPXlYGs5bsUECcoeU79nPP82v568a9FPfsyo+vH8OkUUWktIGeRykpRnHPbhT37MbHhhW8v/yNjXv50Qtr+e6TK3jolY187RNnt5mapXUUEJIw6akpTB5dxB8WbuPAkVp6dEsPu6R2Z/Oew9wXdFnNy8rg+5NGcMO4/u2iZ9FFZ/Vi9t9dzItrKrhv3jrueHwpv/jLJr551dlcMaxAvdvaAQWEJNQnzy/mN3/dwpzlZXzuwgFhl9NuNO6ympGWwu1XDuXLlw0ip0v7Clkz4+Mj+jB+eAFzlpdx//x3+eJvFjF2QE++edUwxg3uFXaJ0gxdg5CEcneufuBVsjJTmf33l4RdTptXdayWX72yiYdf20xNXQOfvaA/X71yyPtdVtu72voG/rhoOz95cT27D1Zz+dn5fHPCMM4rVk+3sOgahITGzJheGuGHz69lU+UhBudnh11Sm3SmXVbbi/TUFG4cN4AZpcX87o0t/HzBRiY9+BoTz+vL1z8xjCEF+vfRlugMQhJu98FjXPTDF7n1iiHcNWFY2OW0KSd2Wb34rGiX1VHFuWGXlhQHj9Xy8KubeeTVTRytrWdGaTF3fHwoxT27hV1ap9HcGYQCQpLi879+i40Vh3j17ivUi4XYXVa/fc1wLgu5y2pY9h6q5ucLNvLom1vB4YZx/blt/BB666a7hGsuINQxWZJiRmmEnfuP8ubmvWGXErpl2/dz48MLufnXb1FVXcuPrx/DM1+9lMvPzu+U4QDQKzuTf7xuBAu+8TGml0Z49M2tXP6jl7nvhXUcOFobdnmdls4gJCmO1dbzkX//M1eN7Mt9nxoddjmhOLHL6u3jh3DDuAFkpOl32ok2VR7i/vnv8szycnp0TecrHz2LL1w8kK4Zbb97b3ujJiZpE771xHKeWV7G2//wcbpldPz+Ee7OwWN17DpwjN+/ufX9Lqtfumxwu+yyGoaVOw/wX/PW8fK6SvJzMrl9/BA+85H+CtU4UkBIm/DW5n18+pdv8N+fGc20kuKwyzlttfUN7D1UQ0XVMSqrqqmoqg7+fni+sqqa6roGANJSjBvG9eer44eSn6N29VP11uZ93PvCWt7e8h798rrytY+fzZQxET3DIg4UENImNDQ4H73vZY7VNjC8bw5ZGWl0y0wlOzONrMw0sjJSo38z08jKSCMreK9bRlqwTvT9zLSUuLfVuzuHqusafdl/8CV//Iv/+GvfkRpi/WfTs1s6BTldyM/JJD8nk4Lgb35OJqX9e9IvTz1zzoS7s+DdSu6du47V5Qc5u082d00YxoQRfTrttZt4UEBIm/H8inL+8NY2DlXXcbi6jsPV9RyuiU7X1rfu32Jqip0QJidPHw+W90MmM40uaSm8d6TmhF/9H4TAsdqGkz4rIzXl/S/5xl/6x4Pg+Hzv7Ew1eyRJQ4Pz3Mpy7p/3Lpv2HGZ0v1zuvmoYlwzp3fLGchIFhLQLNXUNHK6u41B1HUdq6oO/dcGyeo7U1H04WKrrgnCp/9B2x6ePN+/E0qNr+od+4Tf1xd+ja7p+nbZRdfUNzFqygx//eT1lB45xyZBefGPCMEr69wy7tHZFASGdUl19A4eDwDhSU8ex2gZyu6WTn5PZLga7k9Y5VlvPzIXb+NnL0bvQrzq3D9+fdC5FuV3DLq1dUECISId3qLqOX7+2mZ8v2EBaSgrfmTicz36kv27MbIFulBORDi87M43brxzKvDs/yqjiHnzvyZXc8PCbbN17OOzS2i0FhIh0KP17dWPml8bxw+nnsWrnQa564BUefnUT9Q0dp7UkWRQQItLhmBmfvaA/875+ORef1Zt/f3YNM/7nr7y7uyrs0toVBYSIdFiFPbryyM1j+fH1Y9i69zDX/eQ1fvriemrrm+7hJh9QQIhIh2ZmTBkTYf7XP8qEc/vwX/PfZfKDr7Ny54GwS2vzFBAi0in0zs7kwRtK+eVN57PnUDVTfvY6/zl3Lcdq68Murc1SQIhIp3LVuX3589c+yvSSCP+zYCMTf/Iqi7bsC7usNilhAWFm/czsZTNbbWarzOyOYHmemc03s/XB35i3PZpZvZktDV5PJ6pOEel8enRL595PjeZ3X7yA6toGPvXLN/jnp1dxuLou7NJO2bu7q3h2eXlC9p2wG+XMrBAodPclZpYDLAamAl8A9rn7PWb2baCnu38rxvaH3P2UHlCrG+VE5FQdrq7jR3PX8ts3tlLcsys/nH4elw3ND7usZm3be4Q5y8uYs6yMtbuqyOmSxpJ//ATpqaf+m79N3EltZn8CHgxeH3P38iBEFrj7SQ8qVkCISDK9vWUf33piOZv2HObTY4v53rUj6NG17TyzY9eBYzyzvIw5y8tZtn0/AGMH9GTS6CImnld42sPIhx4QZjYQeAUYCWxz99xguQHvHZ8/YZs6YClQB9zj7k81se9bgFsA+vfvf/7WrVvjXr+IdA7Haut54M/reeiVjfTOzuTfp45kwrl9Q6tn3+Eanl9ZztNLy3hryz7c4dyi7kweXcS1owop7nnmQ8iHGhBmlg38BfiBu882s/2NA8HM3nP3k65DmFnE3Xea2WDgJeBKd9/Y3GfpDEJE4mH5jv3c/cRy1u6qYtLoIv550gh6ZSfnQU9Vx2qZt2o3c5aX8dr6PdQ1OIPzs5g8uohJo4s4K/+UGlZa1FxAJPS5j2aWDswCZrr77GDxbjMrbNTEVBFrW3ffGfzdZGYLgBKg2YAQEYmHUcW5PH3bpfziLxv56UvreX3DHr4/aQSTRxclZPj3ozX1vLS2gjnLynhpXQU1dQ1EcrvypcsGM3l0EecU5oQy7HwiL1Ib8FuiF6TvbLT8XmBvo4vUee5+9wnb9gSOuHu1mfUG3gCmuPvq5j5TZxAiEm/rdlVx96zlLNu+nyuHF/CDaefRt0eXM95vTV0Dr66vZM6yMuav3s3hmnryczK59rxCJo0uorR/blJCIZQmJjO7FHgVWAEcv6/9u8BC4I9Af2Ar8Gl332dmY4GvuPuXzOxi4JfBdinAA+7+SEufqYAQkUSob3B+/dpm7pu3jozUFL537Tl85iP9TvkLvL7BWbhpL08vK+P5lbs4cLSWHl3TmXheXyaNKmLc4F5Jf8526Bepk0UBISKJtGXPYb41azkLN+/jkiG9uGf6qBafNe7uLNm2nznLynh2RTmVVdVkZaQy4dy+TBpdyKVD8kN9XK0CQkQkThoanD+8tY17nl9LfYPzzauGcfPFAz/0y9/dWV1+kDnLypmzrIyd+4+SkZbC+GEFTB5TxBXDCuia0TaeaqiAEBGJs7L9R/nukytYsK6S0v65/OiTozAz5iyL3sC2sfIwqSnGZUN7M2lUERPO7UNOl7ZzX8VxCggRkQRwd558Zyf/+sxqDh6tpcHBDMYNymPS6CKuGVlIXlZG2GU2K7RuriIiHZmZMb20mMuG5vOrVzfRp3sXrj2vMC69nNoCBYSIyBnKz8nkuxPPCbuMuNNw3yIiEpMCQkREYlJAiIhITAoIERGJSQEhIiIxKSBERCQmBYSIiMSkgBARkZg61FAbZlZJdAjx09Eb2BPHctoDHXPH19mOF3TMp2qAu+fHeqNDBcSZMLNFTY1H0lHpmDu+zna8oGOOJzUxiYhITAoIERGJSQHxgYfCLiAEOuaOr7MdL+iY40bXIEREJCadQYiISEwKCBERialTB4SZ9TOzl81stZmtMrM7wq4pWcws1czeMbNnwq4lGcws18yeMLO1ZrbGzC4Ku6ZEM7OvBf+uV5rZY2bWMR5z1oiZ/drMKsxsZaNleWY238zWB397hlljvDVxzPcG/7aXm9mTZpYbj8/q1AEB1AF3ufsI4ELgVjMbEXJNyXIHsCbsIpLox8Bcdx8OjKaDH7uZRYDbgbHuPhJIBa4Pt6qE+A1w9QnLvg286O5DgReD+Y7kN5x8zPOBke4+CngX+E48PqhTB4S7l7v7kmC6iuiXRiTcqhLPzIqBa4GHw64lGcysB3A58AiAu9e4+/5Qi0qONKCrmaUB3YCykOuJO3d/Bdh3wuIpwG+D6d8CU5NZU6LFOmZ3n+fudcHsm0BxPD6rUwdEY2Y2ECgBFoZcSjI8ANwNNIRcR7IMAiqB/w2a1R42s6ywi0okd98J3AdsA8qBA+4+L9yqkqaPu5cH07uAPmEWE4IvAs/HY0cKCMDMsoFZwJ3ufjDsehLJzK4DKtx9cdi1JFEaUAr8j7uXAIfpeM0OHxK0u08hGo5FQJaZfS7cqpLPo/34O01ffjP7HtGm85nx2F+nDwgzSycaDjPdfXbY9STBJcBkM9sCPA6MN7Pfh1tSwu0Adrj78bPDJ4gGRkf2cWCzu1e6ey0wG7g45JqSZbeZFQIEfytCricpzOwLwHXAjR6nG9w6dUCYmRFtl17j7veHXU8yuPt33L3Y3QcSvWj5krt36F+W7r4L2G5mw4JFVwKrQywpGbYBF5pZt+Df+ZV08AvzjTwN3BxM3wz8KcRaksLMribabDzZ3Y/Ea7+dOiCI/pq+ieiv6KXBa2LYRUlCfBWYaWbLgTHAf4RbTmIFZ0tPAEuAFUT/W+9wQ1CY2WPAG8AwM9thZn8L3AN8wszWEz2TuifMGuOtiWN+EMgB5gffY7+Iy2dpqA0REYmls59BiIhIExQQIiISkwJCRERiUkCIiEhMCggREYlJASHSDDMb2HjUzCbW2dToHovjyx4ws281s80WM+sdrzpFEkEBIXLmHqfRSKlmlgJ8Mlgu0m4pIERaycwGB4P9feSEtx4DPtNo/nJgq7tvNbOnzGxx8FyGW2Ls80NnKGb2DTP752D6LDObG2z/qpkNT8BhiTQpLewCRNqDoAnpceAL7r6s8XvuvsLMGsxsdPDe9URDA+CL7r7PzLoCb5vZLHff28qPfQj4iruvN7NxwM+B8fE5IpGWKSBEWpZPdDyf6e7e1BhOjwHXm9kqos8f+H6w/HYzmxZM9wOGAi0GRDDC8MXA/4sOpQRA5mlVL3KaFBAiLTtAdPC7S2l6kL/HgXnAX4Dl7r7bzD5GdCygi9z9iJktAE587GcdH27qPf5+CrDf3cfEoX6R06JrECItqwGmAZ83sxtireDuG4E9RAeGO9681AN4LwiH4UQfa3ui3UCBmfUys0yiwzUTPJdks5l9CqIjD5vZ6HgelEhLFBAireDuh4l+eX/NzCY3sdpjwHCiz14AmAukmdkaosHxZoz91gL/CrxF9LnCaxu9fSPwt2a2DFhF9AFAIkmj0VxFRCQmnUGIiEhMCggREYlJASEiIjEpIEREJCYFhIiIxKSAEBGRmBQQIiIS0/8HubFE+xsAIQQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Examine CV results\n",
    "mpg_cv.get_cv_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best k: 11\n",
      "CV MSE: 20.359530356007628\n"
     ]
    }
   ],
   "source": [
    "# Get best K\n",
    "   # This sets \"k\" instance attribute to allow for assessment on test set using tuned k\n",
    "mpg_cv.get_best_k()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test MSE: 17.984278448823908\n",
      "Test RMSE: 4.240787479799466\n",
      "Test MAE: 3.0834498834498842\n",
      "Test MAPE: 12.740852558555368%\n"
     ]
    }
   ],
   "source": [
    "# Examine regressor performance on test set using tuned k hyperparameter\n",
    "mm.assessment_metrics(mpg_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 8-fold CV indicated we should change our k hyperparameter from 7 to 11\n",
    "  # Improved model test MSE by ~3, test MAE by ~0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([18.79090909, 18.51818182, 28.3       , 30.82727273])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Re-compute earlier predictions using tuned KNN regressor\n",
    "gp.generate_predictions(mpg_cv,mpg_to_predict,'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assess model performance on these 4 values\n",
    "new_predicted_mpg=gp.generate_predictions(mpg_cv,mpg_to_predict,'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root mean squared error\n",
      "3.016442543764876\n",
      "Mean squared error\n",
      "9.098925619834715\n",
      "Mean absolute percent error\n",
      "9.280036801489896\n",
      "Mean absolute error\n",
      "2.2136363636363647\n"
     ]
    }
   ],
   "source": [
    "# Compute RMSE, MSE, MAPE, MAE\n",
    "print('Root mean squared error')\n",
    "print(mm.model_rmse(actual_mpg,new_predicted_mpg))\n",
    "print('Mean squared error')\n",
    "print(mm.model_mse(actual_mpg,new_predicted_mpg))\n",
    "print('Mean absolute percent error')\n",
    "print(mm.model_mape(actual_mpg,new_predicted_mpg))\n",
    "print('Mean absolute error')\n",
    "print(mm.model_mae(actual_mpg,new_predicted_mpg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tuned regressor achieves better performance for these 4 values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ubc_mds",
   "language": "python",
   "name": "ubc_mds"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}