notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c0b51b03-2e61-4f79-a902-988c2a39f1d1",
   "metadata": {},
   "source": [
    "# Fake News Detection "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "aa9329a6-3fd5-46a8-a08c-8dd2938d7aaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' The following imports are used for getting bert embedings.\\n    The embeddings were pickle and saved since running this takes up to an hour'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "import pickle\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from nltk.util import ngrams\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from scipy import stats\n",
    "\"\"\" The following imports are used for getting bert embedings.\n",
    "    The embeddings were pickle and saved since running this takes up to an hour\"\"\"\n",
    "# from flair.embeddings import TransformerWordEmbeddings\n",
    "# from flair.data import Sentence\n",
    "# import torch\n",
    "# from nltk.tokenize import sent_tokenize\n",
    "# import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b90cdca-e368-40d3-875e-61483913ca27",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Reading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0ada1c83-bce1-41fc-95a9-057ac9f144aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/train.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53980835-9b09-47ca-b936-909901be192f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Discovering the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe48aa88-83e9-4402-a00d-def93c9f09ae",
   "metadata": {},
   "source": [
    "We start off with a quick glance at the data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "58166e8f-9a46-475b-a869-7a8cd1d2ef7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Get the latest from TODAY Sign up for our news...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2d  Conan On The Funeral Trump Will Be Invited...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>It’s safe to say that Instagram Stories has fa...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Much like a certain Amazon goddess with a lass...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>At a time when the perfect outfit is just one ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label\n",
       "0  Get the latest from TODAY Sign up for our news...     1\n",
       "1  2d  Conan On The Funeral Trump Will Be Invited...     1\n",
       "2  It’s safe to say that Instagram Stories has fa...     0\n",
       "3  Much like a certain Amazon goddess with a lass...     0\n",
       "4  At a time when the perfect outfit is just one ...     0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dde1006a-b504-4c3a-aff2-273ae55f5f93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4987, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e1c4e7d-bc80-406d-aef5-31da59d6e5be",
   "metadata": {},
   "source": [
    "Let's start with investigating the target variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "88a1ec1d-dadc-4a67-8239-00faed9b79fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdsElEQVR4nO3de5xcdX3/8debeyAJFwPbkACBmlICkSALhR9SF/EnAUWgCsQiBEEjFGqp8VcCUsTSID9/BSpWkKD8ICCXcI8EVKAukTYQwjWES0lJgCVpIqDkIg0sfvrH+S4eJrN7Zpc9s7O77+fjMY+Z+Z7bZ2b3zHvO95w5RxGBmZlZVzbo6wLMzKzxOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCkPRDSX/fS/PaUdIaSRum562Svtwb807zu0fS5N6aXzeW+4+SXpP0XzWOf56k68qua6CSdLakH9U47tWS/rGL4SHpw71X3eDksBjgJC2V9Jak1ZJ+K+nfJZ0i6b2/fUScEhHn1zivT3Y1TkS8HBFDI+LdXqh9vQ/ciDg0Iq75oPPuZh07AFOBcRHxR1WGt0hqK2nZ96TwXSPpHUlv557/sIxldlJHpx+4kvaXtFbSsCrDHpd0eneXFxEXRESvfcmwD85hMTgcHhHDgJ2AC4EzgR/39kIkbdTb82wQOwGvR8TKei84hePQiBgK/AT4bsfziDillnkoU9q6HhHzgDbgcxXL3QMYB9zQnfkN4P+jfs1hMYhExJsRMRs4FpicVub3bcZLGiHprrQV8oakX0naQNK1wI7AT9O32r+TNCZ94zxZ0svAv+ba8iv8H0uaL+lNSXdK2iYta71v5B1bL5ImAmcDx6blPZmGv9etleo6R9JLklZKmilpyzSso47Jkl5OXUjf7Oy9kbRlmv7XaX7npPl/ErgX2D7VcXXFdFsA9+SGr5G0fRq8SZrnakmLJDXnptte0q1peUskfa07f0tJW6e/068l/SY9Hp0b3ippuqR/A34H7CLpU5KeT3+HyyQ9oFwXoaSTJD2b5vdzSTul9rlplCfT6zu2SknXACdUtJ0AzImI1yV9T9IrklZJelTSgbnlnifpFknXSVoFnKiKrUpJN0v6r1T7XEm7VyxrhKR703v9QEftVd63TSX9U/qfWKGsC3ZI4RtuDovBKCLmk30TPLDK4Klp2LZAE9kHdkTE8cDLZFspQyPiu7lpPg7sBhzSySJPAE4CtgfagUtrqPFnwAXATWl5e1YZ7cR0OwjYBRgK/EvFOB8DdgUOBs6VtFsni/w+sGWaz8dTzV+KiPuAQ4FlqY4TK+pcWzF8aEQsS4M/C9wIbAXM7qhN2bf8nwJPAqNSbWdI6uz9q2YD4P+TbfXsCLxV5bUfD0wBhgFvArcAZwEfAp4H/lfHiJKOJPtb/wXZ3/5XpC2CiPjzNNqe6fXdVKWea4EDJe2Ye41/CcxMwx8BJgDbANcDN0vaLDf9Eam+rci2oCrdA4wFtgMeqzLOccD5wAjgiU7mAfB/gT9JtXyY7P0/t5NxLcdhMXgtI1txK70DjAR2ioh3IuJXUXwCsfMiYm1EvNXJ8Gsj4un0wfr3wDFKO8A/oOOAiyPixYhYQ/ZBOKliq+bbEfFWRDxJ9uG8XuikWo4FzoqI1RGxFLiI7MP2g3gwIu5O+2+uzS17H2DbiPiHiHg7Il4ErgQm1TrjiHg9Im6NiN9FxGpgOlnI5V0dEYsiop0s0BZFxG3p+aVAfmf9V4HvRMSzafgFwITOvqFXqecV4AHgi6npYGAzYE4afl2quT0iLgI2JQvxDvMi4o6I+H21/6OIuCr9bdYB5wF7dmxFJnMiYm4a/k1gf2X7mt4jScBXgL+NiDfS+3YB3XjfBzOHxeA1CnijSvv/AxYDv5D0oqRpNczrlW4MfwnYmOwb4Ae1fZpfft4bkW0Rdch/IP6ObOuj0ghgkyrzGvUB66tc9mYpyHYi67b6bceN7Ft9U5V5VCVpc0lXpC6zVcBcYKuKEM6/79vnn6cvAPkuwJ2A7+XqeQMQ3XsP8l1RxwPXR8Q7qd6pqYvrzTT/LXn//0Cn/0OSNpR0oaT/TK91aRpUdfr0xeENstecty2wOfBo7nX+LLVbAYfFICRpH7IPgQcrh6Vvb1MjYhfgcODrkg7uGNzJLIu2PPLf8HYk23p5DVhLtvJ21LUh719xi+a7jOxDLj/vdmBFwXSVXks1Vc7r1Rqn7+6pm18BlkTEVrnbsIg4rBvzmEr2zfzPImI40NFVpE7qWg7k92ko/zzV9NWKmoZExL93o6bbgFGSDiLrzpqZlnUg2UEVxwBbR8RWZN1indVa6S/Juqk+SRYyYzpeRm6c9/7HJA0l22pexvu9RtZdt3vuNW6ZDh6wAg6LQUTScEmfIetHvy4iFlYZ5zOSPpw+TFYB76YbZB/Cu/Rg0V+UNE7S5sA/ALekrpn/IPu2/WlJGwPnkHVPdFgBjFHnR/LcAPytpJ3TB0THPo727hSXapkFTJc0LHW9fB2o9XcSK4APVXSLdGU+sErSmZKGpG/Oe6QQr9Uwsg++3yo7YOBbBePPAcZLOjJt3ZwG5A8D/iFwVseOY2U7/I/ODS/826duxlvI9qW8FBELcrW2A78GNpJ0LjC8htfYYRiwDnid7MvFBVXGOUzSxyRtQrbv4uHUNZav7/dk3X2XSNouvc5R3dxXNGg5LAaHn0paTfbt8ZvAxcCXOhl3LHAfsAaYB1wWEa1p2HeAc9Im/De6sfxrgavJumU2A74G2dFZwF8BPyL7Fr+W93eN3JzuX5f0WJX5XpXmPRdYAvw38NfdqCvvr9PyXyTb4ro+zb9QRDxHFlwvpvemsvujcvx3ybbaJqS6XyN7D2oNG4B/BoakaR8i607papmvAUcD3yX70B0HLCD7ECYibifb+Xtj6up5mmw/R4fzgGvS6zumi0VdQ7aFNjPX9nOyHdT/Qda9998Ud13mzUzTvQo8Q/Z6K11PFphvAHuT7c+q5kyybtaH0uu8j/fvO7FOyBc/Mht80tZaG3BcRPyyr+uxxuctC7NBQtIhkraStCnZDnVR/Vu62XocFmaDx/7Af5J1XR0OHNnF4c5m7+NuKDMzK+QtCzMzKzRgT9g1YsSIGDNmTF+X0am1a9eyxRZb9HUZZlZH/WG9f/TRR1+LiPV+qDhgw2LMmDEsWLCgeMQ+0traSktLS1+XYWZ11B/We0kvVWt3N5SZmRVyWJiZWSGHhZmZFXJYmJlZodLCQtJmyq6O9qSyq4R9O7Vvk65o9UK63zo3zVmSFiu7mtchufa9JS1Mwy5NJ7kzM7M6KXPLYh3wiXSFswnAREn7AdOA+yNiLHB/eo6kcWQXIdkdmAhcljs3/+VkV/wam24TS6zbzMwqlHkR90gXIYHsYjcbk52z/giyM1OS7o9Mj48AboyIdRGxhOzMkPtKGgkMj4h56YItM3PTmJlZHZT6O4u0ZfAo2bVufxARD0tqiojlABGxvOO88mQX48mf1Kwttb3D+09b3dFebXlTyLZAaGpqorW1tRdfTe9as2ZNQ9dnZr2vP6/3pYZFOm//BElbAbdL2qOL0avth4gu2qstbwYwA6C5uTka+ccv/eHHOWbWu/rzel+XX3BHxG8ltZLta1ghaWTaqhgJrEyjtfH+y2+OJrssYhvvv/xjR7uZWa8ZM21O6cuYOr6dE0teztILP13KfMs8GmrbtEWBpCFk1899DpgNTE6jTQbuTI9nA5MkbSppZ7Id2fNTl9VqSfulo6BOyE1jZmZ1UOaWxUiyyzBuSBZKsyLiLknzgFmSTgZeJrvUIxGxSNIssssmtgOnpW4sgFPJLss5hOzyjPeUWLeZmVUoLSwi4ilgryrtrwMHdzLNdGB6lfYFQFf7O8zMrET+BbeZmRVyWJiZWSGHhZmZFXJYmJlZIYeFmZkVcliYmVkhh4WZmRVyWJiZWSGHhZmZFXJYmJlZIYeFmZkVcliYmVkhh4WZmRVyWJiZWSGHhZmZFXJYmJlZIYeFmZkVcliYmVkhh4WZmRVyWJiZWSGHhZmZFXJYmJlZIYeFmZkVcliYmVkhh4WZmRUqLSwk7SDpl5KelbRI0t+k9vMkvSrpiXQ7LDfNWZIWS3pe0iG59r0lLUzDLpWksuo2M7P1bVTivNuBqRHxmKRhwKOS7k3DLomIf8qPLGkcMAnYHdgeuE/Sn0TEu8DlwBTgIeBuYCJwT4m1m5lZTmlbFhGxPCIeS49XA88Co7qY5AjgxohYFxFLgMXAvpJGAsMjYl5EBDATOLKsus3MbH112WchaQywF/Bwajpd0lOSrpK0dWobBbySm6wttY1KjyvbzcysTsrshgJA0lDgVuCMiFgl6XLgfCDS/UXASUC1/RDRRXu1ZU0h666iqamJ1tbWD1x/WdasWdPQ9ZkNNlPHt5e+jKYh5S+nrM+VUsNC0sZkQfGTiLgNICJW5IZfCdyVnrYBO+QmHw0sS+2jq7SvJyJmADMAmpubo6WlpVdeRxlaW1tp5PrMBpsTp80pfRlTx7dz0cJyv6MvPa6llPmWeTSUgB8Dz0bExbn2kbnRjgKeTo9nA5MkbSppZ2AsMD8ilgOrJe2X5nkCcGdZdZuZ2frKjLgDgOOBhZKeSG1nA1+QNIGsK2kp8FWAiFgkaRbwDNmRVKelI6EATgWuBoaQHQXlI6HMzOqotLCIiAepvr/h7i6mmQ5Mr9K+ANij96ozM7Pu8C+4zcyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKlRYWknaQ9EtJz0paJOlvUvs2ku6V9EK63zo3zVmSFkt6XtIhufa9JS1Mwy6VpLLqNjOz9ZW5ZdEOTI2I3YD9gNMkjQOmAfdHxFjg/vScNGwSsDswEbhM0oZpXpcDU4Cx6TaxxLrNzKzCRmXNOCKWA8vT49WSngVGAUcALWm0a4BW4MzUfmNErAOWSFoM7CtpKTA8IuYBSJoJHAncU1btY6bNKWvW75k6vp0T67CcpRd+uvRlmNnAV1pY5EkaA+wFPAw0pSAhIpZL2i6NNgp4KDdZW2p7Jz2ubK+2nClkWyA0NTXR2trao3qnjm/v0XTd0TSkPsvp6XtgNtgMlPW+rHW+9LCQNBS4FTgjIlZ1sbuh2oDoon39xogZwAyA5ubmaGlp6Xa9QF2+8U8d385FC8vP6qXHtZS+DLOBYKCs92Wt86UeDSVpY7Kg+ElE3JaaV0gamYaPBFam9jZgh9zko4FlqX10lXYzM6uTMo+GEvBj4NmIuDg3aDYwOT2eDNyZa58kaVNJO5PtyJ6fuqxWS9ovzfOE3DRmZlYHZW4PHQAcDyyU9ERqOxu4EJgl6WTgZeBogIhYJGkW8AzZkVSnRcS7abpTgauBIWQ7tkvbuW1mZusr82ioB6m+vwHg4E6mmQ5Mr9K+ANij96ozM7Pu8C+4zcyskMPCzMwK1RQWktwFZGY2iNW6ZfFDSfMl/ZWkrcosyMzMGk9NYRERHwOOI/sdxAJJ10v636VWZmZmDaPmfRYR8QJwDtl5nD4OXCrpOUl/UVZxZmbWGGrdZ/ERSZcAzwKfAA5PZ5P9BHBJifWZmVkDqPV3Fv8CXAmcHRFvdTRGxDJJ55RSmZmZNYxaw+Iw4K2OX1RL2gDYLCJ+FxHXlladmZk1hFr3WdxHdqqNDpunNjMzGwRqDYvNImJNx5P0ePNySjIzs0ZTa1islfTRjieS9gbe6mJ8MzMbQGrdZ3EGcLOkjutIjASOLaUiMzNrODWFRUQ8IulPgV3JziT7XES8U2plZmbWMLpzivJ9gDFpmr0kEREzS6nKzMwaSk1hIela4I+BJ4COCxIF4LAwMxsEat2yaAbGRUSUWYyZmTWmWo+Gehr4ozILMTOzxlXrlsUI4BlJ84F1HY0R8dlSqjIzs4ZSa1icV2YRZmbW2Go9dPYBSTsBYyPiPkmbAxuWW5qZmTWKWk9R/hXgFuCK1DQKuKOkmszMrMHUuoP7NOAAYBW8dyGk7coqyszMGkutYbEuIt7ueCJpI7LfWZiZ2SBQa1g8IOlsYEi69vbNwE/LK8vMzBpJrWExDfg1sBD4KnA32fW4zcxsEKgpLCLi9xFxZUQcHRGfT4+77IaSdJWklZKezrWdJ+lVSU+k22G5YWdJWizpeUmH5Nr3lrQwDbtUknryQs3MrOdqPRpqiaQXK28Fk10NTKzSfklETEi3u9P8xwGTgN3TNJdJ6jg093JgCjA23arN08zMStSdc0N12Aw4GtimqwkiYq6kMTXO/wjgxohYByyRtBjYV9JSYHhEzAOQNBM4ErinxvmamVkvqPVHea9XNP2zpAeBc3uwzNMlnQAsAKZGxG/IfrfxUG6cttT2Tnpc2V6VpClkWyE0NTXR2trag/Jg6vj2Hk3XHU1D6rOcnr4HZoPNQFnvy1rnaz1F+UdzTzcg29IY1oPlXQ6cT3bY7fnARcBJZBdUqhRdtFcVETOAGQDNzc3R0tLSgxLhxGlzejRdd0wd385FC7tzOZGeWXpcS+nLMBsIBsp6X9Y6X2vVF+UetwNLgWO6u7CIWNHxWNKVwF3paRuwQ27U0cCy1D66SruZmdVRrd1QB/XGwiSNjIjl6elRZKc+B5gNXC/pYmB7sh3Z8yPiXUmrJe0HPAycAHy/N2oxM7Pa1doN9fWuhkfExVWmuQFoAUZIagO+BbRImkDWlbSU7DcbRMQiSbOAZ8i2XE6LiI4r8p1KdmTVELId2965bWZWZ905Gmofsi0AgMOBucArnU0QEV+o0vzjLsafDkyv0r4A2KPGOs3MrATdufjRRyNiNWQ/rgNujogvl1WYmZk1jlpP97Ej8Hbu+dvAmF6vxszMGlKtWxbXAvMl3U62v+EoYGZpVZmZWUOp9Wio6ZLuAQ5MTV+KiMfLK8vMzBpJrd1QAJsDqyLie0CbpJ1LqsnMzBpMrScS/BZwJnBWatoYuK6soszMrLHUumVxFPBZYC1ARCyjZ6f7MDOzfqjWsHg7Xb8iACRtUV5JZmbWaGoNi1mSrgC2kvQV4D7gyvLKMjOzRlJ4NFS6Mt1NwJ8Cq4BdgXMj4t6SazMzswZRGBYREZLuiIi9AQeEmdkgVGs31EOS9im1EjMza1i1/oL7IOCUdJnTtWQXJYqI+EhZhZmZWePoMiwk7RgRLwOH1qkeMzNrQEVbFneQnW32JUm3RsTn6lCTmZk1mKJ9FvlrYO9SZiFmZta4isIiOnlsZmaDSFE31J6SVpFtYQxJj+EPO7iHl1qdmZk1hC7DIiI2rFchZmbWuLpzinIzMxukHBZmZlbIYWFmZoUcFmZmVshhYWZmhRwWZmZWyGFhZmaFSgsLSVdJWinp6VzbNpLulfRCut86N+wsSYslPS/pkFz73pIWpmGXposxmZlZHZW5ZXE1MLGibRpwf0SMBe5Pz5E0DpgE7J6muUxSxw8CLwemAGPTrXKeZmZWstLCIiLmAm9UNB8BXJMeXwMcmWu/MSLWRcQSYDGwr6SRwPCImBcRAczMTWNmZnVS68WPektTRCwHiIjlkrZL7aOAh3LjtaW2d9LjyvaqJE0h2wqhqamJ1tbWHhU5dXx7j6brjqYh9VlOT98Ds8FmoKz3Za3z9Q6LzlTbDxFdtFcVETOAGQDNzc3R0tLSo2JOnDanR9N1x9Tx7Vy0sPy3f+lxLaUvw2wgGCjrfVnrfL2PhlqRupZI9ytTexuwQ2680cCy1D66SruZmdVRvcNiNjA5PZ4M3JlrnyRpU0k7k+3Inp+6rFZL2i8dBXVCbhozM6uT0raHJN0AtAAjJLUB3wIuBGZJOhl4GTgaICIWSZoFPAO0A6dFxLtpVqeSHVk1BLgn3czMrI5KC4uI+EIngw7uZPzpwPQq7QuAPXqxNDMz6yb/gtvMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8LMzAo5LMzMrFCfhIWkpZIWSnpC0oLUto2keyW9kO63zo1/lqTFkp6XdEhf1GxmNpj15ZbFQRExISKa0/NpwP0RMRa4Pz1H0jhgErA7MBG4TNKGfVGwmdlg1UjdUEcA16TH1wBH5tpvjIh1EbEEWAzsW//yzMwGr436aLkB/EJSAFdExAygKSKWA0TEcknbpXFHAQ/lpm1LbeuRNAWYAtDU1ERra2uPips6vr1H03VH05D6LKen74HZYDNQ1vuy1vm+CosDImJZCoR7JT3Xxbiq0hbVRkyhMwOgubk5WlpaelTcidPm9Gi67pg6vp2LFpb/9i89rqX0ZZgNBANlvS9rne+TbqiIWJbuVwK3k3UrrZA0EiDdr0yjtwE75CYfDSyrX7VmZlb3sJC0haRhHY+BTwFPA7OByWm0ycCd6fFsYJKkTSXtDIwF5te3ajOzwa0vuqGagNsldSz/+oj4maRHgFmSTgZeBo4GiIhFkmYBzwDtwGkR8W4f1G1mNmjVPSwi4kVgzyrtrwMHdzLNdGB6yaWZmVknGunQWTMza1AOCzMzK+SwMDOzQg4LMzMr5LAwM7NCDgszMyvksDAzs0IOCzMzK+SwMDOzQg4LMzMr5LAwM7NCDgszMyvksDAzs0IOCzMzK+SwMDOzQg4LMzMr5LAwM7NCDgszMyvksDAzs0IOCzMzK+SwMDOzQg4LMzMr5LAwM7NCDgszMyvksDAzs0IOCzMzK9RvwkLSREnPS1osaVpf12NmNpj0i7CQtCHwA+BQYBzwBUnj+rYqM7PBo1+EBbAvsDgiXoyIt4EbgSP6uCYzs0FDEdHXNRSS9HlgYkR8OT0/HviziDi9YrwpwJT0dFfg+boW2j0jgNf6uggzq6v+sN7vFBHbVjZu1BeV9ICqtK2XchExA5hRfjkfnKQFEdHc13WYWf305/W+v3RDtQE75J6PBpb1US1mZoNOfwmLR4CxknaWtAkwCZjdxzWZmQ0a/aIbKiLaJZ0O/BzYELgqIhb1cVkfVL/oLjOzXtVv1/t+sYPbzMz6Vn/phjIzsz7ksDAzs0IOizqTdJWklZKe7utazKw+BsLpihwW9Xc1MLGvizCz+hgopytyWNRZRMwF3ujrOsysbgbE6YocFmZm5RoFvJJ73pba+hWHhZlZuWo6XVGjc1iYmZVrQJyuyGFhZlauAXG6IodFnUm6AZgH7CqpTdLJfV2TmZUnItqBjtMVPQvM6o+nK/LpPszMrJC3LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKyQw8IGPUlrujHueZK+8UHnL6lV0iEVbWdIuqyL+bRKau7Oss16i8PCrG/cQPbjrLxJqd2s4TgszKqQdLikhyU9Luk+SU25wXtK+ldJL0j6Sm6a/yPpEUlPSfp2wSJuAT4jadM07Rhge+BBSZdLWiBpUWfzyW+tSPq8pKvT420l3ZrqeETSAT16A8wqOCzMqnsQ2C8i9iI7pfTf5YZ9BPg0sD9wrqTtJX0KGEt2OuoJwN6S/ryzmUfE68B8/nBtk0nATZH9SvabEdGclvNxSR/pRt3fAy6JiH2AzwE/6sa0Zp3aqK8LMGtQo4GbJI0ENgGW5IbdGRFvAW9J+iVZQHwM+BTweBpnKFl4zO1iGR1dUXem+5NS+zGSppCtnyPJLpjzVI11fxIYJ713otPhkoZFxOoapzerymFhVt33gYsjYrakFuC83LDKc+QE2WmovxMRV3RjGXcAF0v6KDAkIh6TtDPwDWCfiPhN6l7arMq0+RrywzcA9k9hZtZr3A1lVt2WwKvp8eSKYUdI2kzSh4AWsrOK/hw4SdJQAEmjJG3X1QIiYg3QClzFH3ZsDwfWAm+m/SSHdjL5Ckm7SdoAOCrX/guyk9aR6pjQVQ1mtfKWhRlsLqkt9/xisi2JmyW9CjwE7JwbPh+YA+wInB8Ry4BlknYD5qUuoDXAF4GVBcu+AbiNdGRURDwp6XFgEfAi8G+dTDcNuIvsCmxPk3V7AXwN+IGkp8jW77nAKQU1mBXyWWfNzKyQu6HMzKyQw8LMzAo5LMzMrJDDwszMCjkszMyskMPCzMwKOSzMzKzQ/wDzROompvWHowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "bins = [-.1, .1, .9, 1.1]\n",
    "data['label'].hist(bins=bins)\n",
    "plt.title('Distribution of the Target Variable')\n",
    "plt.xlabel('Label Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5067a052-4ac7-453a-a906-b6647e95902f",
   "metadata": {},
   "source": [
    "The histogram shows that the data is not balanced. We have to keep this in mind when evaluating the performance of the models and avoid some metrics that would mislead us.\n",
    "\n",
    "Something else to notice is the third value **label** on the x-axis. Let's have a closer look to see what that is. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "18e7a7d8-30fb-4082-90a0-05d54eaf6476",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        2972\n",
       "1        2014\n",
       "label       1\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94884d1f-b5ea-499e-b1ef-ff3e4d5bdbf9",
   "metadata": {},
   "source": [
    "There is a single value called label. It could be a missing value. Let's look at that row in the dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bc42f095-b4d1-4fb4-8ac1-0de03ca74934",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1615</th>\n",
       "      <td>content</td>\n",
       "      <td>label</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         text  label\n",
       "1615  content  label"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[data['label']=='label']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b5bb13d-719e-4167-86b7-8ce17c9d3c86",
   "metadata": {},
   "source": [
    "The whole row is junk. We will delete it. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "81137d99-4d76-4464-ba24-607612809cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop(1615)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9fcb82de-a8f6-42fb-b73f-23f96fadd6bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2972\n",
       "1    2014\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec715765-5675-41ad-a901-be2a75d6c031",
   "metadata": {},
   "source": [
    "Let's make sure we don't have any missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9218537a-cc9d-484f-88c5-f97f1351129d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 4986 entries, 0 to 4986\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    4986 non-null   object\n",
      " 1   label   4986 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 116.9+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a2372fd5-5cf9-4f7e-ac21-0d96837ac788",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text     False\n",
       "label    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().any()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75052e02-955b-412e-85ff-cfd34bb0af19",
   "metadata": {},
   "source": [
    "It seems that we don't have any missing values. \n",
    "\n",
    "\n",
    "Now, let's investigate the single feature we have **text**. We start with learning some statistics about this feature e.g. the length of the article. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d35351c6-4ba6-4815-b20b-ce3aef4ef1dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average length: 661.996991576414 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEWCAYAAACjYXoKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhsklEQVR4nO3de7wVdb3/8debDXJREBTEzUVQIxP9eUXjpJlGHRGv1THpV0neSLNOpv1M09OxU/wOdX52IVOzNMQuilqKHv2dzLzk8UJoKuIlUFAQ5GYIKHL9nD/mu3Fc7MvazFp774Xv5+OxHmvWd2a+85nvmrU+852ZNUsRgZmZWRGd2jsAMzOrfU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk0kbkXS1pH+pUF27SVotqS69vl/SmZWoO9V3t6RxlaqvFcv9rqRlkl4rc/rLJP2qQsueLOm7lahrK5Y9T9LHqryM/pIelLRK0uUdJa5aV+7nutKf0Y7IyaQC0oduTfqgrpD0sKSzJW1u34g4OyK+U2ZdzX6AI+KViNghIjZWIPYtvpAj4piIuL5o3a2MYzBwATA8InZtZPyRkha0ZUzV0I5JazywDOgVEReUjqx2XJJC0sz8ZyLtPEyu1jLLkbb/kHRoGdN+QdJD+bJyP9fvBU4mlXN8RPQEhgATgW8A11Z6IZI6V7rODmIIsDwilrR3INuoIcCz0b6/Uh4AjG3H5b+LJAGfB14Hmu2Jb8Ofu8qJCD8KPoB5wMdKyg4FNgH7pteTge+m4b7AncAKsg35z2SJ/YY0zxpgNXAhMBQI4AzgFeDBXFnnVN/9wL8D04E3gNuBndK4I4EFjcULjAbWAevT8p7K1XdmGu4EXAq8DCwBpgA7pnENcYxLsS0DLmmmnXZM8y9N9V2a6v9YWudNKY7JJfNtXzJ+NdkX02XA1FTnKmAWMCI33wDg1rS8ucA/NxPb5vcnvT4OeDK9Rw8D+5W039eBp1N73wR0y42/EFgELATOTG30PrLewfrU5quBO1qqjya2lSbW4UPAX1IdfwE+lFu3/HJLt9VWx9VSGzUSW5DtYM3mne32u/n3GhiZ6lkBPAUcmcqPAmbmpvsjMD33+iHgpDT8DeDVtD28AIxqJqYjyLarzwHLge1y474A/Dfww9TutwJvAxtTG61oYrs5MbXJSuBFYHTpZyq9Ph14Dvg78F/AkFSutMwlqc2fJn2HdPRHuwewLTxoJJmk8leAc9Lw5o2O7Iv/aqBLenwYUGN18c4X9hSyL9XuNJ5MXgX2TdPcCvwqjTuSJpJJGr6sYdrc+M0bftro5wB7ADsAvwNuKInt5ymu/YG1wN5NtNMUskTXM837N+CMpuIsmbex9bgsfcDHAHWpXR9N4zoBjwPfArZL8b8EHN1E/fn356D0Yf5gqndcarOuufabTpasdiL7Ujg7jRsNvAbsA/Qg20EI4H2lyyl5P5qqr8ltpaSOnci+mD4PdAY+k17v3NRym1r/MuNqto0aqT+AYek9adi2NicTYCDZF/qY9N59PL3uB3Qj+9Lvm9btNbJE3ZNsu1sD7AzsBcwHBuS2zz2bWedryXZGuqRlfTI37gvABuAraZndU9lDzWw3h5IlgI+ndRgIfKCRz9RJZJ+pvVPdlwIPp3FHpzbqTZZY9gbq2/s7rpyHD3NV10KyD2Gp9UA92d7I+oj4c6QtqRmXRcSbEbGmifE3RMQzEfEm8C/ApxtO0Bf0WeAHEfFSRKwGLgbGlnT7vx0RayLiKbI9yv1LK0mxnAJcHBGrImIecDnZl18RD0XEXZGdP7oht+xDgH4R8W8RsS4iXiJLeuUcZjkL+FlEPBYRGyM7f7SWbM+5waSIWBgRrwN3AAek8k8Dv4yIWRHxFvDtMtejqfrK3VaOBWZHxA0RsSEifgs8Dxxf5vJbG1c5bVQqyLbNb0nqWjLuc8Bd6b3cFBH3ADOAMRHxdho+AhhBtrf+EHBYWt7siFhO1mvoCgyX1CUi5kXEi40FIqkHcDLwm4hYD9zCloe6FkbET1J7NvW5yzsDuC4i7knr8GpEPN/IdF8E/j0inouIDcD/BQ6QNITs/e4JfIBsp+G5iFhUxrLbnZNJdQ0k6yKX+g+yPZM/SHpJ0kVl1DW/FeNfJtvb6ltWlM0bkOrL190Z6J8ry1999RZZD6ZUX7IeQmldAwvGV7rsbinRDQEGpAsiVkhaAXyzJO6mDAEuKJl3MFlbNLXchnUewLvfi5bet5bqK3dbKX2foDrt2xBXOW20hYi4i6zHPr5k1BDg5JL6DidLpAAPkPVOj0jD9wMfSY8HUt1zgPPIeqxLJN0oqal4PkHW87grvf41cIykfrlpyn3vGgwmO7TVkiHAj3Pr+TpZL2RgRPwJuAL4KbBY0jWSerUyjnbhZFIlkg4h+yA/VDou7ZlfEBF7kO05ni9pVMPoJqpsqecyODe8G9kezjLgTbLDLQ1x1ZEdOii33oVkG3++7g3A4hbmK7UsxVRa16tlzt/aE8fzgbkR0Tv36BkRY8qcd0LJvD3S3n5LFgGDcq8Hl4xv1Xq0sK3klb5PUP323do2uhS4hNx2meq7oaS+7SNiYhpfmkweoCSZAETEbyLicLK2COB7TcQwjiwxvpIuRb+ZbAfsM7lpStukpTaaD+zZwjQN032xZF27R8TDaR0mRcTBZIdK3w/8nzLqbHdOJhUmqZek44Abyc5FzGxkmuMkvS9dTbKSrHvecJnvYrLj+631OUnDU/f934Bb0qGfv5HtrR8rqQvZBzl/iGExMDR/yWaJ3wJfk7S7pB3IuuQ3pe552VIsU4EJknqmLv35QLm/E1kM7CxpxzKnnw6slPQNSd0l1UnaNyX5lvwcOFvSB5XZPrVfzzLmnQqcJmnv9F58q5H1KPv9bWFbybsLeL+k/y2ps6RTgOFkJ+/L0drtbqvbKCLuB2by7sNKvwKOl3R0eq+6pcvBGxLzw2TnRA4lO/k+iyxhfJDsohQk7SXpo+kQ2ttk51K2aCtJA4FRZBcQHJAe+5MlntJDXXmLgUGStmti/LVk7/0oSZ0kDZT0gUamuxq4WNI+KZ4dJZ2chg9JbdqFbEew4aR/h+dkUjl3SFpFttdxCfAD4LQmph1GdkXKauAR4Mr0AYPshOulqQv89VYs/wayk4GvkZ2w/GeAiHgD+BLwC7K91DeB/O81bk7PyyU90Ui916W6HyS7IuptspOSW+MrafkvkfXYfpPqb1E69vxb4KXUNi0dTtlItid/QIp7GVkbtJiMImIG2TmBK8hOYs8hO/laTpx3A5OA+9J8j6RRa9PztWTH9FdIuq2MKpvbVvLLXU725XgB2cnkC4HjImJZOXG3Nq4ibZRcSu58YkTMJ7sS6ptkV9/NJ9sj75TGvwk8AcyKiHVptkeAl+Ody8m7kl2Wv4zsc7BLqq/U54EnI+IPEfFaw4PsfdtP0r5NxPwnsisGX5O0RbtGxHSyz/wPyU7EP8CWvUUi4vdkietGSSuBZ4Bj0uheZIn672SHKZcD/6+JeDqUhiuIzKwKJO1N9mXRtbW9ObNa4p6JWYVJ+oSk7ST1IdsDvcOJxLZ1TiZmlfdFskM1L5Id7z6nfcMxqz4f5jIzs8LcMzEzs8K22ZuX9e3bN4YOHdreYZiZ1ZTHH398WUT0a3nKd9tmk8nQoUOZMWNGe4dhZlZTJJXeSaEsPsxlZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhW2zv4AvYuhF/9kuy5038dh2Wa6ZWVHumZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFVTyaS6iT9VdKd6fVOku6RNDs998lNe7GkOZJekHR0rvxgSTPTuEmSVO24zcysfG3RM/kq8Fzu9UXAvRExDLg3vUbScGAssA8wGrhSUl2a5ypgPDAsPUa3QdxmZlamqiYTSYOAY4Ff5IpPBK5Pw9cDJ+XKb4yItRExF5gDHCqpHugVEY9ERABTcvOYmVkHUO2eyY+AC4FNubL+EbEIID3vksoHAvNz0y1IZQPTcGn5FiSNlzRD0oylS5dWZAXMzKxlVUsmko4DlkTE4+XO0khZNFO+ZWHENRExIiJG9OvXr8zFmplZUZ2rWPdhwAmSxgDdgF6SfgUsllQfEYvSIawlafoFwODc/IOAhal8UCPlZmbWQVStZxIRF0fEoIgYSnZi/U8R8TlgGjAuTTYOuD0NTwPGSuoqaXeyE+3T06GwVZJGpqu4Ts3NY2ZmHUA1eyZNmQhMlXQG8ApwMkBEzJI0FXgW2ACcGxEb0zznAJOB7sDd6WFmZh1EmySTiLgfuD8NLwdGNTHdBGBCI+UzgH2rF6GZmRXhX8CbmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkVVrVkIqmbpOmSnpI0S9K3U/lOku6RNDs998nNc7GkOZJekHR0rvxgSTPTuEmSVK24zcys9arZM1kLfDQi9gcOAEZLGglcBNwbEcOAe9NrJA0HxgL7AKOBKyXVpbquAsYDw9JjdBXjNjOzVqpaMonM6vSyS3oEcCJwfSq/HjgpDZ8I3BgRayNiLjAHOFRSPdArIh6JiACm5OYxM7MOoKrnTCTVSXoSWALcExGPAf0jYhFAet4lTT4QmJ+bfUEqG5iGS8vNzKyDqGoyiYiNEXEAMIisl7FvM5M3dh4kminfsgJpvKQZkmYsXbq01fGamdnWKSuZtJAEWhQRK4D7yc51LE6HrkjPS9JkC4DBudkGAQtT+aBGyhtbzjURMSIiRvTr169IyGZm1grl9kyuTldmfUlS73JmkNSvYVpJ3YGPAc8D04BxabJxwO1peBowVlJXSbuTnWifng6FrZI0Ml3FdWpuHjMz6wA6lzNRRBwuaRhwOjBD0nTglxFxTzOz1QPXpyuyOgFTI+JOSY8AUyWdAbwCnJyWMUvSVOBZYANwbkRsTHWdA0wGugN3p4eZmXUQZSUTgIiYLelSYAYwCTgw9RS+GRG/a2T6p4EDGylfDoxqYhkTgAmNlM8ACh1qMzOz6in3nMl+kn4IPAd8FDg+IvZOwz+sYnxmZlYDyu2ZXAH8nKwXsqahMCIWpt6KmZm9h5WbTMYAaxrOYUjqBHSLiLci4oaqRWdmZjWh3Ku5/kh28rtBj1RmZmZWdjLplrs1Cmm4R3VCMjOzWlNuMnlT0kENLyQdDKxpZnozM3sPKfecyXnAzZIafnleD5xSlYjMzKzmlPujxb9I+gCwF9m9sp6PiPVVjczMzGpG2T9aBA4BhqZ5DpREREypSlRmZlZTykomkm4A9gSeBBpucdLw3yJmZvYeV27PZAQwPP05lZmZ2buUezXXM8Cu1QzEzMxqV7k9k77As+luwWsbCiPihKpEZWZmNaXcZHJZNYMwM7PaVu6lwQ9IGgIMi4g/SuoB1FU3NDMzqxXl3oL+LOAW4GepaCBwW5ViMjOzGlPuCfhzgcOAlZD9URawS7WCMjOz2lJuMlkbEesaXkjqTPY7EzMzs7KTyQOSvgl0l/Rx4GbgjuqFZWZmtaTcZHIRsBSYCXwRuAvwPyyamRlQ/tVcm8j+tvfn1Q3HzMxqUbn35ppLI+dIImKPikdkZmY1pzX35mrQDTgZ2Kny4ZiZWS0q65xJRCzPPV6NiB8BH61uaGZmVivKPcx1UO5lJ7KeSs+qRGRmZjWn3MNcl+eGNwDzgE9XPBozM6tJ5V7NdVS1AzEzs9pV7mGu85sbHxE/qEw4ZmZWi1pzNdchwLT0+njgQWB+NYIyM7Pa0po/xzooIlYBSLoMuDkizqxWYGZmVjvKvZ3KbsC63Ot1wNCKR2NmZjWp3J7JDcB0Sb8n+yX8J4ApVYvKzMxqSrlXc02QdDfw4VR0WkT8tXphmZlZLSn3MBdAD2BlRPwYWCBp9yrFZGZmNabcv+39V+AbwMWpqAvwq2oFZWZmtaXcnskngBOANwEiYiG+nYqZmSXlJpN1ERGk29BL2r6lGSQNlnSfpOckzZL01VS+k6R7JM1Oz31y81wsaY6kFyQdnSs/WNLMNG6SJLVuNc3MrJrKTSZTJf0M6C3pLOCPtPxHWRuACyJib2AkcK6k4WT/2nhvRAwD7k2vSePGAvsAo4ErJdWluq4CxgPD0mN0mXGbmVkbaPFqrtQLuAn4ALAS2Av4VkTc09x8EbEIWJSGV0l6DhgInAgcmSa7Hrif7HzMicCNEbEWmCtpDnCopHlAr4h4JMUzBTgJuLsV62lmZlXUYjKJiJB0W0QcDDSbQJoiaShwIPAY0D8lGiJikaRd0mQDgUdzsy1IZevTcGl5Y8sZT9aDYbfddtuaUM3MbCuUe5jrUUmHbM0CJO0A3AqcFxErm5u0kbJopnzLwohrImJERIzo169f64M1M7OtUu4v4I8Czk6HnN4k+4KPiNivuZkkdSFLJL+OiN+l4sWS6lOvpB5YksoXAINzsw8CFqbyQY2Um5lZB9FsMpG0W0S8AhzT2orTuZZrgedKblE/DRgHTEzPt+fKfyPpB8AAshPt0yNio6RVkkaSHSY7FfhJa+MxM7PqaalnchvZ3YJflnRrRHyqFXUfBnwemCnpyVT2TbIkMlXSGcArwMkAETFL0lTgWbIrwc6NiI1pvnOAyUB3shPvPvluZtaBtJRM8ucr9mhNxRHxEI2f7wAY1cQ8E4AJjZTPAPZtzfLNzKzttHQCPpoYNjMz26ylnsn+klaS9TC6p2F45wR8r6pGZ2ZmNaHZZBIRdc2NNzMzg9bdgt7MzKxRTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhVUtmUi6TtISSc/kynaSdI+k2em5T27cxZLmSHpB0tG58oMlzUzjJklStWI2M7OtU82eyWRgdEnZRcC9ETEMuDe9RtJwYCywT5rnSkl1aZ6rgPHAsPQordPMzNpZ1ZJJRDwIvF5SfCJwfRq+HjgpV35jRKyNiLnAHOBQSfVAr4h4JCICmJKbx8zMOoi2PmfSPyIWAaTnXVL5QGB+broFqWxgGi4tb5Sk8ZJmSJqxdOnSigZuZmZN6ygn4Bs7DxLNlDcqIq6JiBERMaJfv34VC87MzJrX1slkcTp0RXpeksoXAINz0w0CFqbyQY2Um5lZB9LWyWQaMC4NjwNuz5WPldRV0u5kJ9qnp0NhqySNTFdxnZqbx8zMOojO1apY0m+BI4G+khYA/wpMBKZKOgN4BTgZICJmSZoKPAtsAM6NiI2pqnPIrgzrDtydHmZm1oFULZlExGeaGDWqieknABMaKZ8B7FvB0MzMrMI6ygl4MzOrYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZmZWmJOJmZkV5mRiZmaFOZmYmVlhTiZmZlaYk4mZmRXmZGJmZoU5mZiZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYZ3bO4Btxaa3V7P87kmsW/YKAH3HfJWuA/dm5eN3sOqJO5Hq6L7nCPocdTpr5v6VFQ9MJjZuQHWd6X3U6XQfsv8Wdb7++uuccsopzJs3j6FDhzJ16lT69OnT1qtmZtYi90wq5PV7r6HbHgcz8KyrGXD6T+iy82Defvlp1sx+lAGnXcGAM6+k16GfBKCuRy/6fepbDDjjp+x87NdYfufljdY5ceJERo0axezZsxk1ahQTJ05sy1UyMyubk0kFbFr7Fm/Pn8UO+/0jAKrrQqduO7Dqr3fRa+TJqHMXAOq27w3Adv33pHPPnQHo0ncIsWE9sWH9FvXefvvtjBs3DoBx48Zx2223VX9lzMy2gg9zVcCGFa9R16MXy+/6EeuWzKXrru+jz6jxrP/7q6ydP4sVD05Bnbejz1Gn07X+/e+a960X/pvt+u+xOeHkLV68mPr6egDq6+tZsmRJm6yPmVlruWdSAbFpI+tee5GeB45hwGmTUJeurHz0Zti0kU1rV7Pr5y+nz5GnsfT27xERm+dbt/RlVjwwmZ2O/nI7Rm9mVpyTSQV07tmXup596TpgLwB67HUY6xa/SF3PvvR4/z8gia4D9kISm9asBGDDymUs/f0Edj72fLr0qW+03v79+7No0SIAFi1axC677NI2K2Rm1ko+zFXACbPu48IHpzBg5TJG1tWx12O38ucPfoq3X36KLn13o3PvXXn75afpttt+rH/9VWLjBjp178Wmt1ez5JbL6P2RcXQbNHxzfUMv+s931f/33vsy/DMXs+PIk3nj0ZvZ1HvfLaaplHkTj61KvWb23uBkspVOmHUfE///FfTYsBaAazZu4PT7J7P68TtYt+v72HnMeXTq0pXld/2Yhdd+CdV1Yedjv4Yk3njiTjasWMQbD9/IGw/fCED/T3+Huu17s/zuSexwwDF0rR9Gr5H/xLLbJ7L66T/QuVc/+p54cXuusplZk5Q/hr8tGTFiRMyYMWOr5i1n7/+hq05j0MqlW5Qv6NWPw8/55VYttz25Z2JmAJIej4gRrZ3P50y20oCVy1pVbma2LXMy2UoLe/VtVbmZ2basZs6ZSBoN/BioA34REe36c/DvH3Hqu86ZALzVuSvfP+LUdoxq61XrxH5LfHjNbNtQE8lEUh3wU+DjwALgL5KmRcSz7RXTtH2OAth8NdfCXn35/hGnbi638rRXEgMnMrNKqolkAhwKzImIlwAk3QicCLRbMoEsoTh51K72TGTtxQm07bzXevu1kkwGAvNzrxcAHyydSNJ4YHx6uVrSC1u5vL5ALZ1Jr7V4ofZirrV4oZGY9b12iqQ820Qbt7cW3uNy4h2yNcutlWSiRsq2uKY5Iq4Brim8MGnG1lwa115qLV6ovZhrLV6ovZhrLV6ovZirGW+tXM21ABicez0IWNhOsZiZWYlaSSZ/AYZJ2l3SdsBYYFo7x2RmZklNHOaKiA2Svgz8F9mlwddFxKwqLrLwobI2VmvxQu3FXGvxQu3FXGvxQu3FXLV4t9nbqZiZWduplcNcZmbWgTmZmJlZYU4mOZJGS3pB0hxJF7VjHIMl3SfpOUmzJH01lV8m6VVJT6bHmNw8F6e4X5B0dK78YEkz07hJkhq7zLpScc9Ly3pS0oxUtpOkeyTNTs99OkLMkvbKteOTklZKOq+jtbGk6yQtkfRMrqxibSqpq6SbUvljkoZWId7/kPS8pKcl/V5S71Q+VNKaXFtf3dbxNhNzxbaDNmrjm3KxzpP0ZCpvuzaOCD+y80Z1wIvAHsB2wFPA8HaKpR44KA33BP4GDAcuA77eyPTDU7xdgd3TetSlcdOBfyD7rc7dwDFVjHse0Lek7PvARWn4IuB7HSnm3Hv/GtmPtTpUGwNHAAcBz1SjTYEvAVen4bHATVWI9x+Bzmn4e7l4h+anK6mnTeJtJuaKbQdt0cYl4y8HvtXWbeyeyTs237IlItYBDbdsaXMRsSginkjDq4DnyO4C0JQTgRsjYm1EzAXmAIdKqgd6RcQjkW0ZU4CTqht9o7Fdn4avzy2/I8U8CngxIl5uZpp2iTciHgRebySWSrVpvq5bgFFFelaNxRsRf4iIDenlo2S/E2tSW8bbVMzN6JBt3CDV+2ngt83VUY14nUze0dgtW5r7Am8TqYt5IPBYKvpyOlxwXe7wRlOxD0zDpeXVEsAfJD2u7NY2AP0jYhFkSRJo+CP7jhIzZHtf+Q9fR25jqGybbp4nfeG/AexctcjhdLK94Aa7S/qrpAckfTgXU0eIt1LbQVvG/GFgcUTMzpW1SRs7mbyjrFu2tCVJOwC3AudFxErgKmBP4ABgEVl3FpqOva3X6bCIOAg4BjhX0hHNTNshYlb2I9gTgJtTUUdv4+ZsTYxtFr+kS4ANwK9T0SJgt4g4EDgf+I2kXi3E1FbxVnI7aMtt5DO8e8eozdrYyeQdHeqWLZK6kCWSX0fE7wAiYnFEbIyITcDPyQ7NQdOxL+DdhxSquk4RsTA9LwF+n+JbnLrUDV3rJR0pZrLE90RELE6xd+g2TirZppvnkdQZ2JHyD/mUTdI44Djgs+mwCulQ0fI0/DjZ+Yf3d4R4K7wdtFUbdwY+CdyUW482a2Mnk3d0mFu2pOOT1wLPRcQPcuX1uck+ATRczTENGJuuwtgdGAZMT4dAVkkameo8Fbi9SjFvL6lnwzDZSddnUmzj0mTjcstv95iTd+3JdeQ2zqlkm+br+ifgTw1f9pWi7I/tvgGcEBFv5cr7KfuvIiTtkeJ9qb3jTfFUcjtok5iBjwHPR8Tmw1dt2satuYpgW38AY8iunHoRuKQd4zicrFv5NPBkeowBbgBmpvJpQH1unktS3C+Qu5oIGEH2QXgRuIJ014MqxLwH2VUuTwGzGtqP7FjrvcDs9LxTB4q5B7Ac2DFX1qHamCzRLQLWk+0xnlHJNgW6kR3im0N2dc8eVYh3Dtkx+IZtueFKoU+lbeUp4Ang+LaOt5mYK7YdtEUbp/LJwNkl07ZZG/t2KmZmVpgPc5mZWWFOJmZmVpiTiZmZFeZkYmZmhTmZmJlZYU4mZq0kaXWV6z9PUo+2Wp5ZJTiZmHU855H9BsasZtTEf8CbdXSS9gR+CvQD3gLOiojnJU0GVpL9QGxX4MKIuEVSJ7Ifin0EmEu2Y3cdMCA97pO0LCKOSvVPILsdyRrgxEi3fzHrKNwzMauMa4CvRMTBwNeBK3Pj6snuanAcMDGVfZLsvyb+F3Am2f9KEBGTyO6RdFRDIgG2Bx6NiP2BB4GzqromZlvBPROzgtLdnT8E3Jz724euuUlui+yGgc9K6p/KDgduTuWvSbqvmUWsA+5Mw48DH69Y8GYV4mRiVlwnYEVEHNDE+LW5YZU8l2N9vHPfo434c2sdkA9zmRUU2X/NzJV0MmR3fZa0fwuzPQR8SlKn1Fs5MjduFdnfNZvVDCcTs9brIWlB7nE+8FngDEkNd01u6S+fbyW74+szwM/I/knzjTTuGuDuFg59mXUovmuwWTuRtENErJa0M9mtvg+LiNfaOy6zreFjr2bt505JvYHtgO84kVgtc8/EzMwK8zkTMzMrzMnEzMwKczIxM7PCnEzMzKwwJxMzMyvsfwCifT55gAjkowAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "article_lengths = [len(line.split()) for line in data['text']]\n",
    "avg_len = np.mean(article_lengths) \n",
    "print(f'Average length: {avg_len} \\n')\n",
    "plt.hist(article_lengths) \n",
    "plt.plot(avg_len, 100 ,'ro') \n",
    "plt.annotate(round(avg_len, 2), (avg_len, 160))\n",
    "plt.title('Distribution of the lengths of the News Articles')\n",
    "plt.xlabel('Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b98de1df-4cd7-4daf-aa52-6f83742c030b",
   "metadata": {},
   "source": [
    "Most of the lengths are below 2500. We will have to truncate the samples to a specific length later.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3ecde66-3714-4d2e-932d-06c8b58ccff2",
   "metadata": {},
   "source": [
    "Now, let's look at the common bigrams (combination of 2 words) in both classes: fake and not fake.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2f5db84a-cc5f-41d3-ba14-2b9fbca64620",
   "metadata": {},
   "outputs": [],
   "source": [
    "fake = data.loc[data['label']=='1'].reset_index(drop=True)\n",
    "not_fake = data.loc[data['label'] == '0'].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a71558a2-236b-4957-87a5-9e0779097df2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Get the latest from TODAY Sign up for our news...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2d  Conan On The Funeral Trump Will Be Invited...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>On March 24, 2018, one month and ten days afte...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A mosque in Georgia is under fire after they s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Scott Disick is sticking up for Khloe Kardashi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label\n",
       "0  Get the latest from TODAY Sign up for our news...     1\n",
       "1  2d  Conan On The Funeral Trump Will Be Invited...     1\n",
       "2  On March 24, 2018, one month and ten days afte...     1\n",
       "3  A mosque in Georgia is under fire after they s...     1\n",
       "4  Scott Disick is sticking up for Khloe Kardashi...     1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e5e4cd6c-8a26-4455-84ba-a217dbe3b956",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>It’s safe to say that Instagram Stories has fa...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Much like a certain Amazon goddess with a lass...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>At a time when the perfect outfit is just one ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>When Reese Witherspoon was 17, she had already...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>It’s official! WAGS Miami stars Ashley Nicole ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text label\n",
       "0  It’s safe to say that Instagram Stories has fa...     0\n",
       "1  Much like a certain Amazon goddess with a lass...     0\n",
       "2  At a time when the perfect outfit is just one ...     0\n",
       "3  When Reese Witherspoon was 17, she had already...     0\n",
       "4  It’s official! WAGS Miami stars Ashley Nicole ...     0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_fake.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d8820f3a-dbc8-4ce0-80c9-da4e46ea52d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_n_bigrams(n, corpus):\n",
    "    \"\"\" Returns the most probable n bigrams in the given corpus\"\"\"\n",
    "    vec = TfidfVectorizer(ngram_range=(2, 2),use_idf=False)\n",
    "    matrix = vec.fit_transform(corpus).toarray()\n",
    "    vocabulary = vec.get_feature_names() \n",
    "    return sorted(zip(matrix[0], vocabulary), key=lambda t: t[0])[-n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "54728392-8bc1-4785-8232-15e80f5955a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bigram</th>\n",
       "      <th>probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>at the</td>\n",
       "      <td>0.189514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>to the</td>\n",
       "      <td>0.193482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>on the</td>\n",
       "      <td>0.217805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>in the</td>\n",
       "      <td>0.430540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>of the</td>\n",
       "      <td>0.438623</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bigram  probability\n",
       "0  at the     0.189514\n",
       "1  to the     0.193482\n",
       "2  on the     0.217805\n",
       "3  in the     0.430540\n",
       "4  of the     0.438623"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake_corpus = [' '.join(fake['text'])]\n",
    "df = pd.DataFrame(get_top_n_bigrams(5, fake_corpus))\n",
    "df.columns = ['probability', 'bigram']\n",
    "df = df[['bigram', 'probability']]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e21e1761-8be4-47ad-ba80-30b23170c61d",
   "metadata": {},
   "source": [
    "In the fake articles, all of the top n bigrams are stop words. Those are words that are very common in the language and don't carry a lot of information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4de68def-88cf-4346-abe9-c80c1ba3571d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bigram</th>\n",
       "      <th>probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>at the</td>\n",
       "      <td>0.183098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>to the</td>\n",
       "      <td>0.202521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>on the</td>\n",
       "      <td>0.211595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>in the</td>\n",
       "      <td>0.407152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>of the</td>\n",
       "      <td>0.473564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bigram  probability\n",
       "0  at the     0.183098\n",
       "1  to the     0.202521\n",
       "2  on the     0.211595\n",
       "3  in the     0.407152\n",
       "4  of the     0.473564"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_fake_corpus = [' '.join(not_fake['text'])]\n",
    "df = pd.DataFrame(get_top_n_bigrams(5, not_fake_corpus))\n",
    "df.columns = ['probability', 'bigram']\n",
    "df = df[['bigram', 'probability']]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d7d7284-85a8-4a53-9d0a-04af7cf16cb7",
   "metadata": {},
   "source": [
    "Again here, as expected, the most common bigrams are stop words. \n",
    "\n",
    "\n",
    "The expectations is that we don't see these after cleaning the data of those stop words. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d22320ff-0eb7-4e35-af5a-69eeff4177e4",
   "metadata": {},
   "source": [
    "## Cleaning the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adac0556-9dd1-4c9e-96ed-06a9262b3448",
   "metadata": {},
   "source": [
    "The text feature contains raw text. The text needs to be cleaned before vectorizing it to get rid the tokens that would hurt the model. Tokens like punctuations, qoutation marks, emojis, links..etc. Such tokens could actually contribute to the target's value e.g. fake news usually contain punctuations, however, we tackle the task first using the clean text. We may later engineer more such features to help us improve the models' performance. \n",
    "\n",
    "\n",
    "Let's look at one news article before cleaning it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1ed31619-a770-4c6f-a4ea-8d4c9c3cdddc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Get the latest from TODAY Sign up for our newsletter  No one ever truly gets over losing a loved one, and Blake Shelton is no exception. He was just 14 when his older brother Richie died on Nov. 13, 1990. And, as Shelton noted in a tweet Monday, \"It changed my life forever.\"  Richie was 24 when he died in a car accident in the Sheltons\\' home state of Oklahoma. Two years ago, Shelton sent out a message for the 25th anniversary of his loss:  Richie, who was Blake\\'s half-brother (they shared a mother), was a passenger in a car that collided with a school bus in Ada, south of Oklahoma City.  Richie, driver Redena McManus and a 3-year-old boy, Christopher McManus, all died during or shortly after the collision, while the bus driver and passengers were uninjured, according to police reports.  The accident has clearly remained with Blake, who told 60 Minutes in 2014, \"I remember picking up the phone to call him a week after he was dead, to tell him something. I was picking up the phone to call him, to tell him something I just saw on TV or, and it was like constantly a shock to me that he was dead.\"  Blake Shelton playing at TODAY\\'s Halloween Extravaganza in New York City on Oct. 31. Getty Images  In 2011, Blake and his then-wife Miranda Lambert wrote a single called \"Over You,\" which was inspired by Richie.  Still, the two brothers had bonded despite the age difference; both shared a love of country music. \"His bedroom was right across the hallway from mine when I was little,\" Blake said in that interview. \"And he was listening to Hank Williams, Jr. or Waylon, Lynyrd Skynyrd or Bob Seeger. I just, whatever was popular really, Richie loved all music.  \"And I would be sitting there going, \\'Man, that guy\\'s my hero. That\\'s the coolest guy. He’s my big brother.\\'\"  Follow Randee Dawn on Twitter.'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = data['text'][0]\n",
    "text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f94e8a98-20f5-42b3-95c3-c9aa8788c07d",
   "metadata": {},
   "source": [
    "The text contains some junk that we need to clean. We will clean the text in the following steps: \n",
    "\n",
    "1) Removing punctuations.\n",
    "2) Removing digits.\n",
    "3) Remove the empty string, the one charachter tokens, and stop-words."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ccafa99-3ccf-40cd-b1ac-06da4102c4b8",
   "metadata": {},
   "source": [
    "The following function applies the above steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "109ddb49-5917-4b05-9f2f-9f4df3643be3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    \"\"\" Cleans a given text from punctuations, stop words, and numbers\"\"\"\n",
    "    stop_w = stopwords.words('English')\n",
    "    words = re.split(r'\\W+', text)  #punctionation\n",
    "    string_words = ' '.join((item for item in words if not item.isdigit())) #numbers\n",
    "    tokens = [token.lower() for token in string_words.split(\" \") if (token != \"\" and len(token)>1 and token not in stop_w)]  \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24ffda4e-61c8-4e6b-99d9-eaf57ef404ff",
   "metadata": {},
   "source": [
    "Let's now look at the output of the function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ce7e1bfb-04c0-4072-9302-9272a25b1b34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['get', 'latest', 'today', 'sign', 'newsletter', 'no', 'one', 'ever', 'truly', 'gets', 'losing', 'loved', 'one', 'blake', 'shelton', 'exception', 'he', 'older', 'brother', 'richie', 'died', 'nov', 'and', 'shelton', 'noted', 'tweet', 'monday', 'it', 'changed', 'life', 'forever', 'richie', 'died', 'car', 'accident', 'sheltons', 'home', 'state', 'oklahoma', 'two', 'years', 'ago', 'shelton', 'sent', 'message', '25th', 'anniversary', 'loss', 'richie', 'blake', 'half', 'brother', 'shared', 'mother', 'passenger', 'car', 'collided', 'school', 'bus', 'ada', 'south', 'oklahoma', 'city', 'richie', 'driver', 'redena', 'mcmanus', 'year', 'old', 'boy', 'christopher', 'mcmanus', 'died', 'shortly', 'collision', 'bus', 'driver', 'passengers', 'uninjured', 'according', 'police', 'reports', 'the', 'accident', 'clearly', 'remained', 'blake', 'told', 'minutes', 'remember', 'picking', 'phone', 'call', 'week', 'dead', 'tell', 'something', 'picking', 'phone', 'call', 'tell', 'something', 'saw', 'tv', 'like', 'constantly', 'shock', 'dead', 'blake', 'shelton', 'playing', 'today', 'halloween', 'extravaganza', 'new', 'york', 'city', 'oct', 'getty', 'images', 'in', 'blake', 'wife', 'miranda', 'lambert', 'wrote', 'single', 'called', 'over', 'you', 'inspired', 'richie', 'still', 'two', 'brothers', 'bonded', 'despite', 'age', 'difference', 'shared', 'love', 'country', 'music', 'his', 'bedroom', 'right', 'across', 'hallway', 'mine', 'little', 'blake', 'said', 'interview', 'and', 'listening', 'hank', 'williams', 'jr', 'waylon', 'lynyrd', 'skynyrd', 'bob', 'seeger', 'whatever', 'popular', 'really', 'richie', 'loved', 'music', 'and', 'would', 'sitting', 'going', 'man', 'guy', 'hero', 'that', 'coolest', 'guy', 'he', 'big', 'brother', 'follow', 'randee', 'dawn', 'twitter']\n"
     ]
    }
   ],
   "source": [
    "print(clean_text(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6af70b0-5941-486e-a988-dcdf71017561",
   "metadata": {},
   "source": [
    "We applied the function to only a single row. Let's apply the function to the whole dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f41f8cd4-1a59-4189-a690-35d9947559c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['text'] = data['text'].apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12a74ccd-b0d5-40f3-8541-870571ca1958",
   "metadata": {},
   "source": [
    "Let's look at the articles' lengths now. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b71cbfb7-e87c-4dbd-ab3d-3fd9bb8b16f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average length: 393.6187324508624 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjVElEQVR4nO3de7xVdZ3/8debO6ioCBg3QY1MtNQkdNTMSw3kXX8/jSaTUqPMaWqqUUinsTEms+kyalpMmWCloVZeRkbJS+Z4QXS8oaKEKEeQi4aAF66f+eP7PbDdnHPWPrD3ORt4Px+P/dhrf9da3/XZa6+9P2t9v2utrYjAzMysJR3aOwAzM6t/ThZmZlbIycLMzAo5WZiZWSEnCzMzK+RkYWZmhZws2oikn0r65yrVtZukFZI65tf3Sjq7GnXn+qZKGlOt+lqx3O9IWiLp1Qqnv0jSr6q07GskfacadW3CsudK+liNl7GrpPskLZf0g3qJa0tX6fe62t/R9uBkUQX5S/V2/iIulfSApC9KWr9+I+KLEXFxhXW1+AWNiJcjYvuIWFuF2Df6wY2IT0TEpM2tu5VxDAK+DgyLiPc0Mf4ISQ1tGVMttGNSGgssAXpGxNfLR9Y6Lkkh6anS70TeObimVsusRN7+Q9KICqb9rKT7S8sq/V5vDZwsquf4iNgBGAxcApwP/KLaC5HUqdp11onBwGsRsai9A9lKDQaeifa9Crc/MLodl/8ukgR8BngdaPFIeiv+3lUuIvzYzAcwF/hYWdkIYB2wb359DfCdPNwbuA1YStpQ/0xK3Nfmed4GVgDnAUOAAM4CXgbuKynrlOu7F/guMB14A7gZ6JXHHQE0NBUvMApYBazOy3uipL6z83AH4ELgJWARMBnYMY9rjGNMjm0JcEEL62nHPP/iXN+Fuf6P5fe8LsdxTdl825WNX0H64bkImJLrXA7MBIaXzNcfuCkv70XgH1qIbf3nk18fBzyeP6MHgA+Wrb9vAE/m9f1boFvJ+POABcB84Oy8jt5L2rtfndf5CuDWovpoZltp5j0cAjyS63gEOKTkvZUut3xbbXVcReuoidiCtAP1Ahu22++UftbAwbmepcATwBG5/EjgqZLp/ghML3l9P3BSHj4feCVvD7OAo1uI6XDSdnU68BrQpWTcZ4H/AX6U1/tNwDvA2ryOljaz3ZyY18ky4C/AqPLvVH59JvAs8FfgDmBwLlde5qK8zp8k/4a096PdA9gaHjSRLHL5y8A55RsV6Yf9p0Dn/PgIoKbqYsMP8mTSj2Z3mk4WrwD75mluAn6Vxx1BM8kiD1/UOG3J+PUbdt6oZwN7ANsDvwOuLYvtP3Nc+wErgb2bWU+TSYlshzzv88BZzcVZNm9T7+Oi/AU+BuiY1+tDeVwH4FHgW0CXHP8cYGQz9Zd+Ph/KX9aDcr1j8jrrWrL+ppOSUS/Sl/6Ledwo4FVgH6AHaQcggPeWL6fs82iuvma3lbI6epF+eD4DdAI+lV/v0txym3v/FcbV4jpqov4AhubPpHHbWp8sgAGkH+xj8mf38fy6D9CN9KPeO7+3V0mJeAfSdvc2sAuwFzAP6F+yfe7Zwnv+BWlno3Ne1ikl4z4LrAG+nJfZPZfd38J2M4L0A//x/B4GAO9v4jt1Euk7tXeu+0LggTxuZF5HO5ESx95Av/b+jYsIN0PV2HzSl6zcaqAfaW9idUT8OfKW0oKLIuLNiHi7mfHXRsTTEfEm8M/AaY0d4Jvp08API2JORKwAxgOjyw7Lvx0Rb0fEE6Q9wv3KK8mxfBIYHxHLI2Iu8APSj9vmuD8ibo/Uf3NtybI/DPSJiH+NiFURMYeU1CppBvk88LOIeDgi1kbqv1lJ2vNtdFlEzI+I14Fbgf1z+WnALyNiZkS8BXy7wvfRXH2VbivHAi9ExLURsSYirgOeA46vcPmtjauSdVQuSNvmtyR1LRt3OnB7/izXRcQ0YAZwTES8k4cPB4aT9rbvBw7Ny3shIl4j7fV3BYZJ6hwRcyPiL00FIqkHcCrwm4hYDdzIxk1R8yPi8rw+m/velToLuDoipuX38EpEPNfEdF8AvhsRz0bEGuDfgP0lDSZ93jsA7yftFDwbEQsqWHbNOVnU1gDSIWy575P2LO6UNEfSuArqmteK8S+R9pZ6VxRly/rn+krr7gTsWlJWevbSW6QjkHK9SXv45XUN2Mz4ypfdLSeywUD/fMLBUklLgW+Wxd2cwcDXy+YdRFoXzS238T33592fRdHnVlRfpdtK+ecEtVm/jXFVso42EhG3k464x5aNGgycWlbfYaRECfAn0tHl4Xn4XuCj+fGnXPds4KukI85Fkq6X1Fw8J5OOHG7Pr38NfEJSn5JpKv3sGg0iNT0VGQz8R8n7fJ10FDEgIu4GrgB+AiyUNFFSz1bGURNOFjUi6cOkL+r95ePynvXXI2IP0p7f1yQd3Ti6mSqLjjwGlQzvRtpDWQK8SWoOaYyrI+nQvtJ655M27tK61wALC+YrtyTHVF7XKxXO39qO2XnAixGxU8ljh4g4psJ5J5TN2yPvrRdZAAwseT2obHyr3kfBtlKq/HOC2q/fTV1HFwIXULJd5vquLatvu4i4JI8vTxZ/oixZAETEbyLiMNK6COB7zcQwhpT4Xs6nat9A2sH6VMk05eukaB3NA/YsmKZxui+UvdfuEfFAfg+XRcSBpKbM9wH/VEGdNedkUWWSeko6Drie1BfwVBPTHCfpvflsjGWkw+fG02AXktrXW+t0ScPy4fW/AjfmppnnSXvbx0rqTPqiljYBLASGlJ7SWOY64B8l7S5pe9Ih82/z4XPFcixTgAmSdsiH3F8DKr1OYiGwi6QdK5x+OrBM0vmSukvqKGnfnMSL/CfwRUkHKdkur78dKph3CvA5SXvnz+JbTbyPij/fgm2l1O3A+yT9naROkj4JDCN1jleitdvdJq+jiLgXeIp3N/v8Cjhe0sj8WXXLp0s3Jt4HSH0SI0id2zNJCeEg0kkfSNpL0lG5iesdUl/GRutK0gDgaFIH/f75sR8psZQ3RZVaCAyU1KWZ8b8gffZHS+ogaYCk9zcx3U+B8ZL2yfHsKOnUPPzhvE47k3b0GjvV252TRfXcKmk5aa/hAuCHwOeamXYo6YyOFcCDwJX5CwSpQ/PCfIj6jVYs/1pSZ9urpA7BfwCIiDeALwE/J+1lvgmUXq9wQ35+TdJjTdR7da77PtIZRe+QOv02xZfz8ueQjrh+k+svlNt+rwPm5HVT1NyxlrQnvn+OewlpHRQmm4iYQWqTv4LUSTyb1LlZSZxTgcuAe/J8D+ZRK/PzL0ht6ksl/aGCKlvaVkqX+xrpx+/rpM7a84DjImJJJXG3Nq7NWUfZhZT050XEPNKZRN8knb02j7RH3SGPfxN4DJgZEavybA8CL8WG0627kk5bX0L6HvTN9ZX7DPB4RNwZEa82Pkif2wcl7dtMzHeTzrh7VdJG6zUippO+8z8idXT/iY2P9oiI35MS0/WSlgFPA5/Io3uSEvFfSc2IrwH/3kw8barxDBwzqwFJe5N+DLq29mjMrJ74yMKsyiSdLKmLpJ1Je5C3OlHYls7Jwqz6vkBqSvkLqb35nPYNx2zz1bQZStJc0pWUa4E1ETFcUi/SlaBDSBfxnBYRf83Tjyedq7yWdLXtHbn8QFJ7fHdSR95XmjnX3MzMaqAtjiyOjIj9I2J4fj0OuCsihgJ35ddIGka6YGof0lWwV2rDRWVXkc7LHpofo9ogbjMzy9rj5lgnks6XBphEurjm/Fx+fUSsBF6UNBsYkY9OekbEgwCSJpMul5/a0kJ69+4dQ4YMqX70ZmZbsUcffXRJRPQpL691sgjSladBujXARGDXxsvXI2KBpL552gHAQyXzNuSy1bz7VM/G8o1IGku+MnS33XZjxowZ1XwvZmZbPUnldwIAap8sDo2I+TkhTJPU1H1SGqmJsmihfOPClIwmAgwfPtx9GmZmVVLTPouImJ+fFwG/J119uVBSP4D83HhBTQPvvjXCQNItDBp49+0TGsvNzKyN1CxZ5Mv/d2gcBv6WdHHSLWy4pH4M6ZbV5PLRkrpK2p3UkT09N1ktl3RwvuXBGSXzmJlZG6hlM9SuwO/T7zudSLcC/m9JjwBTJDX+mc+pABExU9IU4BnSjerOjQ1/G3oOG06dnUpB57aZmVXXVnu7j+HDh4c7uM3MWkfSoyWXOqznK7jNzKyQk4WZmRVysjAzs0JOFmZmVqg9bvdR94aM+692We7cS45tl+WamRXxkYWZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZoZonC0kdJf2vpNvy616Spkl6IT/vXDLteEmzJc2SNLKk/EBJT+Vxl0lSreM2M7MN2uLI4ivAsyWvxwF3RcRQ4K78GknDgNHAPsAo4EpJHfM8VwFjgaH5MaoN4jYzs6ymyULSQOBY4OclxScCk/LwJOCkkvLrI2JlRLwIzAZGSOoH9IyIByMigMkl85iZWRuo9ZHFj4HzgHUlZbtGxAKA/Nw3lw8A5pVM15DLBuTh8nIzM2sjNUsWko4DFkXEo5XO0kRZtFDe1DLHSpohacbixYsrXKyZmRWp5ZHFocAJkuYC1wNHSfoVsDA3LZGfF+XpG4BBJfMPBObn8oFNlG8kIiZGxPCIGN6nT59qvhczs21azZJFRIyPiIERMYTUcX13RJwO3AKMyZONAW7Ow7cAoyV1lbQ7qSN7em6qWi7p4HwW1Bkl85iZWRvo1A7LvASYIuks4GXgVICImClpCvAMsAY4NyLW5nnOAa4BugNT88PMzNpImySLiLgXuDcPvwYc3cx0E4AJTZTPAPatXYRmZtYSX8FtZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRWqWbKQ1E3SdElPSJop6du5vJekaZJeyM87l8wzXtJsSbMkjSwpP1DSU3ncZZJUq7jNzGxjFSULSftuQt0rgaMiYj9gf2CUpIOBccBdETEUuCu/RtIwYDSwDzAKuFJSx1zXVcBYYGh+jNqEeMzMbBNVemTx03yU8CVJO1UyQyQr8svO+RHAicCkXD4JOCkPnwhcHxErI+JFYDYwQlI/oGdEPBgRAUwumcfMzNpARckiIg4DPg0MAmZI+o2kjxfNJ6mjpMeBRcC0iHgY2DUiFuR6FwB98+QDgHklszfksgF5uLy8qeWNlTRD0ozFixdX8tbMzKwCFfdZRMQLwIXA+cBHgcskPSfplBbmWRsR+wMDSUcJLTVnNdUPES2UN7W8iRExPCKG9+nTp4VFmZlZa1TaZ/FBST8CngWOAo6PiL3z8I+K5o+IpcC9pL6Ghblpify8KE/WQDpyaTQQmJ/LBzZRbmZmbaTSI4srgMeA/SLi3Ih4DCAi5pOONjYiqU9j/4ak7sDHgOeAW4AxebIxwM15+BZgtKSuknYndWRPz01VyyUdnM+COqNkHjMzawOdKpzuGODtiFgLIKkD0C0i3oqIa5uZpx8wKZ/R1AGYEhG3SXoQmCLpLOBl4FSAiJgpaQrwDLAGOLdxecA5wDVAd2BqfpiZWRupNFn8kXRk0Hh2Uw/gTuCQ5maIiCeBA5oofw04upl5JgATmiifAWzK6btmZlYFlTZDdSs5DZY83KM2IZmZWb2pNFm8KelDjS8kHQi8XZuQzMys3lTaDPVV4AZJjWch9QM+WZOIzMys7lSULCLiEUnvB/YiXffwXESsrmlkZmZWNyo9sgD4MDAkz3OAJCJick2iMjOzulJRspB0LbAn8DjQeDpr432azMxsK1fpkcVwYFi+kZ+ZmW1jKj0b6mngPbUMxMzM6lelRxa9gWckTSf9TwUAEXFCTaIyM7O6UmmyuKiWQZiZWX2r9NTZP0kaDAyNiD9K6gF0LJrPzMy2DpXeovzzwI3Az3LRAOAPNYrJzMzqTKUd3OcChwLLYP0fIfVtcQ4zM9tqVJosVkbEqsYXkjrRzL/VmZnZ1qfSZPEnSd8Euuf/3r4BuLV2YZmZWT2pNFmMAxYDTwFfAG6nmX/IMzOzrU+lZ0OtA/4zP8zMbBtT6b2hXqSJPoqI2KPqEZmZWd1pzb2hGnUj/W92r+qHY2Zm9aiiPouIeK3k8UpE/Bg4qrahmZlZvai0GepDJS87kI40dqhJRGZmVncqbYb6QcnwGmAucFrVozEzs7pU6dlQR9Y6EDMzq1+VNkN9raXxEfHD6oRjZmb1qDVnQ30YuCW/Ph64D5hXi6DMzKy+tObPjz4UEcsBJF0E3BARZ9cqMDMzqx+V3u5jN2BVyetVwJCqR2NmZnWp0iOLa4Hpkn5PupL7ZGByzaIyM7O6UunZUBMkTQU+kos+FxH/W7uwzMysnlTaDAXQA1gWEf8BNEjavUYxmZlZnan0b1X/BTgfGJ+LOgO/qlVQZmZWXyo9sjgZOAF4EyAi5uPbfZiZbTMqTRarIiLItymXtF3tQjIzs3pTabKYIulnwE6SPg/8Ef8RkpnZNqMwWUgS8FvgRuAmYC/gWxFxecF8gyTdI+lZSTMlfSWX95I0TdIL+XnnknnGS5otaZakkSXlB0p6Ko+7LMdkZmZtpDBZ5OanP0TEtIj4p4j4RkRMq6DuNcDXI2Jv4GDgXEnDSP/nfVdEDAXuyq/J40YD+wCjgCsldcx1XQWMBYbmx6jWvEkzM9s8lTZDPSTpw62pOCIWRMRjeXg58CwwADgRmJQnmwSclIdPBK6PiJUR8SIwGxghqR/QMyIezIlrcsk8ZmbWBiq9gvtI4IuS5pLOiBLpoOODlcwsaQhwAPAwsGtELCBVsEBS3zzZAOChktkactnqPFxe3tRyxpKOQNhtt90qCc3MzCrQYrKQtFtEvAx8YlMXIGl7Ul/HVyNiWQvdDU2NiBbKNy6MmAhMBBg+fHiT05iZWesVNUP9ASAiXgJ+GBEvlT6KKpfUmZQofh0Rv8vFC3PTEvl5US5vAAaVzD4QmJ/LBzZRbmZmbaQoWZTu1e/RmorzGUu/AJ4t+3OkW4AxeXgMcHNJ+WhJXfOtRIYC03OT1XJJB+c6zyiZx8zM2kBRn0U0M1yJQ4HPAE9JejyXfRO4hHTdxlnAy8CpABExU9IU4BnSmVTnRsTaPN85wDVAd2BqfpiZWRspShb7SVpGOsLonodhQwd3z+ZmjIj7abq/AeDoZuaZAExoonwGsG9BrGZmViMtJouI6NjSeDMz2za05hblZma2jXKyMDOzQk4WZmZWyMnCzMwKOVmYmVkhJwszMyvkZGFmZoWcLMzMrJCThZmZFXKyMDOzQk4WZmZWyMnCzMwKOVmYmVkhJwszMyvkZGFmZoWcLMzMrJCThZmZFXKyMDOzQk4WZmZWyMnCzMwKOVmYmVkhJwszMyvkZGFmZoWcLMzMrJCThZmZFXKyMDOzQk4WZmZWyMnCzMwKOVmYmVkhJwszMyvkZGFmZoWcLMzMrJCThZmZFapZspB0taRFkp4uKeslaZqkF/LzziXjxkuaLWmWpJEl5QdKeiqPu0ySahWzmZk1rZZHFtcAo8rKxgF3RcRQ4K78GknDgNHAPnmeKyV1zPNcBYwFhuZHeZ1mZlZjNUsWEXEf8HpZ8YnApDw8CTippPz6iFgZES8Cs4ERkvoBPSPiwYgIYHLJPGZm1kbaus9i14hYAJCf++byAcC8kukactmAPFxe3iRJYyXNkDRj8eLFVQ3czGxbVi8d3E31Q0QL5U2KiIkRMTwihvfp06dqwZmZbevaOlkszE1L5OdFubwBGFQy3UBgfi4f2ES5mZm1obZOFrcAY/LwGODmkvLRkrpK2p3UkT09N1Utl3RwPgvqjJJ5zMysjXSqVcWSrgOOAHpLagD+BbgEmCLpLOBl4FSAiJgpaQrwDLAGODci1uaqziGdWdUdmJofZmbWhmqWLCLiU82MOrqZ6ScAE5oonwHsW8XQzMysleqlg9vMzOqYk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFnCzMzKyQk4WZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLMzMr5GRhZmaFOrV3AFuqWLOKV39zPrFmNaxbR4+9DmWnj3yaVYvm8NodPyFWvUOnHfvS+/h/okPXHqycP4vX7rgizxzsdNjf0eN9hzRZ9+WXX84VV1xBp06dOPbYY7n00kuZNm0a48aNY9WqVXTp0oXvf//7HHXUUW34js1sW+Zksak6dmbX0f9Ghy7dibVrePXX59F9jwN5/Y8/Y+cjz6Tbbh9gxZN3suzhm9jp8M/Quc9g+o35MerQkTUrXmfBL79M9/cehDp0fFe199xzDzfffDNPPvkkXbt2ZdGiRQD07t2bW2+9lf79+/P0008zcuRIXnnllfZ452a2DXIz1CaSRIcu3QGIdWtg3VqQWP16A10H7QtAtyEH8NbzDwDQoXO39Ykh1qwC1GS9V111FePGjaNr164A9O3bF4ADDjiA/v37A7DPPvvwzjvvsHLlypq9PzOzUk4WmyHWrWX+L79Mw+Wn023I/nTtvxddeg/m7dkPA/DWc/ezZvmS9dOvnD+L+T//Eguu/nt2GfmljY4qAJ5//nn+/Oc/c9BBB/HRj36URx55ZKNpbrrpJg444ID1CcXMrNbcDLUZ1KEj/T93OeveWcGi309g1eK57HLMV3j9jxN543+uy81MG1Zx1/570f/sK1m9ZB5Lbv8h3fcYjjp1eVeda9as4a9//SsPPfQQjzzyCKeddhpz5sxBSkciM2fO5Pzzz+fOO+9s0/dqZts2J4sq6NBte7oN+gBvz3mMHQ86hV0/eTEAq19/hbfnbHxk0Ln3INS5G6sWv0TXfkPfNW7gwIGccsopSGLEiBF06NCBJUuW0KdPHxoaGjj55JOZPHkye+65Z5u8NzMzcDNUq5ww8x7uv+pzzPne8dz8kzGMfHwqAOtWr+Sdlx6n8y4DWfvmUgAi1vHGA9ezw/6fAGD10leJdWsBWPPGIta8/gqdduy70TJOOukk7r77biA1Sa1atYrevXuzdOlSjj32WL773e9y6KGHtsG7NTPbwEcWFTph5j1c8t9X0GNN6lTWitd46I4rWfHAb1nWtQc93v8Rerx3BMtm3Mzyx/4LgB7vO4TtPvBxAFY2PMPih26Ejh2ROtDr4+fQsceOALw29TK2z0nlzDPP5Mwzz2TfffelS5cuTJo0CUlcccUVzJ49m4svvpiLL05HLnfeeef6DnAzs1pSRLR3DDUxfPjwmDFjxibNO2Tcf21Udv9Vn2PgssUblTf07MNh5/xyk5ZTT+Zecmx7h2BmdUDSoxExvLzczVAV6r9sSavKzcy2Jk4WFZrfs3erys3MtiZbTLKQNErSLEmzJY1r6+VfevgZvNXp3dc1vNWpK5cefkZbh2Jm1ua2iA5uSR2BnwAfBxqARyTdEhHPtFUMt+xzJADn3TeZ/suWML9nby49/Iz15Vu6pvpp2oL7Ssy2DFtEsgBGALMjYg6ApOuBE4E2SxaQEsbWkhzqRXslqW2Vk7Ntqi0lWQwA5pW8bgAOKp9I0lhgbH65QtKsTVxeb6Dee67rPcZ6jw+2wRj1vWrVtN42tw5rpJ5iHNxU4ZaSLJq6695G5/xGxERg4mYvTJrR1Klj9aTeY6z3+MAxVkO9xweOsVq2lA7uBmBQyeuBwPx2isXMbJuzpSSLR4ChknaX1AUYDdzSzjGZmW0ztohmqIhYI+nvgTuAjsDVETGzhovc7KasNlDvMdZ7fOAYq6He4wPHWBVb7e0+zMyseraUZigzM2tHThZmZlbIyaJEe95SRNIgSfdIelbSTElfyeW9JE2T9EJ+3rlknvE51lmSRpaUHyjpqTzuMjX+zV514uwo6X8l3Van8e0k6UZJz+V1+Td1GOM/5s/4aUnXSerW3jFKulrSIklPl5RVLSZJXSX9Npc/LGlIFeL7fv6cn5T0e0k7tVd8zcVYMu4bkkJS75KyNo9xs0SEH6nfpiPwF2APoAvwBDCsDZffD/hQHt4BeB4YBlwKjMvl44Dv5eFhOcauwO459o553HTgb0jXp0wFPlHFOL8G/Aa4Lb+ut/gmAWfn4S7ATvUUI+kC0xeB7vn1FOCz7R0jcDjwIeDpkrKqxQR8CfhpHh4N/LYK8f0t0CkPf68942suxlw+iHRyzktA7/aMcbO23bZcWD0/8odzR8nr8cD4doznZtK9sGYB/XJZP2BWU/HljfFv8jTPlZR/CvhZlWIaCNwFHMWGZFFP8fUk/RCrrLyeYmy8G0Ev0tmIt+UfvXaPERjCu3+MqxZT4zR5uBPpamVtTnxl404Gft2e8TUXI3AjsB8wlw3Jot1i3NSHm6E2aOqWIgPaI5B8eHkA8DCwa0QsAMjPjX+N11y8A/JweXk1/Bg4D1hXUlZP8e0BLAZ+mZvKfi5pu3qKMSJeAf4deBlYALwREXfWU4wlqhnT+nkiYg3wBrBLFWM9k7QXXlfxSToBeCUinigbVTcxVsrJYoOKbilS8yCk7YGbgK9GxLKWJm2iLFoo39y4jgMWRcSjlc7STBy1XM+dSM0AV0XEAcCbpOaT5rR5jLnd/0RS00N/YDtJp7c0SzOxtOf2uikx1XKdXgCsAX5dsKw2jU9SD+AC4FtNjW5mee2yDivhZLFBu99SRFJnUqL4dUT8LhcvlNQvj+8HLMrlzcXbkIfLyzfXocAJkuYC1wNHSfpVHcXXuMyGiHg4v76RlDzqKcaPAS9GxOKIWA38DjikzmJsVM2Y1s8jqROwI/D65gYoaQxwHPDpyO0zdRTfnqSdgify92Yg8Jik99RRjBVzstigXW8pks94+AXwbET8sGTULcCYPDyG1JfRWD46nyGxOzAUmJ6bC5ZLOjjXeUbJPJssIsZHxMCIGEJaN3dHxOn1El+O8VVgnqS9ctHRpNvY102MpOangyX1yHUfDTxbZzE2qmZMpXX9f9L2s7l77qOA84ETIuKtsrjbPb6IeCoi+kbEkPy9aSCdxPJqvcTY2jfkx4bOpGNIZyH9BbigjZd9GOmQ8kng8fw4htQmeRfwQn7uVTLPBTnWWZScCQMMB57O466gyp1gwBFs6OCuq/iA/YEZeT3+Adi5DmP8NvBcrv9a0hkx7RojcB2pD2U16UftrGrGBHQDbgBmk8722aMK8c0mteE3fl9+2l7xNRdj2fi55A7u9opxcx6+3YeZmRVyM5SZmRVysjAzs0JOFmZmVsjJwszMCjlZmJlZIScLs1aStKLG9X81X/3bJsszq4SThVn9+SrQo2gis7a0RfwHt1m9k7Qn8BOgD/AW8PmIeE7SNcAy0oVW7wHOi4gbJXUgXXD1UdKdcjsAV5PuF9UfuEfSkog4Mtc/gXRbi7eBEyNiYVu+PzMfWZhVx0TgyxFxIPAN4MqScf1IV+gfB1ySy04h3c76A8DZpNtTExGXke4FdGRjogC2Ax6KiP2A+4DP1/SdmDXBRxZmmynfKfgQ4AZt+LO6riWT/CEi1gHPSNo1lx0G3JDLX5V0TwuLWEX63wuAR0n/c2LWppwszDZfB2BpROzfzPiVJcMqe67E6thwX561+Htr7cDNUGabKdL/jrwo6VRIdxCWtF/BbPcD/09Sh3y0cUTJuOWkv9Y1qxtOFmat10NSQ8nja8CngbMkPQHMJP3BUUtuIt2Z9GngZ6R/RXwjj5sITC1omjJrU77rrFk7kbR9RKyQtAvpltOHRvqvA7O647ZPs/Zzm6SdgC7AxU4UVs98ZGFmZoXcZ2FmZoWcLMzMrJCThZmZFXKyMDOzQk4WZmZW6P8AP99IE/OmxdEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "article_lengths = [len(line) for line in data['text']]\n",
    "avg_len = np.mean(article_lengths) \n",
    "print(f'Average length: {avg_len} \\n')\n",
    "plt.hist(article_lengths) \n",
    "plt.plot(avg_len, 100 ,'ro') \n",
    "plt.annotate(round(avg_len, 2), (avg_len, 160))\n",
    "plt.title('Distribution of the lengths of the News Articles')\n",
    "plt.xlabel('Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51084a02-6cd6-4606-874e-7c4bf2db329e",
   "metadata": {},
   "source": [
    "The average length dropped from about 661 to almost 394. Besides, we don't see wild lengths (beyond 10k) as we saw earlier.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3a98cee-5271-402d-91b7-651e83dfb67e",
   "metadata": {},
   "source": [
    "Before we move on, let's look at the most common bigrams again after cleaning the data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3f9ac445-8292-4a64-a1a0-2e91bd66cfba",
   "metadata": {},
   "outputs": [],
   "source": [
    "fake = data.loc[data['label']=='1'].reset_index(drop=True)\n",
    "not_fake = data.loc[data['label'] == '0'].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0a652de7-0f25-45c3-adc1-2160a1b4fe54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bigram</th>\n",
       "      <th>probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>donald trump</td>\n",
       "      <td>0.104320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meghan markle</td>\n",
       "      <td>0.109695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>prince harry</td>\n",
       "      <td>0.110959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>kim kardashian</td>\n",
       "      <td>0.115701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>united states</td>\n",
       "      <td>0.138146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>brad pitt</td>\n",
       "      <td>0.143836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>getty images</td>\n",
       "      <td>0.151739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>los angeles</td>\n",
       "      <td>0.184299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>year old</td>\n",
       "      <td>0.236460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>new york</td>\n",
       "      <td>0.270601</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           bigram  probability\n",
       "0    donald trump     0.104320\n",
       "1   meghan markle     0.109695\n",
       "2    prince harry     0.110959\n",
       "3  kim kardashian     0.115701\n",
       "4   united states     0.138146\n",
       "5       brad pitt     0.143836\n",
       "6    getty images     0.151739\n",
       "7     los angeles     0.184299\n",
       "8        year old     0.236460\n",
       "9        new york     0.270601"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake_corpus = [' '.join([' '.join(text) for text in fake['text']])]\n",
    "df = pd.DataFrame(get_top_n_bigrams(10, fake_corpus))\n",
    "df.columns = ['probability', 'bigram']\n",
    "df = df[['bigram', 'probability']]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f78660f6-4f3c-4652-84fa-399d8d78be5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bigram</th>\n",
       "      <th>probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>donald trump</td>\n",
       "      <td>0.090138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>york city</td>\n",
       "      <td>0.095077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>first time</td>\n",
       "      <td>0.102486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>prince harry</td>\n",
       "      <td>0.105943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>social media</td>\n",
       "      <td>0.132615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>los angeles</td>\n",
       "      <td>0.141011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>united states</td>\n",
       "      <td>0.153112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>getty images</td>\n",
       "      <td>0.233866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>year old</td>\n",
       "      <td>0.270415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>new york</td>\n",
       "      <td>0.332154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          bigram  probability\n",
       "0   donald trump     0.090138\n",
       "1      york city     0.095077\n",
       "2     first time     0.102486\n",
       "3   prince harry     0.105943\n",
       "4   social media     0.132615\n",
       "5    los angeles     0.141011\n",
       "6  united states     0.153112\n",
       "7   getty images     0.233866\n",
       "8       year old     0.270415\n",
       "9       new york     0.332154"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_fake_corpus = [' '.join([' '.join(text) for text in not_fake['text']])]\n",
    "df = pd.DataFrame(get_top_n_bigrams(10, not_fake_corpus))\n",
    "df.columns = ['probability', 'bigram']\n",
    "df = df[['bigram', 'probability']]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ee79b8e-1a87-4de7-9d45-48854cad8ff6",
   "metadata": {},
   "source": [
    "We don't see any of the stop words we saw last time. The bigrams give us a clue about what the news talk about, but we can't tell why e.g. the donald trump bigram in the fake corpus could be an indicator of fake news and the one in the not-fake corpus is an indicator of true news. The context matters a lot. Maybe the verbs used in the context decide that, or even the nouns.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0a4b5f1-3dfb-4daa-9517-b3d291c46930",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Vectorization\n",
    "\n",
    "At this point, our data is clean. However, we need to represent each news article as an embedding vector. Vectorization can be done in many ways. There are simple methods such as TFIDF and sophisticated methods such as Word2Vec and Bert. We assume that Bert embeddings would perform better the the simple TFIDF embeddings. We will use the model trained with TFIDF embeddings as the baseline and benchmark the models trained with Bert embeddings against it. We also use Word2Vec embeddings besides the former two methods.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f1143e7-f081-427f-9ff2-6e5f9c27ba16",
   "metadata": {},
   "source": [
    "### TFIDF\n",
    "\n",
    "TF-IDF stands for Term Frequency and Inverse Document Frequency. Term Frequency is bascially the frequency of a word in a document. We could use the frequency alone as a weight for each word. However, this is misleading because the frequency of a word does not necessarily tell us that. TFIDF handles this by normalizing these frequencies by the inverse document frequency, which captures the rarity of the word. Due to this simplicity of the method, it does not capture any semantic of the words. Besides, it produces a sparse matrix of size N*V, where n is the number of documents and V is the size of the Vocabulary, all the unique words in the text. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a18d7fd3-9923-4eb1-9c61-494adb621b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = [' '.join(text) for text in data['text'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b44bf5f3-14e0-4771-8927-7517ec8daef8",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "50f3a569-264e-4d7b-978a-c65e3ec9c869",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_embeddings = tfidf_vectorizer.fit_transform(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d747fddc-99c7-4fbe-81b4-eea5d8e47f41",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_embeddings = tfidf_embeddings.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ad0b774a-f115-49eb-b9d1-3052b90075e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4986, 62335)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b946ac48-22cf-4120-ac11-96ff93e55c9a",
   "metadata": {},
   "source": [
    "### Word2Vec \n",
    "\n",
    "\n",
    "Word2Vec is a more sophisticated approach than TFIDF. It tries to learn an embedding vector for a word from its context. A model tries to predict a word given its context. In the end, we don't use those probabilities, but the weights of the neural netwroks as vector representations of the words. Word2Vec produces a dense vector. The size is generally 100-300. Due to this sophisticated method, the embedding vectors carry semantic information. Hence, applying vector operations on the embeddings vectors produces meaning results. \n",
    "\n",
    "\n",
    "The model below takes as an input a word. For our case, we need a vector representaiton for the whole document not just for a single word in the document. To tackle this problem, we get the embedding of each word then average all the vectors to end up with a single vector representing the document. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2545d90e-e102-4cd2-94b4-e194af0fe8b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model = Word2Vec(data['text'], vector_size = 300, window = 5, sg=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4ace6d39-7ea7-4f8b-8637-dd2e35d9ef9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_w2v_embeddings(corpus):\n",
    "    w2v_embeddings = []\n",
    "    for text in corpus:\n",
    "        word_embeddings = []\n",
    "        for word in text:\n",
    "            if not word in w2v_model.wv: continue  \n",
    "            word_embeddings.append(w2v_model.wv[word])\n",
    "        doc_embedding = np.mean(np.array(word_embeddings), axis=0)\n",
    "        w2v_embeddings.append(doc_embedding)\n",
    "    w2v_embeddings = np.array(w2v_embeddings)\n",
    "    return w2v_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f64d5d3f-ae05-406a-91cc-fd9886458d3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_embeddings = get_w2v_embeddings(data['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d2d8a612-5e8a-48cd-92e6-7724d7326aa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4986, 300)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83ead687-cb9b-4534-b963-33d871317774",
   "metadata": {},
   "source": [
    "### BERT\n",
    "\n",
    "\n",
    "Bert is a context dependent model. It captures all the word senses of a word. The input to the model is not a single word but a sentence. For example, the vector representation of a word depends on the sense of the word in that context. Word2Vec, on the other hand, would represent a word with a single vector for all the senses. Besides, Bert takes into account the position of the word unlike Word2Vec and TFIDF. \n",
    "\n",
    "\n",
    "Bert takes as an input a sentence, not a single word. That sentence's length should not exceed 512. And as we have seen above, some documents' length is way more than this. Therefore, we split such long documents into chunks to be able to input them to Bert. Then, as we did for Word2Vec, we get the embedding vector for each word in a document and average them to have a single embedding representation for the document.  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fda01540-99f5-4554-899d-a243861d5010",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\" This cell takes up to an hour to finish. \n",
    "    We pickled the embeddings so there would be no need to rerun it again\"\"\"\n",
    "\n",
    "def encode_text(text): \n",
    "    \"\"\" Returns a single bert vector representation for a text of any length\"\"\"\n",
    "    word_embeds = []\n",
    "    tokens = np.array(text.split())\n",
    "    num_sents = (len(tokens) + 512-1) // 512\n",
    "    sentences = np.array_split(tokens,  num_sents)\n",
    "    for idx, sent in enumerate(sentences):\n",
    "        sentence = Sentence(' '.join(sent))\n",
    "        embeddings.embed(sentence)\n",
    "        for tok in sentence: \n",
    "            word_embeds.append(tok.embedding.cpu().detach().numpy())\n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()\n",
    "    res = np.mean(np.array(word_embeds), axis=0)\n",
    "    return res\n",
    "\n",
    "def get_bert_embeddings():\n",
    "    \"\"\" Iterates through all the documents and embeds one at a time\"\"\"\n",
    "    embeddings = TransformerWordEmbeddings('bert-base-uncased')\n",
    "    bert_embeddings = []\n",
    "    for text in texts:\n",
    "        embed = encode_text(' '.join(text)) \n",
    "        bert_embeddings.append(embed)\n",
    "    return bert_embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b39bcce-de5a-4b60-9e19-34b6988854dc",
   "metadata": {},
   "source": [
    "Let's load the pickled embeddigns of the articles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0b88f466-e5da-4380-a052-cc44327d4b04",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/bert_embeddings.pickle', 'rb') as f:\n",
    "    bert_embeddings = np.array(pickle.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "25773baf-3cfe-4c1d-808d-d62f0a1f6969",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4986, 768)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bbd478e-3579-41d0-9b4c-d1cbb6c625e8",
   "metadata": {},
   "source": [
    "## Preprocessing the Data \n",
    "\n",
    "\n",
    "Before using the embeddings to train our models, we organize the data we have and preprocess the data such as scaling and splitting the data. We also have the option of reducing the dimensionality of our data using PCA.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "afb452c9-5143-435e-8272-fcf451e7d534",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = {'tfidf': tfidf_embeddings, 'w2v': w2v_embeddings, 'bert': bert_embeddings}\n",
    "y = data['label'].astype(int)\n",
    "\n",
    "class Data:\n",
    "    \"\"\"This class handles the preprocessing of the data: splitting, scaling, PCA\"\"\"\n",
    "    def __init__(self, X, y, embed='tfidf'):\n",
    "        self.embed = embed\n",
    "        self.X = X[self.embed]\n",
    "        self.y = y\n",
    "    \n",
    "    def get_data(self, scaled=False, pca=False):\n",
    "        \"\"\"Returns the splitted data. Scaled if scaled=True. Transformed with PCA if pca=True\"\"\"\n",
    "        X_train, X_test, y_train, y_test = self.split_data()\n",
    "        if pca: X_train, X_test = self.pca_data(X_train, X_test)\n",
    "        if scaled: X_train, X_test = self.scaled_data(X_train, X_test)\n",
    "        return X_train, X_test, y_train, y_test\n",
    "    \n",
    "    def split_data(self):\n",
    "        \"\"\"Splits the data into training and testing (.2 of the data)\"\"\"\n",
    "        return train_test_split(self.X, self.y, test_size=0.2, random_state=42)\n",
    "   \n",
    "    def pca_data(self, X_train, X_test):\n",
    "        \"\"\"Transforms the data using PCA\"\"\"\n",
    "        X_train, X_test = self.scaled_data(X_train, X_test)\n",
    "        pca = PCA(.95)\n",
    "        X_train = pca.fit_transform(X_train)\n",
    "        X_test = pca.transform(X_test)\n",
    "        return X_train, X_test\n",
    "        \n",
    "    def scaled_data(self, X_train, X_test):\n",
    "        \"\"\"Scales the data using the standard scaler\"\"\"\n",
    "        if self.embed == 'tfidf': return X_train, X_test\n",
    "        scaler = StandardScaler()\n",
    "        X_train = scaler.fit_transform(X_train)\n",
    "        X_test = scaler.transform(X_test)\n",
    "        return X_train, X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "398078ad-291b-4025-9c27-c13075d4121d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_data = Data(X, y, embed='tfidf')\n",
    "w2v_data = Data(X, y, embed='w2v')\n",
    "bert_data = Data(X, y, embed='bert')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14940593-e3e7-46cc-a240-b7195385924e",
   "metadata": {},
   "source": [
    "## Modelling\n",
    "\n",
    "\n",
    "Now, it is time to train some models on the embeddings we have. We train several classification mehtods compare their results. We compare the results of the models on the actual data and the data transformed with PCA. We combine differnt models of different embedding techniques (TFIDF, Word2Vec, and Bert) and have a voting system on their output. Finally, we interpret the results and discuss some possible improvements.\n",
    "\n",
    "\n",
    "We start with defninig a class and some functions to organize the section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "879a7070-abc2-448c-b9e0-95e640d2352c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model():\n",
    "    \"\"\"This class organizes the modelling process\"\"\"\n",
    "    def __init__(self, model):\n",
    "        self.model = model\n",
    "        \n",
    "    def train(self, X, y):\n",
    "        \"\"\"trains the model on the given X and y training data\"\"\"\n",
    "        self.model.fit(X, y)\n",
    "    \n",
    "    def accuracy_score(self, X, y, cv=False): \n",
    "        \"\"\"Returns the accuracy score. cv=True will cause it to return the mean of scores\"\"\"\n",
    "        if cv: return np.mean(cross_val_score(self.model, X, y, cv=3, scoring='accuracy'))\n",
    "        preds = self.model.predict(X)\n",
    "        return accuracy_score(y, preds)\n",
    "    \n",
    "    def f1_score(self, X, y, cv=False):\n",
    "        \"\"\"Returns the f1_score. cv=True will cause it to return the mean of the scores\"\"\"\n",
    "        if cv: return np.mean(cross_val_score(self.model, X, y, cv=3, scoring='f1'))\n",
    "        preds = self.model.predict(X) \n",
    "        return f1_score(y, preds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "fce5df25-963e-48b6-8992-9d81b3d14c6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_dict(models, data_all, pca=False):\n",
    "    \"\"\"Returns a result dict of all embeddings methods for all methods for all the metrics\"\"\"\n",
    "    scores_ = {}\n",
    "    for key, data in data_all.items():\n",
    "        X_train, X_test, y_train, y_test =  data.get_data(pca=pca, scaled=True) \n",
    "        sub_scores = {}\n",
    "        for model_name, model in models.items(): \n",
    "            clf = Model(model)\n",
    "            clf.train(X_train, y_train)\n",
    "            scores = [clf.accuracy_score(X_train, y_train, cv=True), clf.accuracy_score(X_train, y_train),\n",
    "                      clf.accuracy_score(X_test, y_test), clf.f1_score(X_train, y_train, cv=True),\n",
    "                      clf.f1_score(X_train, y_train), clf.f1_score(X_test, y_test)]\n",
    "            sub_scores[model_name] = scores \n",
    "        scores_[key] = sub_scores\n",
    "    return scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "bf4af74d-972f-4e3b-85cb-f51df2db7db4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_result_tables(result_dict, indexes):\n",
    "    \"\"\"This functions shows the result tables given the result dict\"\"\"\n",
    "    score_names = ['Accuracy Score (CV=3)', 'Accuracy Score (Train)', 'Accuracy Score (Test)', 'F1_Score (CV=3)', 'F1_Score (Train)', 'F1_Score (Test)']\n",
    "    for key, val in result_dict.items():\n",
    "        scrs = np.array(list(val.values()))\n",
    "        df = pd.DataFrame(scrs)\n",
    "        df.index = indexes\n",
    "        df.columns = score_names\n",
    "        display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6c331e0-ce78-4df1-8173-abe31477cfea",
   "metadata": {},
   "source": [
    "Now, let's look at the results using the actual data without applying PCA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "87dd3298-96f8-4cb9-bac5-a8758c6e9573",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {'lg': LogisticRegression(C=.5), 'svc':  LinearSVC(C=.01) , 'rf': RandomForestClassifier(max_depth=5)}\n",
    "data_all = {'tfidf': tfidf_data, 'w2v': w2v_data, 'bert': bert_data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "02f4c5c3-bc17-4b8d-97bf-1de82fdbcc0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dict = get_results_dict(models, data_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "745b02ac-bd9b-4647-b98e-e03cdf36e11b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy Score (CV=3)</th>\n",
       "      <th>Accuracy Score (Train)</th>\n",
       "      <th>Accuracy Score (Test)</th>\n",
       "      <th>F1_Score (CV=3)</th>\n",
       "      <th>F1_Score (Train)</th>\n",
       "      <th>F1_Score (Test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.740471</td>\n",
       "      <td>0.802909</td>\n",
       "      <td>0.764529</td>\n",
       "      <td>0.571098</td>\n",
       "      <td>0.697460</td>\n",
       "      <td>0.643399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.657724</td>\n",
       "      <td>0.696841</td>\n",
       "      <td>0.678357</td>\n",
       "      <td>0.266510</td>\n",
       "      <td>0.408223</td>\n",
       "      <td>0.397749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.633902</td>\n",
       "      <td>0.639920</td>\n",
       "      <td>0.608216</td>\n",
       "      <td>0.155641</td>\n",
       "      <td>0.183163</td>\n",
       "      <td>0.129176</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Accuracy Score (CV=3)  Accuracy Score (Train)  \\\n",
       "Logistic Regression                  0.740471                0.802909   \n",
       "LinearSVC                            0.657724                0.696841   \n",
       "RandomForestClassifier               0.633902                0.639920   \n",
       "\n",
       "                        Accuracy Score (Test)  F1_Score (CV=3)  \\\n",
       "Logistic Regression                  0.764529         0.571098   \n",
       "LinearSVC                            0.678357         0.266510   \n",
       "RandomForestClassifier               0.608216         0.155641   \n",
       "\n",
       "                        F1_Score (Train)  F1_Score (Test)  \n",
       "Logistic Regression             0.697460         0.643399  \n",
       "LinearSVC                       0.408223         0.397749  \n",
       "RandomForestClassifier          0.183163         0.129176  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy Score (CV=3)</th>\n",
       "      <th>Accuracy Score (Train)</th>\n",
       "      <th>Accuracy Score (Test)</th>\n",
       "      <th>F1_Score (CV=3)</th>\n",
       "      <th>F1_Score (Train)</th>\n",
       "      <th>F1_Score (Test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.738216</td>\n",
       "      <td>0.785607</td>\n",
       "      <td>0.765531</td>\n",
       "      <td>0.655043</td>\n",
       "      <td>0.712024</td>\n",
       "      <td>0.692105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.746742</td>\n",
       "      <td>0.786359</td>\n",
       "      <td>0.766533</td>\n",
       "      <td>0.660933</td>\n",
       "      <td>0.708618</td>\n",
       "      <td>0.691391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.742478</td>\n",
       "      <td>0.792377</td>\n",
       "      <td>0.741483</td>\n",
       "      <td>0.599013</td>\n",
       "      <td>0.675803</td>\n",
       "      <td>0.603077</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Accuracy Score (CV=3)  Accuracy Score (Train)  \\\n",
       "Logistic Regression                  0.738216                0.785607   \n",
       "LinearSVC                            0.746742                0.786359   \n",
       "RandomForestClassifier               0.742478                0.792377   \n",
       "\n",
       "                        Accuracy Score (Test)  F1_Score (CV=3)  \\\n",
       "Logistic Regression                  0.765531         0.655043   \n",
       "LinearSVC                            0.766533         0.660933   \n",
       "RandomForestClassifier               0.741483         0.599013   \n",
       "\n",
       "                        F1_Score (Train)  F1_Score (Test)  \n",
       "Logistic Regression             0.712024         0.692105  \n",
       "LinearSVC                       0.708618         0.691391  \n",
       "RandomForestClassifier          0.675803         0.603077  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy Score (CV=3)</th>\n",
       "      <th>Accuracy Score (Train)</th>\n",
       "      <th>Accuracy Score (Test)</th>\n",
       "      <th>F1_Score (CV=3)</th>\n",
       "      <th>F1_Score (Train)</th>\n",
       "      <th>F1_Score (Test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.700098</td>\n",
       "      <td>0.844032</td>\n",
       "      <td>0.735471</td>\n",
       "      <td>0.628242</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.674074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.718655</td>\n",
       "      <td>0.834504</td>\n",
       "      <td>0.751503</td>\n",
       "      <td>0.643527</td>\n",
       "      <td>0.784173</td>\n",
       "      <td>0.685279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.699348</td>\n",
       "      <td>0.776830</td>\n",
       "      <td>0.689379</td>\n",
       "      <td>0.477175</td>\n",
       "      <td>0.633443</td>\n",
       "      <td>0.461806</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Accuracy Score (CV=3)  Accuracy Score (Train)  \\\n",
       "Logistic Regression                  0.700098                0.844032   \n",
       "LinearSVC                            0.718655                0.834504   \n",
       "RandomForestClassifier               0.699348                0.776830   \n",
       "\n",
       "                        Accuracy Score (Test)  F1_Score (CV=3)  \\\n",
       "Logistic Regression                  0.735471         0.628242   \n",
       "LinearSVC                            0.751503         0.643527   \n",
       "RandomForestClassifier               0.689379         0.477175   \n",
       "\n",
       "                        F1_Score (Train)  F1_Score (Test)  \n",
       "Logistic Regression             0.800000         0.674074  \n",
       "LinearSVC                       0.784173         0.685279  \n",
       "RandomForestClassifier          0.633443         0.461806  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "indexes = ['Logistic Regression', 'LinearSVC', 'RandomForestClassifier']\n",
    "show_result_tables(result_dict, indexes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b844886-cb04-4518-bc11-1c57fe217de0",
   "metadata": {},
   "source": [
    "### PCA\n",
    "\n",
    "Here, we apply PCA transformation to the data first before running the above experiment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a3d824c2-a8d9-4b78-9542-80ed9c74b2c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dict_pca = get_results_dict(models, data_all, pca=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "88e88329-417a-441c-8e95-4bece098991b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy Score (CV=3)</th>\n",
       "      <th>Accuracy Score (Train)</th>\n",
       "      <th>Accuracy Score (Test)</th>\n",
       "      <th>F1_Score (CV=3)</th>\n",
       "      <th>F1_Score (Train)</th>\n",
       "      <th>F1_Score (Test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.742477</td>\n",
       "      <td>0.800401</td>\n",
       "      <td>0.765531</td>\n",
       "      <td>0.575510</td>\n",
       "      <td>0.693846</td>\n",
       "      <td>0.645455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.657472</td>\n",
       "      <td>0.695587</td>\n",
       "      <td>0.678357</td>\n",
       "      <td>0.267165</td>\n",
       "      <td>0.405485</td>\n",
       "      <td>0.397749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.677532</td>\n",
       "      <td>0.746239</td>\n",
       "      <td>0.691383</td>\n",
       "      <td>0.382667</td>\n",
       "      <td>0.542909</td>\n",
       "      <td>0.465278</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Accuracy Score (CV=3)  Accuracy Score (Train)  \\\n",
       "Logistic Regression                  0.742477                0.800401   \n",
       "LinearSVC                            0.657472                0.695587   \n",
       "RandomForestClassifier               0.677532                0.746239   \n",
       "\n",
       "                        Accuracy Score (Test)  F1_Score (CV=3)  \\\n",
       "Logistic Regression                  0.765531         0.575510   \n",
       "LinearSVC                            0.678357         0.267165   \n",
       "RandomForestClassifier               0.691383         0.382667   \n",
       "\n",
       "                        F1_Score (Train)  F1_Score (Test)  \n",
       "Logistic Regression             0.693846         0.645455  \n",
       "LinearSVC                       0.405485         0.397749  \n",
       "RandomForestClassifier          0.542909         0.465278  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy Score (CV=3)</th>\n",
       "      <th>Accuracy Score (Train)</th>\n",
       "      <th>Accuracy Score (Test)</th>\n",
       "      <th>F1_Score (CV=3)</th>\n",
       "      <th>F1_Score (Train)</th>\n",
       "      <th>F1_Score (Test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.740472</td>\n",
       "      <td>0.768054</td>\n",
       "      <td>0.761523</td>\n",
       "      <td>0.647678</td>\n",
       "      <td>0.682021</td>\n",
       "      <td>0.680108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.742979</td>\n",
       "      <td>0.765045</td>\n",
       "      <td>0.761523</td>\n",
       "      <td>0.647398</td>\n",
       "      <td>0.673633</td>\n",
       "      <td>0.673973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.721414</td>\n",
       "      <td>0.757773</td>\n",
       "      <td>0.719439</td>\n",
       "      <td>0.506109</td>\n",
       "      <td>0.579634</td>\n",
       "      <td>0.522184</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Accuracy Score (CV=3)  Accuracy Score (Train)  \\\n",
       "Logistic Regression                  0.740472                0.768054   \n",
       "LinearSVC                            0.742979                0.765045   \n",
       "RandomForestClassifier               0.721414                0.757773   \n",
       "\n",
       "                        Accuracy Score (Test)  F1_Score (CV=3)  \\\n",
       "Logistic Regression                  0.761523         0.647678   \n",
       "LinearSVC                            0.761523         0.647398   \n",
       "RandomForestClassifier               0.719439         0.506109   \n",
       "\n",
       "                        F1_Score (Train)  F1_Score (Test)  \n",
       "Logistic Regression             0.682021         0.680108  \n",
       "LinearSVC                       0.673633         0.673973  \n",
       "RandomForestClassifier          0.579634         0.522184  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy Score (CV=3)</th>\n",
       "      <th>Accuracy Score (Train)</th>\n",
       "      <th>Accuracy Score (Test)</th>\n",
       "      <th>F1_Score (CV=3)</th>\n",
       "      <th>F1_Score (Train)</th>\n",
       "      <th>F1_Score (Test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.721165</td>\n",
       "      <td>0.785105</td>\n",
       "      <td>0.741483</td>\n",
       "      <td>0.636818</td>\n",
       "      <td>0.717255</td>\n",
       "      <td>0.673418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.724174</td>\n",
       "      <td>0.783099</td>\n",
       "      <td>0.741483</td>\n",
       "      <td>0.639045</td>\n",
       "      <td>0.712720</td>\n",
       "      <td>0.665803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.638415</td>\n",
       "      <td>0.696088</td>\n",
       "      <td>0.638277</td>\n",
       "      <td>0.186821</td>\n",
       "      <td>0.389113</td>\n",
       "      <td>0.246347</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Accuracy Score (CV=3)  Accuracy Score (Train)  \\\n",
       "Logistic Regression                  0.721165                0.785105   \n",
       "LinearSVC                            0.724174                0.783099   \n",
       "RandomForestClassifier               0.638415                0.696088   \n",
       "\n",
       "                        Accuracy Score (Test)  F1_Score (CV=3)  \\\n",
       "Logistic Regression                  0.741483         0.636818   \n",
       "LinearSVC                            0.741483         0.639045   \n",
       "RandomForestClassifier               0.638277         0.186821   \n",
       "\n",
       "                        F1_Score (Train)  F1_Score (Test)  \n",
       "Logistic Regression             0.717255         0.673418  \n",
       "LinearSVC                       0.712720         0.665803  \n",
       "RandomForestClassifier          0.389113         0.246347  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_result_tables(result_dict_pca, indexes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8aa7395b-6bdc-4061-b9f9-530fd3a2274e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Ensemble Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ccaad2bb-bc90-4ca8-8d43-d274c1ffee30",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_distribution(y_true, predictions, voting):\n",
    "    \"\"\"plots the distributin of y(true) Vs. the predictions\"\"\"\n",
    "    bins = [-.1, .1, .9, 1.1]\n",
    "    plt.hist(y_true, alpha=.7, color='blue', label='true', bins=bins)\n",
    "    plt.hist(predictions, color='red', label = 'predictions')\n",
    "    plt.legend(fontsize=14)\n",
    "    plt.xlabel('Label')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title(f'{voting}: Gold Vs. Predictions')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f39886ae-949b-4919-9741-1bdf2904a5e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ensemble(voting=\"soft\"):\n",
    "    svc = LinearSVC(C=.01)\n",
    "    svc = CalibratedClassifierCV(svc) \n",
    "    lg = LogisticRegression(max_iter=2000, C=.5)\n",
    "    data_to_use = [tfidf_data, w2v_data, bert_data]\n",
    "    scores = []\n",
    "    for idx, model in enumerate([Model(lg), Model(svc), Model(svc)]):\n",
    "        X_train, X_test, y_train, y_test =  data_to_use[idx].get_data() \n",
    "        model.train(X_train, y_train)\n",
    "        if voting==\"soft\":\n",
    "            s1, s2 = model.model.predict_proba(X_train), model.model.predict_proba(X_test)\n",
    "        else: \n",
    "            s1, s2 = model.model.predict(X_train), model.model.predict(X_test)\n",
    "        scores.append([s1, s2])\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b73456b8-d43e-476a-8691-694244daf98a",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Hard Voting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bcf6f816-32ae-4518-b8cb-afce344b7bbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = ensemble(voting=\"hard\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "3b0f5f50-f261-4d3a-9355-fe5a95b615ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_preds = stats.mode(np.array([sc[0] for sc in scores]))[0][0]\n",
    "test_preds = stats.mode(np.array([sc[1] for sc in scores]))[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "6cccced8-5f53-457d-8ddc-faca7200c282",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy score (train)</th>\n",
       "      <th>accuracy score (test)</th>\n",
       "      <th>f1 score (train)</th>\n",
       "      <th>f1 score (test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ensemble</th>\n",
       "      <td>0.777834</td>\n",
       "      <td>0.755511</td>\n",
       "      <td>0.678753</td>\n",
       "      <td>0.651429</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          accuracy score (train)  accuracy score (test)  f1 score (train)  \\\n",
       "ensemble                0.777834               0.755511          0.678753   \n",
       "\n",
       "          f1 score (test)  \n",
       "ensemble         0.651429  "
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def show_ensemble_table(y_train, y_test, train_preds, test_preds):\n",
    "    res_d = {'accuracy score (train)': accuracy_score(y_train, train_preds), 'accuracy score (test)': accuracy_score(y_test, test_preds),\n",
    "         'f1 score (train)': f1_score(y_train, train_preds), 'f1 score (test)': f1_score(y_test, test_preds)}\n",
    "    df = pd.DataFrame(res_d, index=['ensemble'])\n",
    "    return df\n",
    "hard_df = show_ensemble_table(y_train, y_test, train_preds, test_preds)\n",
    "hard_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "e4da91de-7a51-42af-9db0-1eb73109799c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of missclassified: 244\n"
     ]
    }
   ],
   "source": [
    "hard_miss_count = len(y_test[y_test!=test_preds])\n",
    "print(f'# of missclassified: {hard_miss_count}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "40c75aad-59d8-4acd-bc29-f2e08470b4bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjH0lEQVR4nO3deZwV5Z3v8c9XBaRBBGUJNgiiTNw3WsVropjMKCpuM6PBEATRMRJNNDMa11G8XryaqBGzuCQqos4oEcaFiMYQ0TguBARlVwxGtgDGMSgS1t/9o6r7HppD12noc/o0/X2/XufVVU89VfV7Tnef36mnqp5SRGBmZlaXnRo7ADMzK39OFmZmlsnJwszMMjlZmJlZJicLMzPL5GRhZmaZnCys7EkaKum1Eu5vtKT/U8fykLRfqeIpNkn9JC3OmZ8tqd82bOerkuY3ZGxWPpwsbLtI+lDS39cqK+mHe56YWkq6UdJ8SaslLZE0UdJJRd7v/ZLG5Ck/VNJaSXtsx7ZHSFov6XNJn0p6XdKx2xdxfhFxUERMLiCmzZJmRPw+Ir5cjJis8TlZWFmRtEsDbOYp4EzgfKADsA8wCjitAbZdl9HAP0pqU6v8fGBCRHyyndt/MiLaAp2A14DxklS7kqSdt3M/ZltwsrCik3SNpA8kfSZpjqSzc5YNlfTfkn4s6RNghKQ9JT0raZWkKcC+9djX3wP/AJwZEW9FxLr09UJEXJ5T7wBJk9Nv6bMlnVHHNq+StEzSUknDtlYvIt4AlgD/lLPuzsA3gUfS+aMlTU3btlzSXYW2LWc/69PtfQnYM+02u1fS85JWAydK2kvSOEkrJS2U9L2cmFqn6/yPpDnAUbXaW3O0KGlnSdfl/P6mSeou6dW0+jvp0c438nRnbfU9Tvf/M0m/Trf7lqR902VK/x5WSPqrpHclHVzf98kalpOFlcIHwFeB3YGbgcckdc1ZfgzwR6AzMBL4GfA3oCswLH3VkDRB0jVb2dffA29FxOKtLEdSC+A54DfpPr8LPC5piy4USf2BK0kSUO90+3UZQ3IkkRtPC2BiOj8KGBUR7UiS4NiM7eWLvxUwFFgcER+nxd8kee92A14nad87QCXwdeAKSSendW9K970vcDIwpI7d/StwHnAq0I7kd/FFRByfLj8sItpGxJO1YizkPT6P5O+hA7AgjR/gJOB44O+A9sA3gL9kvC1WZE4W1hCeTr89firpU+DnuQsj4lcRsTQiNqUfKu8DR+dUWRoRP4mIDcA6km/mN0bE6oiYRfqtPGd7AyLitq3E0hH4c/WMpD3SuP4q6W9pcV+gLXBbetTxO2ACyYdXbecCD0fErIhYDYzIeC8eBU6Q1C2dPx/4j/RoAGA9sJ+kjhHxeUS8mbG9zWJJ399FQB/grJxlz0TEf0fEJuAQoFNE/O+0fX8EfgEMzGnTyIj4JCIWAffUsc+LgBsiYn4k3omIQj64C3mPx0fElPT3/jhweFq+niTp7Q8oIuZGxLIC9mlF5GRhDeGsiGhf/QK+k7tQ0vmSZuQkk4NJPtSrLcqZ7gTsUqvsT/WI5S8kRyQApB+I7Uk+XFulxXsBi9IP1tx9VObZ3l71iSUiPgJeBb4lqS3JB3pusruQ5BvzPEl/kDSggDZVG5u+x50j4msRMS1nWW6MPYC9aiXw64Au29Cm7iRHhvVVyHv855zpL0iSC2li+SnJEeZySQ9IarcNMVgDcrKwopLUg+Rb7WXAnukH9ywg98Rs7tDHK4ENJB9S1fauxy4nAUflfLPPZynQXVLu3//eJOcbalu2DbE8QnJE8U/Awoh4u3pBRLwfEeeRdM3cDjyV54T4tsh9Dxel+22f89otIk5Nl9enTYuoxzmjHPV5j7cQEfdERB/gIJLketU2xGANyMnCiq0NyQfZSgBJF5AcWeQVERuB8SQnuiskHUjdfeq11/8N8DJJ19gxSi6jbUHSLVLtLWA18ANJLZTcU3A68ESeTY4Fhko6UFIFSX9/lnEkH8Y3U6sLTdK3JHVKv3F/mhZvLLR9BZoCrJJ0dXoye2dJB0uqPpE9FrhWUoc0qX63jm39ErhFUu/0xPOhkvZMly0Hem1lvfq8x5uRdFT6u2uRbuNvNPx7ZPXkZGFFFRFzgDuBN0g+XA4B/jtjtctIuiT+THI56sO5C5XcM3FdHev/I0n/+GMkH8gLgUFA/zSmdcAZwCnAxyTnWM6PiHl54p8I3A38juQk7O8yYic9t1GdMB6vtbg/MFvS5yQnuwdGxN/Sdn0u6atZ2y9g/xtJPpgPJ2n7xyQf+runVW4m6RJaSHIC+tE6NncXSXL5DbAKeBBonS4bATySdnWdWyuGgt/jPNqRHI3+TxrnX4A7CljPikh++JGZmWXxkYWZmWVysjAzs0xOFmZmlsnJwszMMjXEoG1lqWPHjtGzZ8/GDsPMrEmZNm3axxHRqXb5DpssevbsydSpUxs7DDOzJkVS3jv63Q1lZmaZnCzMzCyTk4WZmWVysjAzs0xOFmZmlmmHvRrKzLZu1apVrFixgvXr12dXth1GixYt6Ny5M+3a1f/xIE4WZs3MqlWrWL58OZWVlbRu3RpJ2StZkxcRrFmzhiVLkkeK1DdhuBvKrJlZsWIFlZWVVFRUOFE0I5KoqKigsrKSFStW1Ht9JwuzZmb9+vW0bt06u6LtkFq3br1N3Y/uhmospfpG5+eVWB4+omi+tvV37yMLMzPL5GRhZmaZ3A1lZgCcfnpp9/fcc6Xdn20fH1mYWZPRr18/LrvsssYOo1lysjCzHYpvNCwOJwszaxKGDh3KK6+8ws9+9jMkIYnRo0cjieeff56jjz6ali1b8uKLLzJixAgOPvjgzdYfPXo0bdu23azsueeeo0+fPuy6667ss88+XH/99axbt66UzWoyfM7CzJqEUaNG8d5777H//vtz6623AjB79mwArr76au688072228/dtttt4IefPbiiy8yaNAgRo0axfHHH89HH33EJZdcwtq1a7njjjuK2pamyMnCzJqE3XffnZYtW1JRUcGXvvQlAObNmwfAiBEjOOmkk+q1vZEjR3LVVVdxwQUXALDvvvty++23861vfYsf/ehHvhelFicLM2vyqqqq6r3OtGnTmDJlCrfffntN2aZNm1izZg1//vOf6dq1a0OG2OQ5WZhZk9emTZvN5nfaaSei1ugFtU98b9q0iZtuuolzzjlni+116tSp4YNs4pwszKzJaNmyJRs3bsys16lTJ5YvX05E1HQnzZgxY7M6Rx55JPPmzWO//fYrRqg7HCcLM2syevbsyZQpU/jwww9p27YtmzZtyluvX79+fPLJJ9x6660MHDiQyZMn89RTT21W58Ybb2TAgAH06NGDc889l1122YVZs2YxZcoUfvjDH5aiOU1LROyQrz59+kRZS4b4K/7LrJY5c+Y0dgjbbP78+dG3b99o3bp1APHwww8HECtXrtyi7n333Rd77713VFRUxDe+8Y24++67o02bNpvVefHFF+MrX/lKtG7dOnbbbbfo06dP/OQnPylVcxpNXX8DwNTI85mq2EFHJa2qqopCLp9rNB511hrJ3LlzOeCAAxo7DGtEdf0NSJoWEVtcMeCb8szMLJOThZmZZXKyMDOzTE4WZmaWycnCzMwyOVmYmVmmoiULSd0lvSxprqTZki5Py0dIWiJpRvo6NWedayUtkDRf0sk55X0kzUyX3SOP8GVmVlLFvIN7A/BvEfG2pN2AaZJeSpf9OCI2GwNY0oHAQOAgYC/gt5L+LiI2AvcCFwNvAs8D/YGJRYzdzMxyFO3IIiKWRcTb6fRnwFygso5VzgSeiIi1EbEQWAAcLakr0C4i3kjvLhwDnFWsuM3MbEslOWchqSdwBPBWWnSZpHclPSSpQ1pWCSzKWW1xWlaZTtcuz7efiyVNlTR15cqVDdkEsx2fVNpXGRswYABDhw6tmW+IZ38PHTqUAQMGbGdkjafoAwlKaguMA66IiFWS7gVuASL9eScwDMj31xN1lG9ZGPEA8AAkw31sf/RmZjB+/HhatGhRUN3Jkydz4oknsnLlSjp27FhTPmrUqC2GTW9KiposJLUgSRSPR8R4gIhYnrP8F8CEdHYx0D1n9W7A0rS8W55yM7OtWrduHS1btmyQbe2xxx7bvY3dd9+9ASJpPMW8GkrAg8DciLgrpzz38VNnA7PS6WeBgZJaSdoH6A1MiYhlwGeS+qbbPB94plhxm1l56tevH5dccgmXX345HTp0oEOHDlx11VU1w5T37NmTESNGMGzYMNq3b8+gQYMAeP311znhhBOoqKigsrKS4cOHs2rVqprtfvHFFwwdOpS2bdvSpUuXmud71953bjfUunXruO666+jRowetWrWiV69e3HPPPXz44YeceOKJQPJMDUk13Vm1u6HWrl3LFVdcQZcuXdh1113p27cvr732Ws3yyZMnI4lJkyZxzDHHUFFRQVVVFW+//XZNnb/+9a8MHjyYzp07s+uuu9KrVy/uvvvu7X+z8yjmOYvjgMHA12pdJvvD9DLYd4ETge8DRMRsYCwwB3gBuDS9EgpgOPBLkpPeH+Arocyapccff5xNmzbxxhtvcP/99/PAAw9s9uF41113sf/++zN16lRuvfVWZs6cyUknncQZZ5zBO++8w/jx45kxYwbDhg2rWefKK6/kpZdeYty4cUyaNInp06fz6quv1hnHkCFDGDNmDHfddRdz587lwQcfpH379nTv3p1x48YBMHv2bJYtW8aoUaPybuMHP/gBTz75JA899BDTp0/nkEMOoX///ixbtmyzetdeey233XYbb7/9NnvuuSeDBg2q6c664YYbmDlzJhMmTGDevHk89NBDVFbWdR3Rdsg3bvmO8PLzLPw8C8tvq88yKNXf5Db+bZ5wwgnRu3fv2LRpU03ZLbfcEpWVlRER0aNHjxgwYMBm6wwePDiGDRu2Wdn06dMDiOXLl8dnn30WLVu2jMcee6xm+WeffRa77757DBkyZLN9X3rppRER8d577wUQEydOzBvnyy+/nPcZG0OGDInTTjstIiI+//zzaNGiRTzyyCM1yzds2BC9evWK66+/frPtvPDCCzV1XnvttQBi0aJFERFx+umnx9ChQ+t41/LbludZ+A5uM2sy+vbtS+49ucceeyxLliyp6Vaqqtr8MQzTpk3jscceo23btjWv4447DoAPPviADz74gHXr1nHsscfWrNO2bVsOOeSQrcYwffp0dtppp5rupm3xwQcfsH79+ppYAHbeeWeOPfZY5syZs1ndQw89tGZ6r732AmDFihUADB8+nLFjx3LYYYdx5ZVX8sorr2xzTFn8WFUz22G0adNms/lNmzZx0UUX8f3vf3+LupWVlcyfP7/e+4gGuKKpehv5BqOoXZZ7FVb1surzNKeccgp/+tOfmDhxIpMmTeK0007jnHPO4eGHH97uGGtzssjj9NOLv4/nir8Lsx3OW2+9lTziM/3QfPPNN9lrr71o165d3vpHHnkks2fPZr/99su7fL/99qNFixa8+eab9OrVC4DVq1cza9Ys9t13361uc9OmTbz88sv0799/i+XVV2Bt3Lhxi2W5+23ZsiWvvfZazX43btzIG2+8wTe/+c2trpdPx44dGTx4MIMHD+aUU07hvPPO47777qNVq1b12k4Wd0OZWZOxdOlSrrjiCubPn89TTz3Fj370o7xHDdWuvvpqpkyZwiWXXML06dNZsGABEyZM4Nvf/jaQdDldeOGFXH311bz00kvMnj2bYcOG1flB37t3b84991wuuugixo0bx8KFC/n973/Po48+CkCPHj2QxK9//WtWrlzJ559/vsU22rRpw/Dhw7nmmmt4/vnnmTt3LsOHD2f58uV85zvfKfj9uPHGG3n66ad5//33mTt3LuPHj6dXr14NnijARxZmVq0J3DA2aNAgNm7cyDHHHIMkLrzwwjqTxaGHHsqrr77KDTfcwAknnMDGjRvp1asXZ599dk2dO+64g9WrV3P22WdTUVHBd7/7XVavXl1nHGPGjOHf//3f+d73vsfHH39Mt27dauKorKzk5ptv5vrrr+eiiy7i/PPPZ/To0Vts4/bbbwfgggsu4NNPP+WII47ghRdeoGvXrlvU3ZpWrVpx/fXXs3DhwprLb597rjj9FmqI/rdyVFVVFVOnTt2mdUvSDTWhRMMd7KC/X9t2c+fO5YADDmjsMOqtX79+HHzwwfz0pz9t7FCavLr+BiRNi4iq2uXuhjIzs0xOFmZmlsnnLMysSZg8eXJjh9Cs+cjCzMwyOVmYNUM76oUtlm1bf/dOFmbNTIsWLVizZk1jh2GNZM2aNQU/myOXk4VZM9O5c2eWLFnCF1984SOMZiQi+OKLL1iyZAmdO3eu9/o+wW3WzFQPjbF06VLWr1/fyNFYKbVo0YIuXbpsdXiUujhZmDVD7dq126YPDGu+3A1lZmaZnCzMzCyTk4WZmWVysjAzs0xOFmZmlsnJwszMMjlZmJlZJicLMzPL5GRhZmaZnCzMzCyTk4WZmWVysjAzs0xFSxaSukt6WdJcSbMlXZ6W7yHpJUnvpz875KxzraQFkuZLOjmnvI+kmemyeySpWHGbmdmWinlksQH4t4g4AOgLXCrpQOAaYFJE9AYmpfOkywYCBwH9gZ9L2jnd1r3AxUDv9NW/iHGbmVktRUsWEbEsIt5Opz8D5gKVwJnAI2m1R4Cz0ukzgSciYm1ELAQWAEdL6gq0i4g3InlSy5icdczMrARKcs5CUk/gCOAtoEtELIMkoQDVj2yqBBblrLY4LatMp2uX59vPxZKmSpq6cuXKBm2DmVlzVvRkIaktMA64IiJW1VU1T1nUUb5lYcQDEVEVEVWdOnWqf7BmZpZXUZOFpBYkieLxiBifFi9Pu5ZIf65IyxcD3XNW7wYsTcu75Sk3M7MSKebVUAIeBOZGxF05i54FhqTTQ4BncsoHSmolaR+SE9lT0q6qzyT1Tbd5fs46ZmZWAsV8BvdxwGBgpqQZadl1wG3AWEkXAh8B5wBExGxJY4E5JFdSXRoRG9P1hgOjgdbAxPRlZmYlUrRkERGvkf98A8DXt7LOSGBknvKpwMENF52ZmdWH7+A2M7NMThZmZpbJycLMzDI5WZiZWSYnCzMzy+RkYWZmmZwszMwsk5OFmZllcrIwM7NMBSULSb572sysGSv0yOI+SVMkfUdS+2IGZGZm5aegZBERXwEGkQwhPlXSf0j6h6JGZmZmZaPgcxYR8T5wA3A1cAJwj6R5kv6xWMGZmVl5KPScxaGSfkzyHO2vAadHxAHp9I+LGJ+ZmZWBQoco/ynwC+C6iFhTXRgRSyXdUJTIzMysbBSaLE4F1lQ/jEjSTsCuEfFFRDxatOjMzKwsFHrO4rckT6mrVpGWmZlZM1Bostg1Ij6vnkmnK4oTkpmZlZtCk8VqSUdWz0jqA6ypo76Zme1ACj1ncQXwK0lL0/muwDeKEpGZmZWdgpJFRPxB0v7AlwEB8yJifVEjMzOzslHokQXAUUDPdJ0jJBERY4oSlZmZlZWCkoWkR4F9gRnAxrQ4ACcLM7NmoNAjiyrgwIiIYgZjZmblqdBkMQv4ErCsiLGYmTWa009v7AgaxnPPFWe7hSaLjsAcSVOAtdWFEXFGUaIyM7OyUmiyGFHMIMzMrLwVeunsK5J6AL0j4reSKoCdixuamZmVi0KHKP8X4Cng/rSoEng6Y52HJK2QNCunbISkJZJmpK9Tc5ZdK2mBpPmSTs4p7yNpZrrsHkmqR/vMzKwBFDrcx6XAccAqqHkQUueMdUYD/fOU/zgiDk9fzwNIOhAYCByUrvNzSdVHLvcCFwO901e+bZqZWREVmizWRsS66hlJu5DcZ7FVEfEq8EmB2z8TeCIi1kbEQmABcLSkrkC7iHgjvWx3DHBWgds0M7MGUmiyeEXSdUDr9NnbvwK29QKtyyS9m3ZTdUjLKoFFOXUWp2WV6XTt8rwkXSxpqqSpK1eu3MbwzMystkKTxTXASmAm8G3geZLncdfXvSR3gh9Ocs/GnWl5vvMQUUd5XhHxQERURURVp06dtiE8MzPLp9CroTaRPFb1F9uzs4hYXj0t6RfAhHR2MdA9p2o3YGla3i1PuZmZlVChV0MtlPTH2q/67iw9B1HtbJI7wwGeBQZKaiVpH5IT2VMiYhnwmaS+6VVQ5wPP1He/Zma2feozNlS1XYFzgD3qWkHSfwL9gI6SFgM3Af0kHU7SlfQhSZcWETFb0lhgDrABuLT6ed/AcJIrq1oDE9OXmZmVUKHdUH+pVXS3pNeAG+tY57w8xQ/WUX8kMDJP+VTg4ELiNDOz4ih0iPIjc2Z3IjnS2K0oEZmZWdkptBvqzpzpDSRdSOc2eDRmZlaWCu2GOrHYgZiZWfkqtBvqX+taHhF3NUw4ZmZWjupzNdRRJJe4ApwOvMrmd12bmdkOqj4PPzoyIj6DZPRY4FcRcVGxAjMzs/JR6HAfewPrcubXAT0bPBozMytLhR5ZPApMkfRfJDfUnU0yAqyZmTUDhV4NNVLSROCradEFETG9eGGZmVk5KbQbCqACWBURo4DF6RhOZmbWDBQ6kOBNwNXAtWlRC+CxYgVlZmblpdAji7OBM4DVABGxFA/3YWbWbBSaLNaljzUNAEltiheSmZmVm0KTxVhJ9wPtJf0L8Fu280FIZmbWdGReDZU+dOhJYH9gFfBl4MaIeKnIsZmZWZnITBYREZKejog+gBOEmVkzVGg31JuSjipqJGZmVrYKvYP7ROASSR+SXBElkoOOQ4sVmJmZlY86k4WkvSPiI+CUEsVjZmZlKOvI4mmS0Wb/JGlcRPxTCWIyMyu55yao6Ps4fUAUfR/FknXOIvfd61XMQMzMrHxlJYvYyrSZmTUjWd1Qh0laRXKE0Tqdhv9/grtdUaMzM7OyUGeyiIidSxWImZmVr/oMUW5mZs2Uk4WZmWVysjAzs0xFSxaSHpK0QtKsnLI9JL0k6f30Z4ecZddKWiBpvqSTc8r7SJqZLrsnHdjQzMxKqJhHFqOB/rXKrgEmRURvYFI6j6QDgYHAQek6P5dUfXL9XuBioHf6qr1NMzMrsqIli4h4FfikVvGZwCPp9CPAWTnlT0TE2ohYCCwAjpbUFWgXEW+kD18ak7OOmZmVSKnPWXSJiGUA6c/OaXklsCin3uK0rDKdrl1uZmYlVC4nuPOdh4g6yvNvRLpY0lRJU1euXNlgwZmZNXelThbL064l0p8r0vLFQPecet2ApWl5tzzleUXEAxFRFRFVnTp1atDAzcyas1Ini2eBIen0EOCZnPKBklpJ2ofkRPaUtKvqM0l906ugzs9Zx8zMSqTQhx/Vm6T/BPoBHSUtBm4CbgPGSroQ+Ag4ByAiZksaC8wBNgCXRsTGdFPDSa6sag1MTF9mZlZCRUsWEXHeVhZ9fSv1RwIj85RPBQ5uwNDMzKyeyuUEt5mZlTEnCzMzy+RkYWZmmZwszMwsk5OFmZllcrIwM7NMThZmZpbJycLMzDI5WZiZWSYnCzMzy+RkYWZmmZwszMwsk5OFmZllcrIwM7NMThZmZpbJycLMzDI5WZiZWSYnCzMzy+RkYWZmmZwszMwsk5OFmZllcrIwM7NMThZmZpbJycLMzDI5WZiZWSYnCzMzy+RkYWZmmZwszMwsU6MkC0kfSpopaYakqWnZHpJekvR++rNDTv1rJS2QNF/SyY0Rs5lZc9aYRxYnRsThEVGVzl8DTIqI3sCkdB5JBwIDgYOA/sDPJe3cGAGbmTVX5dQNdSbwSDr9CHBWTvkTEbE2IhYCC4CjSx+emVnz1VjJIoDfSJom6eK0rEtELANIf3ZOyyuBRTnrLk7LtiDpYklTJU1duXJlkUI3M2t+dmmk/R4XEUsldQZekjSvjrrKUxb5KkbEA8ADAFVVVXnrmJlZ/TXKkUVELE1/rgD+i6RbabmkrgDpzxVp9cVA95zVuwFLSxetmZmVPFlIaiNpt+pp4CRgFvAsMCStNgR4Jp1+FhgoqZWkfYDewJTSRm1m1rw1RjdUF+C/JFXv/z8i4gVJfwDGSroQ+Ag4ByAiZksaC8wBNgCXRsTGRojbzKzZKnmyiIg/AoflKf8L8PWtrDMSGFnk0MzMbCvK6dJZMzMrU04WZmaWycnCzMwyOVmYmVkmJwszM8vkZGFmZpmcLMzMLJOThZmZZXKyMDOzTE4WZmaWycnCzMwyOVmYmVkmJwszM8vkZGFmZpmcLMzMLJOThZmZZXKyMDOzTE4WZmaWycnCzMwyOVmYmVkmJwszM8vkZGFmZpmcLMzMLJOThZmZZXKyMDOzTE4WZmaWycnCzMwyOVmYmVmmJpMsJPWXNF/SAknXNHY8ZmbNSZNIFpJ2Bn4GnAIcCJwn6cDGjcrMrPloEskCOBpYEBF/jIh1wBPAmY0ck5lZs7FLYwdQoEpgUc78YuCY2pUkXQxcnM5+Lml+CWLbJoKOwMfF35GKvgtK1ZbS2FHasqO0A3aktkxQ0dvSAP/yPfIVNpVkka/5sUVBxAPAA8UPZ/tJmhoRVY0dR0NwW8rPjtIOcFvKRVPphloMdM+Z7wYsbaRYzMyanaaSLP4A9Ja0j6SWwEDg2UaOycys2WgS3VARsUHSZcCLwM7AQxExu5HD2l5NorusQG5L+dlR2gFuS1lQxBZd/2ZmZptpKt1QZmbWiJwszMwsk5NFkWUNU6LEPenydyUd2RhxFqKAtgxK2/CupNclHdYYcWYpdOgYSUdJ2ijpn0sZX30U0hZJ/STNkDRb0iuljrFQBfx97S7pOUnvpG25oDHizCLpIUkrJM3ayvIm8z+/mYjwq0gvkpPxHwC9gJbAO8CBteqcCkwkuZekL/BWY8e9HW35X0CHdPqUcmxLIe3Iqfc74Hngnxs77u34nbQH5gB7p/OdGzvu7WjLdcDt6XQn4BOgZWPHnqctxwNHArO2srxJ/M/XfvnIorgKGabkTGBMJN4E2kvqWupAC5DZloh4PSL+J519k+R+mHJT6NAx3wXGAStKGVw9FdKWbwLjI+IjgIgo1/YU0pYAdpMkoC1JsthQ2jCzRcSrJLFtTVP5n9+Mk0Vx5RumpHIb6pSD+sZ5Icm3p3KT2Q5JlcDZwH0ljGtbFPI7+Tugg6TJkqZJOr9k0dVPIW35KXAAyQ25M4HLI2JTacJrUE3lf34zTeI+iyaskGFKChrKpAwUHKekE0mSxVeKGtG2KaQddwNXR8RGlWZsrW1VSFt2AfoAXwdaA29IejMi3it2cPVUSFtOBmYAXwP2BV6S9PuIWFXk2BpaU/mf34yTRXEVMkxJUxnKpKA4JR0K/BI4JSL+UqLY6qOQdlQBT6SJoiNwqqQNEfF0SSIsXKF/Xx9HxGpgtaRXgcOAcksWhbTlAuC2SDr+F0haCOwPTClNiA2mqfzPb8bdUMVVyDAlzwLnp1dI9AX+GhHLSh1oATLbImlvYDwwuAy/uVbLbEdE7BMRPSOiJ/AU8J0yTBRQ2N/XM8BXJe0iqYJktOa5JY6zEIW05SOSIyQkdQG+DPyxpFE2jKbyP78ZH1kUUWxlmBJJl6TL7yO52uZUYAHwBcm3p7JTYFtuBPYEfp5+K98QZTbCZoHtaBIKaUtEzJX0AvAusAn4ZUTkvaSzMRX4e7kFGC1pJklXztURUXZDl0v6T6Af0FHSYuAmoAU0rf/52jzch5mZZXI3lJmZZXKyMDOzTE4WZmaWycnCzMwyOVmYmVkmJwuz7STp83rUHSHpymJt36xYnCzMzCyTk4VZEUg6XdJbkqZL+m16x3G1wyT9TtL7kv4lZ52rJP0hfcbBzY0QttlWOVmYFcdrQN+IOIJkuO0f5Cw7FDgNOBa4UdJekk4CepMM1X040EfS8aUN2WzrPNyHWXF0A55Mn1PQEliYs+yZiFgDrJH0MkmC+ApwEjA9rdOWJHm8WrqQzbbOycKsOH4C3BURz0rqB4zIWVZ7jJ0gGevo/0bE/SWJzqye3A1lVhy7A0vS6SG1lp0paVdJe5IMOPcHkgH0hklqC8kDmCR1LlWwZll8ZGG2/SrS0UWr3UVyJPErSUtIHjG7T87yKcCvgb2BWyJiKbBU0gEkDycC+Bz4FuX9WFdrRjzqrJmZZXI3lJmZZXKyMDOzTE4WZmaWycnCzMwyOVmYmVkmJwszM8vkZGFmZpn+H7SqW9rBqKIcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_distribution(y_train, train_preds, voting='Hard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "bb4a3d6f-ac52-4318-b879-537c83b9896a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkk0lEQVR4nO3deZwU9Z3/8ddb5BoQQTmCI4IEkohnYDz45RBj1hNFs6shS5TLNTFq4u5PAx5r8Jfg6ibrijl+CRsNeOwagsQrHiFENG5UAoJRroDByBUYdQ1yhPPz+6OK+jVDz0zPMN09A+/n49GPrvrWt6o+3+6Z/nR9q/pbigjMzMwADip3AGZm1nw4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFKzZkDRK0osl3N8USd+qY3lI6leqeIpN0hBJq3LmF0oa0ojtfErS0qaMzZoPJwUriKS3JH22RllJP8TzxNRG0q2SlkraJGm1pKclnVXk/f5I0v15yk+QtFXSYfuw7QmStkvaKOl9Sb+VNHjfIs4vIo6NiNkFxLRHcoyI30TER4sRk5Wfk4KVhaSDm2Az04FhwOVAF+BoYBJwfhNsuy5TgM9J6lCj/HLgyYh4bx+3/9OI6Ah0A14EZkhSzUqSWu3jfsz24qRgTUbSeElvSvpA0iJJF+csGyXpvyX9u6T3gAmSDpf0uKQNkuYAH27Avj4L/A0wLCJeiYht6eOZiPhaTr1jJM1Ov3UvlHRhHdu8QdJaSWskjamtXkS8BKwG/jZn3VbA3wNT0/lTJM1N27ZO0l2Fti1nP9vT7X0IODzt7vq/kp6StAk4Q9IRkh6RVC1phaSv5sTUPl3nfyQtAk6u0d7s6E9SK0k35bx/8yT1kvRCWv219Ojl83m6oWp9jdP9f1/SL9LtviLpw+kypX8P6yX9RdLvJR3X0NfJmpaTgjWlN4FPAYcCtwEPSuqZs/xU4I9Ad2Ai8H3gr0BPYEz6yEh6UtL4Wvb1WeCViFhVy3IktQaeAH6Z7vNa4CFJe3V9SDoHuJ4k0fRPt1+X+0mODHLjaQ08nc5PAiZFRCeSZDetnu3li78tMApYFRHvpMV/T/LaHQL8lqR9rwGVwJnAdZLOTut+I933h4GzgZF17O6fgC8A5wGdSN6LzRHx6XT5iRHRMSJ+WiPGQl7jL5D8PXQBlqfxA5wFfBr4CNAZ+Dzwbj0vixWZk4I1xKPpt8H3Jb0P/CB3YUT8LCLWRMSu9MNjGXBKTpU1EfHdiNgBbCP5pn1rRGyKiDdIv2XnbG9oRNxRSyxdgT/vnpF0WBrXXyT9NS0+DegI3JEeRfwaeJLkQ6qmS4GfRMQbEbEJmFDPa/EAcLqkI9P5y4H/TL/dA2wH+knqGhEbI+Llera3Ryzp67sSGARclLPssYj474jYBRwPdIuI/5O274/AfwDDc9o0MSLei4iVwD117PMK4JaIWBqJ1yKikA/oQl7jGRExJ33fHwJOSsu3kyS3jwGKiMURsbaAfVoROSlYQ1wUEZ13P4Cv5C6UdLmkBTlJ4ziSD+/dVuZMdwMOrlH2pwbE8i7JEQYA6QdfZ5IP0bZp8RHAyvQDNHcflXm2d0RDYomIt4EXgC9K6kjywZ2b1MaSfANeIul3koYW0KbdpqWvcfeI+ExEzMtZlhtjb+CIGon6JqBHI9rUi+RIr6EKeY3/nDO9mSSJkCaQ75EcMa6TNFlSp0bEYE3IScGahKTeJN9SrwEOTz+g3wByT5DmDslbDewg+TDa7agG7HIWcHLON/V81gC9JOX+nR9Fcj6gprWNiGUqyRHC3wIrIuLV3QsiYllEfIGkS+VOYHqeE9ONkfsarkz32znncUhEnJcub0ibVtKAczo5GvIa7yUi7omIQcCxJEn0hkbEYE3IScGaSgeSD6xqAEmjSY4U8oqIncAMkhPOFZIGUHefd831fwk8R9KldaqSy1Nbk3Rn7PYKsAn4uqTWSq7JvwB4OM8mpwGjJA2QVEHSH1+fR0g+dG+jRteXpC9K6pZ+g34/Ld5ZaPsKNAfYIGlcelK5laTjJO0+oTwNuFFSlzR5XlvHtn4MfFNS//QE8AmSDk+XrQP61rJeQ17jPUg6OX3vWqfb+CtN/xpZAzkpWJOIiEXAvwEvkXyIHA/8dz2rXUPSlfBnkss8f5K7UMlvDm6qY/3PkfRfP0jywbsCGAGck8a0DbgQOBd4h+QcyOURsSRP/E8DdwO/JjkZ+ut6Yic997A7MTxUY/E5wEJJG0lOOg+PiL+m7doo6VP1bb+A/e8k+QA+iaTt75B8uB+aVrmNpCtnBcmJ4Afq2NxdJEnkl8AG4F6gfbpsAjA17aK6tEYMBb/GeXQiObr8nzTOd4HvFLCeFZF8kx0zM9vNRwpmZpZxUjAzs4yTgpmZZZwUzMws0xSDkpVN165do0+fPuUOw8ysRZk3b947EdEt37IWnRT69OnD3Llzyx2GmVmLIqnWX7e7+8jMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzToq8+MrO6bdiwgfXr17N9+/b6K9t+oXXr1nTv3p1OnRp3awonBbP91IYNG1i3bh2VlZW0b98eSfWvZC1aRLBlyxZWr05uZ9GYxODuI7P91Pr166msrKSiosIJ4QAhiYqKCiorK1m/fn2jtuGkYLaf2r59O+3bt6+/ou132rdv3+guQ3cfFVspvqH5nhhWCx8hHJj25X0v2pGCpI+mN3Hf/dgg6TpJh0maKWlZ+twlZ50bJS2XtFTS2cWKzczM8itaUoiIpRFxUkScBAwCNgM/B8YDsyKiP8nN18cDpPfoHU5yA+9zgB9IalWs+MzMbG+l6j46E3gzIv4kaRgwJC2fCswGxgHDgIcjYiuwQtJy4BSSe/6aWRO54ILS7u+JJ0q7P9s3pTrRPBz4r3S6R0SsBUifu6fllcDKnHVWpWV7kHSlpLmS5lZXVxcxZDMrhyFDhnDNNdeUO4wDVtGTgqQ2wIXAz+qrmqdsrzOoETE5Iqoioqpbt7zDgZvZfs4/xiueUhwpnAu8GhHr0vl1knoCpM+7L6ZdBfTKWe9IYE0J4jOzZmLUqFE8//zzfP/730cSkpgyZQqSeOqppzjllFNo06YNzz77LBMmTOC4447bY/0pU6bQsWPHPcqeeOIJBg0aRLt27Tj66KO5+eab2bZtWymb1aKUIil8gf/fdQTwODAynR4JPJZTPlxSW0lHA/2BOSWIz8yaiUmTJjF48GBGjx7N2rVrWbt2Lb16Jd8Vx40bx7e+9S2WLFnCqaeeWtD2nn32WUaMGME111zDwoULue+++5g+fTo33XRTMZvRohX1RLOkCuBvgC/lFN8BTJM0FngbuAQgIhZKmgYsAnYAV0fEzmLGZ2bNy6GHHkqbNm2oqKjgQx/6EABLliwBYMKECZx11lkN2t7EiRO54YYbGD16NAAf/vCHufPOO/niF7/It7/9bf+OI4+iJoWI2AwcXqPsXZKrkfLVnwhMLGZMZtYyVVVVNXidefPmMWfOHO68886sbNeuXWzZsoU///nP9OzZsylD3C/4F81m1iJ06NBhj/mDDjqIqPFr/ponoHft2sU3vvENLrnkkr225wtV8nNSMLNmpU2bNuzcWX/Pcbdu3Vi3bh0RkXUDLViwYI86AwcOZMmSJfTr168Yoe6XnBTMrFnp06cPc+bM4a233qJjx47s2rUrb70hQ4bw3nvvcfvttzN8+HBmz57N9OnT96hz6623MnToUHr37s2ll17KwQcfzBtvvMGcOXP413/911I0p+WJiBb7GDRoUDR7yXB1xX2Y5bFo0aJyh9AoS5cujdNOOy3at28fQPzkJz8JIKqrq/eq+8Mf/jCOOuqoqKioiM9//vNx9913R4cOHfao8+yzz8YnP/nJaN++fRxyyCExaNCg+O53v1uq5pRNXe8/MDdq+VxVtOARNquqqmLu3LnlDqNuHiXVymTx4sUcc8wx5Q7DyqSu91/SvIjIe+be91MwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOC2YFGKu2jGRs6dCijRo3K5pvi/tCjRo1i6NCh+xhZ+XhAPDOz1IwZM2jdunVBdWfPns0ZZ5xBdXU1Xbt2zconTZq015DeLYmTgpm1aNu2baNNmzZNsq3DDjtsn7dx6KGHNkEk5ePuIzNrVoYMGcKXv/xlvva1r9GlSxe6dOnCDTfckA2h3adPHyZMmMCYMWPo3LkzI0aMAOC3v/0tp59+OhUVFVRWVnLVVVexYcOGbLubN29m1KhRdOzYkR49enD77bfn3Xdu99G2bdu46aab6N27N23btqVv377cc889vPXWW5xxxhlAcl8HSVk3VM3uo61bt3LdddfRo0cP2rVrx2mnncaLL76YLZ89ezaSmDVrFqeeeioVFRVUVVXx6quvZnX+8pe/cNlll9G9e3fatWtH3759ufvuu/f9xc7DScHMmp2HHnqIXbt28dJLL/GjH/2IyZMn7/EheNddd/Gxj32MuXPncvvtt/P6669z1llnceGFF/Laa68xY8YMFixYwJgxY7J1rr/+embOnMkjjzzCrFmzmD9/Pi+88EKdcYwcOZL777+fu+66i8WLF3PvvffSuXNnevXqxSOPPALAwoULWbt2LZMmTcq7ja9//ev89Kc/5b777mP+/Pkcf/zxnHPOOaxdu3aPejfeeCN33HEHr776KocffjgjRozIuqFuueUWXn/9dZ588kmWLFnCfffdR2VlZWNe2vrVNqZ2S3j4fgq+n4LVrtbx9EvxN7kPf5+nn3569O/fP3bt2pWVffOb34zKysqIiOjdu3cMHTp0j3Uuu+yyGDNmzB5l8+fPDyDWrVsXH3zwQbRp0yYefPDBbPkHH3wQhx56aIwcOXKPfV999dUREfGHP/whgHj66afzxvncc8/lvc/DyJEj4/zzz4+IiI0bN0br1q1j6tSp2fIdO3ZE37594+abb95jO88880xW58UXXwwgVq5cGRERF1xwQYwaNaqOV21vjb2fQlGPFCR1ljRd0hJJiyUNlnSYpJmSlqXPXXLq3yhpuaSlks4uZmxm1nyddtpp2S02AQYPHszq1auz7qCqqj1vBTBv3jwefPBBOnbsmD0+8YlPAPDmm2/y5ptvsm3bNgYPHpyt07FjR44//vhaY5g/fz4HHXRQ1k3UGG+++Sbbt2/PYgFo1aoVgwcPZtGiRXvUPeGEE7LpI444AoD169cDcNVVVzFt2jROPPFErr/+ep5//vlGx1SfYncfTQKeiYiPAScCi4HxwKyI6A/MSueRNAAYDhwLnAP8QFKrIsdnZi1Qhw4d9pjftWsXV1xxBQsWLMger732GsuWLeOkk05q1NVAjVmntm0oz6W5Nctyr3ravWz3eZRzzz2XP/3pT1x//fW88847nH/++YwePXqf48unaElBUifg08C9ABGxLSLeB4YBU9NqU4GL0ulhwMMRsTUiVgDLgVOKFZ+ZNV+vvPLKHh/KL7/8MkcccQSdOnXKW3/gwIEsXLiQfv367fVo3749/fr1o3Xr1rz88svZOps2beKNN96oNYaBAweya9cunnvuubzLd1/xtHPnzlq30a9fP9q0abPHieWdO3fy0ksvMWDAgFrXy6dr165cdtllTJkyhXvvvZepU6eydevWBm2jEMU8UugLVAM/kTRf0o8ldQB6RMRagPS5e1q/EliZs/6qtGwPkq6UNFfS3Orq6iKGb2blsmbNGq677jqWLl3K9OnT+fa3v80//uM/1lp/3LhxzJkzhy9/+cvMnz+f5cuX8+STT/KlL30JSLqKxo4dy7hx45g5cyYLFy5kzJgxdX6g9+/fn0svvZQrrriCRx55hBUrVvCb3/yGBx54AIDevXsjiV/84hdUV1ezcePGvbbRoUMHrrrqKsaPH89TTz3F4sWLueqqq1i3bh1f+cpXCn49br31Vh599FGWLVvG4sWLmTFjBn379qVt27YFb6NQxfydwsHAQODaiHhF0iTSrqJa5Pvp417HbxExGZgMyT2amyJQswNKC/hh1YgRI9i5cyennnoqkhg7dmydSeGEE07ghRde4JZbbuH0009n586d9O3bl4svvjir853vfIdNmzZx8cUXU1FRwbXXXsumTZvqjOP+++/nn//5n/nqV7/KO++8w5FHHpnFUVlZyW233cbNN9/MFVdcweWXX86UKVP22sadd94JwOjRo3n//ff5+Mc/zjPPPEPPnj0Lfj3atm3LzTffzIoVK7LLWp944omC128INUW/Wd4NSx8CXo6IPun8p0iSQj9gSESsldQTmB0RH5V0I0BE/Eta/1lgQkS8VNs+qqqqYu7cuUWJv8mU4mf+LeCf3Eqvrhu3N2dDhgzhuOOO43vf+165Q2nR6nr/Jc2LiKp8y4rWfRQRfwZWSvpoWnQmsAh4HBiZlo0EHkunHweGS2or6WigPzCnWPGZmdneij3MxbXAQ5LaAH8ERpMkommSxgJvA5cARMRCSdNIEscO4OqIqL3Dz8zMmlxRk0JELADyHaKcWUv9icDEYsZkZs3b7Nmzyx3CAe2AHhDvgguKv4/inAoyMysOj31kth8r1oUk1rzty/vupGC2n2rdujVbtmwpdxhWBlu2bCn4vhA1OSmY7ae6d+/O6tWr2bx5s48YDhARwebNm1m9ejXdu3evf4U8DuhzCmb7s91DQqxZs4bt27eXORorldatW9OjR49ahwSpj5OC2X6sU6dOjf5wsAOTu4/MzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWKWpSkPSWpNclLZA0Ny07TNJMScvS5y459W+UtFzSUklnFzM2MzPbWymOFM6IiJMioiqdHw/Mioj+wKx0HkkDgOHAscA5wA8ktSpBfGZmlipH99EwYGo6PRW4KKf84YjYGhErgOXAKaUPz8zswFXspBDALyXNk3RlWtYjItYCpM+7byRaCazMWXdVWrYHSVdKmitpbnV1dRFDNzM78BT7dpyfiIg1kroDMyUtqaOu8pTtdbfxiJgMTAaoqqry3cjNzJpQUY8UImJN+rwe+DlJd9A6ST0B0uf1afVVQK+c1Y8E1hQzPjMz21PRkoKkDpIO2T0NnAW8ATwOjEyrjQQeS6cfB4ZLaivpaKA/MKdY8ZmZ2d6K2X3UA/i5pN37+c+IeEbS74BpksYCbwOXAETEQknTgEXADuDqiNhZxPjMzKyGoiWFiPgjcGKe8neBM2tZZyIwsVgxmZlZ3fyLZjMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMwsU1BSkHRcsQMxM7PyK/RI4YeS5kj6iqTOxQzIzMzKp6CkEBGfBEaQDFg3V9J/SvqbokZmZmYlV/A5hYhYBtwCjANOB+6RtETS54oVnJmZlVah5xROkPTvwGLgM8AFEXFMOv3vRYzPzMxKqNAB8b4H/AdwU0Rs2V2Y3kDnlqJEZmZmJVdoUjgP2LJ7KGtJBwHtImJzRDxQtOjMzKykCj2n8Cugfc58RVpmZmb7kUKTQruI2Lh7Jp2uKE5IZmZWLoUmhU2SBu6ekTQI2FJHfTMza4EKPadwHfAzSWvS+Z7A54sSkZmZlU1BSSEififpY8BHAQFLImJ7USMzM2tiF1xQ7giazhNPFGe7DblH88lAn3Sdj0siIu4vSlRmZlYWhf547QHgO8AnSZLDyUBVgeu2kjRf0pPp/GGSZkpalj53yal7o6TlkpZKOrvBrTEzs31S6JFCFTAgIqIR+/gayS+hO6Xz44FZEXGHpPHp/DhJA4DhwLHAEcCvJH1k928jzMys+Aq9+ugN4EMN3bikI4HzgR/nFA8DpqbTU4GLcsofjoitEbECWA6c0tB9mplZ4xV6pNAVWCRpDrB1d2FEXFjPencDXwcOySnrERFr0/XXSuqellcCL+fUW5WW7UHSlcCVAEcddVSB4ZuZWSEKTQoTGrphSUOB9RExT9KQQlbJU7ZXd1VETAYmA1RVVTWmO8vMzGpR6CWpz0vqDfSPiF9JqgBa1bPaJ4ALJZ0HtAM6SXoQWCepZ3qU0BNYn9ZfRXK/ht2OBNZgZmYlU+jVR/8ATAd+lBZVAo/WtU5E3BgRR0ZEH5ITyL+OiC8CjwMj02ojgcfS6ceB4ZLaSjoa6A/MKbwpZma2rwrtPrqa5KTvK5DccCfnXEBD3QFMkzQWeBu4JN3mQknTgEXADuBqX3lkZlZahSaFrRGxTUq6/SUdTJ7+/tpExGxgdjr9LnBmLfUmAhML3a6ZmTWtQi9JfV7STUD79N7MPwOK9CNrMzMrl0KTwnigGngd+BLwFMn9ms3MbD9S6NVHu0hux/kfxQ3HzMzKqaCkIGkF+X8z0LfJIzIzs7JpyNhHu7UjuWLosKYPx8zMyqmgcwoR8W7OY3VE3A18prihmZlZqRXafTQwZ/YgkiOHQ2qpbmZmLVSh3Uf/ljO9A3gLuLTJozEzs7Iq9OqjM4odiJmZlV+h3Uf/VNfyiLiracIxM7NyasjVRyeTDFoHcAHwArCyGEGZmVl5NOQmOwMj4gMASROAn0XEFcUKzMzMSq/QYS6OArblzG8D+jR5NGZmVlaFHik8AMyR9HOSXzZfDNxftKjMzKwsCr36aKKkp4FPpUWjI2J+8cIyM7NyKLT7CKAC2BARk4BV6d3RzMxsP1Lo7Ti/AYwDbkyLWgMPFisoMzMrj0KPFC4GLgQ2AUTEGjzMhZnZfqfQpLAtIoJ0+GxJHYoXkpmZlUuhSWGapB8BnSX9A/ArfMMdM7P9Tr1JQZKAnwLTgUeAjwK3RsR361mvnaQ5kl6TtFDSbWn5YZJmSlqWPnfJWedGScslLZV09j61zMzMGqzeS1IjIiQ9GhGDgJkN2PZW4DMRsVFSa+DF9LLWzwGzIuIOSeNJ7v88TtIAYDhwLHAE8CtJH4mInQ1tlJmZNU6h3UcvSzq5IRuOxMZ0tnX6CGAYMDUtnwpclE4PAx6OiK0RsQJYDpzSkH2amdm+KTQpnEGSGN6U9HtJr0v6fX0rSWolaQGwHpgZEa8APSJiLUD63D2tXsmeA+ytSstqbvNKSXMlza2uri4wfDMzK0Sd3UeSjoqIt4FzG7PxtOvnJEmdgZ9LOq6u3eXbRJ5tTgYmA1RVVe213MzMGq++I4VHASLiT8BdEfGn3EehO4mI94HZwDnAOkk9AdLn9Wm1VUCvnNWOBNYUug8zM9t39SWF3G/vfRuyYUnd0iMEJLUHPgssIbknw8i02kjgsXT6cWC4pLbpEBr9gTkN2aeZme2b+q4+ilqmC9ETmCqpFUnymRYRT0p6ieR3D2OBt4FLACJioaRpwCKS+0Bf7SuPzMxKq76kcKKkDSRHDO3TadL5iIhOta0YEb8HPp6n/F3gzFrWmQhMLCRwM7OGeuLJfKcum94FQ1vu6c46k0JEtCpVIGZmVn4NGTrbzMz2c04KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs0zRkoKkXpKek7RY0kJJX0vLD5M0U9Ky9LlLzjo3Slouaamks4sVm5mZ5VfMI4UdwP+OiGOA04CrJQ0AxgOzIqI/MCudJ102HDgWOAf4gaRWRYzPzMxqKFpSiIi1EfFqOv0BsBioBIYBU9NqU4GL0ulhwMMRsTUiVgDLgVOKFZ+Zme2tJOcUJPUBPg68AvSIiLWQJA6ge1qtEliZs9qqtKzmtq6UNFfS3Orq6qLGbWZ2oCl6UpDUEXgEuC4iNtRVNU9Z7FUQMTkiqiKiqlu3bk0VppmZUeSkIKk1SUJ4KCJmpMXrJPVMl/cE1qflq4BeOasfCawpZnxmZranYl59JOBeYHFE3JWz6HFgZDo9Engsp3y4pLaSjgb6A3OKFZ+Zme3t4CJu+xPAZcDrkhakZTcBdwDTJI0F3gYuAYiIhZKmAYtIrly6OiJ2FjE+MzOroWhJISJeJP95AoAza1lnIjCxWDGZmVnd/ItmMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWKVpSkHSfpPWS3sgpO0zSTEnL0ucuOctulLRc0lJJZxcrLjMzq10xjxSmAOfUKBsPzIqI/sCsdB5JA4DhwLHpOj+Q1KqIsZmZWR5FSwoR8QLwXo3iYcDUdHoqcFFO+cMRsTUiVgDLgVOKFZuZmeVX6nMKPSJiLUD63D0trwRW5tRblZbtRdKVkuZKmltdXV3UYM3MDjTN5USz8pRFvooRMTkiqiKiqlu3bkUOy8zswFLqpLBOUk+A9Hl9Wr4K6JVT70hgTYljMzM74JU6KTwOjEynRwKP5ZQPl9RW0tFAf2BOiWMzMzvgHVysDUv6L2AI0FXSKuAbwB3ANEljgbeBSwAiYqGkacAiYAdwdUTsLFZsZmaWX9GSQkR8oZZFZ9ZSfyIwsVjxmJlZ/ZrLiWYzM2sGnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLNLukIOkcSUslLZc0vtzxmJkdSJpVUpDUCvg+cC4wAPiCpAHljcrM7MDRrJICcAqwPCL+GBHbgIeBYWWOyczsgHFwuQOooRJYmTO/Cjg1t4KkK4Er09mNkpaWKLZGEXQF3inuTlTUzecofltKY39pB7gtzdOTKnpb9vHfvndtC5pbUsjXzNhjJmIyMLk04ew7SXMjoqrccTSF/aUt+0s7wG1prlpyW5pb99EqoFfO/JHAmjLFYmZ2wGluSeF3QH9JR0tqAwwHHi9zTGZmB4xm1X0UETskXQM8C7QC7ouIhWUOa1+1mK6uAuwvbdlf2gFuS3PVYtuiiKi/lpmZHRCaW/eRmZmVkZOCmZllnBSaSH3DcyhxT7r895IGliPO+hTQjhFp/L+X9FtJJ5YjzkIUOmSKpJMl7ZT0d6WMryEKaYukIZIWSFoo6flSx1iIAv6+DpX0hKTX0naMLkechZB0n6T1kt6oZXmL+J/fS0T4sY8PkpPibwJ9gTbAa8CAGnXOA54m+S3GacAr5Y67ke34X0CXdPrc5tiOQtuSU+/XwFPA35U77n14XzoDi4Cj0vnu5Y67ke24Cbgzne4GvAe0KXfstbTn08BA4I1aljf7//l8Dx8pNI1ChucYBtwfiZeBzpJ6ljrQetTbjoj4bUT8Tzr7MslvSZqjQodMuRZ4BFhfyuAaqJC2/D0wIyLeBoiI5tieQtoRwCGSBHQkSQo7ShtmYSLiBZL4atMS/uf34qTQNPINz1HZiDrl1tAYx5J8E2qO6m2LpErgYuCHJYyrMQp5Xz4CdJE0W9I8SZeXLLrCFdKO7wHHkPxo9XXgaxGxqzThNbmW8D+/l2b1O4UWrN7hOQqsU24FxyjpDJKk8MmiRtR4hbTlbmBcROxU6caPaoxC2nIwMAg4E2gPvCTp5Yj4Q7GDa4BC2nE2sAD4DPBhYKak30TEhiLHVgwt4X9+L04KTaOQ4TlawhAeBcUo6QTgx8C5EfFuiWJrqELaUgU8nCaErsB5knZExKMlibBwhf59vRMRm4BNkl4ATgSaU1IopB2jgTsi6ZRfLmkF8DFgTmlCbFIt4X9+L+4+ahqFDM/xOHB5ekXCacBfImJtqQOtR73tkHQUMAO4rJl9C62p3rZExNER0Sci+gDTga80w4QAhf19PQZ8StLBkipIRhdeXOI461NIO94mOdpBUg/go8AfSxpl02kJ//N78ZFCE4hahueQ9OV0+Q9Jrm45D1gObCb5RtSsFNiOW4HDgR+k37B3RDMcDbLAtrQIhbQlIhZLegb4PbAL+HFE5L1UslwKfE++CUyR9DpJ98u4iGiWw2lL+i9gCNBV0irgG0BraDn/8/l4mAszM8u4+8jMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBWAEkbG1B3gqTri7V9s2JyUjAzs4yTglkjSbpA0iuS5kv6VfoL3N1OlPRrScsk/UPOOjdI+l06vv5tZQjbrE5OCmaN9yJwWkR8nGQY6K/nLDsBOB8YDNwq6QhJZwH9SYaQPgkYJOnTpQ3ZrG4e5sKs8Y4EfpqOkd8GWJGz7LGI2AJskfQcSSL4JHAWMD+t05EkSbxQupDN6uakYNZ43wXuiojHJQ0BJuQsqzl+TJCM5fMvEfGjkkRn1gjuPjJrvEOB1en0yBrLhklqJ+lwkkHTfkcyENwYSR0hucmPpO6lCtasED5SMCtMRToS5m53kRwZ/EzSapJbkx6ds3wO8AvgKOCbEbEGWCPpGJIb4ABsBL5I874VqB1gPEqqmZll3H1kZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWX+H91K3uefjWr8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_distribution(y_test, test_preds, voting='Hard')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a6342d0-fd6a-4875-ae49-592b0d3e8b59",
   "metadata": {},
   "source": [
    "#### Soft Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "9d960774-fb9d-4865-ab85-b7c526db22a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = ensemble(voting=\"soft\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "1090263a-d9d6-4972-af45-a27158289526",
   "metadata": {},
   "outputs": [],
   "source": [
    "soft_train_preds = np.array([0 if i > j else 1 for i, j in np.mean(np.array([sc[0] for sc in scores]), axis=0)])\n",
    "soft_test_preds = np.array([0 if i > j else 1 for i, j in np.mean(np.array([sc[1] for sc in scores]), axis=0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "id": "d69dc71d-73b2-4602-a3a0-21b407482f2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy score (train)</th>\n",
       "      <th>accuracy score (test)</th>\n",
       "      <th>f1 score (train)</th>\n",
       "      <th>f1 score (test)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ensemble</th>\n",
       "      <td>0.787362</td>\n",
       "      <td>0.762525</td>\n",
       "      <td>0.692976</td>\n",
       "      <td>0.667602</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          accuracy score (train)  accuracy score (test)  f1 score (train)  \\\n",
       "ensemble                0.787362               0.762525          0.692976   \n",
       "\n",
       "          f1 score (test)  \n",
       "ensemble         0.667602  "
      ]
     },
     "execution_count": 267,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soft_df = show_ensemble_table(y_train, y_test, soft_train_preds, soft_test_preds)\n",
    "soft_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "2f922af4-08e7-4c88-91d5-ce0c0171f209",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of missclassified: 237\n"
     ]
    }
   ],
   "source": [
    "soft_miss_count = len(y_test[y_test!=soft_test_preds])\n",
    "print(f'# of missclassified: {soft_miss_count}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "2f1c38c8-0aa4-4e25-9673-e6f16e6215b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjjklEQVR4nO3deZhU9Z3v8fdHZWsWxbAIjYIIEzWOG63i1URM7nWFUcdoyDUKLjEQiZpJHLdEyePFCYkaNSYxOlFQJ1GijltcxuAWo0BAHGWRLbiwCG2MYZGwfu8f53RPdVPNqYau6mr683qeerrqd7bv6e1T53dO/Y4iAjMzs23ZpbkLMDOz8uewMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCysLks6Q9IGkNZIOa6YaQtKABqaNlPRqqWsqJkkTJP2/9PnnJc3bzvXcKen7TVudlRuHhTUZScdKek3S3yR9LOmPko4ocPGbgDER0Qn4a/qPe7dGbn+gpAclVUtaJWmBpJ9K6tPonSl8m+0lfSLpi3mm/UTSwzu4/nclrUtDdIWkeyV12pF15hMRf4iIzxZQz1ahGRGjIuKGpq7JyovDwpqEpC7AU8BPgT2BSuAHwPoCV9EXmL0D2x8ATAWWAYdFRBfgGGARcOz2rjdLRPwdeAg4r149uwJfBSY2wWaGpSF6OHAE8L36MzQ2WM0ay2FhTeUfACLiNxGxOSLWRcR/RcRbAJJ2kfQ9Se9JWinpPkm7S2onaQ2wK/DfkhYBr6Tr/CR9R310AdsfC/wxIv4lIpaktayMiFsj4sGamSR9XdLC9MjnCUm9861M0mfS6askTQP228a2JwJnSqrIaTuR5O/rmXR9V0paKmm1pHmSvlTAPtUREUvT9R2UrjMkXSJpAbAgbRsq6c30aOc1SQfn7NNhkt5Ia3gIaJ8zbYikJTmv95b0aHqU9hdJd0g6ALgTODr9uXySzlvbnZW+bvB7nNY8Kj3q+6ukn0lSOm2ApJfTI9OP0hqtTDgsrKnMBzZLmijpZEld600fmT6OB/oDnYA7ImJ9+q4Z4JCI2A/4Qvp6j4joFBGvS9on/Qe4TwPb/9/AI9sqMO0q+jfgbKAX8B7wYAOz/wz4ezrfBekjr4h4DVgO/HNO87nAryNik6TPAmOAIyKiM0mQvLutWhuof2/gFGBmTvPpwFHAgZIOB+4BvgF8Bvgl8EQayG2Bx4D7SY78fguc2cB2diU5SnwP6EdylPhgRMwFRgGvpz+XPfIsW8j3eCjJEdIh6Xwnpu03AP8FdAX6kBylWplwWFiTiIhVJN09AdwNVKfvKnums5wD3BIRf46INcDVwPBCu08i4v2I2CMi3m9glm7AhzUvJI1Jw2WNpLtzargnIt6IiPVpDUdL6pe7ovSf5ZnAdRGxNiJmkd2ddB9pV1TaJXdazjKbgXYk/9DbRMS7EbGokP1OPZa+i38VeBm4MWfav0XExxGxDvg68MuImJoe3U0k6QYcnD7aALdGxMaIeBj4UwPbOxLoDVyR7v/fI6LQk/uFfI9/GBGfpD/LF4FD0/aNJN2RvRu5TSsBh4U1mYiYGxEjI6IPSVdJb+DWdHJvkneZNd4DdgN60jT+QvJOtqaWO9J3vreS/JPcqoY0tP5C8s45V/e0tg/q1bst9wHHS6oEvgwsjIiZ6XYWApeTdJWtTE/C5+3+asDpaVD2jYhvpsFQI7fGvsB30pD8JA2YvUn2uzewNOqOHNrQPu0NvBcRmxpRY41Cvscf5jz/lOQoE+BfAQHTJM2W1ODRnJWew8KKIiLeASaQ9q+TnHjumzPLPsAmYEW+xbdjk5Op2w2UT50aJHUk6a5ZWm++6rS2vXPaGur+ApIjH+APJO+szyUJj9zpv46IY9PtBzA+o9ZC5X6vPgDGpcFS86iIiN+QdJNV1pwfSDW0Tx8A+zRw1Jf1syn0e7z1iiM+jIivR0Rvkq60n6uBS5mt9BwW1iQk7S/pO0ovU037178KTEln+Q3wbUn7Krn080bgoQbevVYDW0jObRRqLPB5Sbek7+6R1A04IGeeXwPnSzpUUru0hqkR8W7uiiJiM/AoMFZShaQDgREF1DCR5NzEMcB/1DRK+qykL6bb/DuwjqRrqqndDYySdJQSHSWdKqkz8DpJAF4qaTdJ/0zS3ZTPNJJw+WG6jvaSjkmnrQD6pOdA8inoe5yPpLP0P5c5/5UkmIrxfbLt4LCwprKa5ETrVElrSUJiFvCddPo9JCdXXwEWk/zT/Fa+FUXEp8A44I9pd8rg9AT3moZOcEfEfJJ++T4kV1WtBv5I8k73++k8k9Pnj5D8M9wPGN7A/owh6R75kOQI6d4CvgcPk5ycnRwRy3Pa2wE/BD5K19cDuAZA0jmStvuS4VwRMZ3kvMUdJP9sF5JcVEBEbCA58hqZTvsKSSDmW89mYBgwAHgfWJLOD/ACySXOH0r6KM+yjfke13cEye/PGuAJ4LKIWFzgslZk8s2PzMwsi48szMwsk8PCzMwyOSzMzCyTw8LMzDLttIOPdevWLfr169fcZZiZtSgzZsz4KCK612/facOiX79+TJ8+vbnLMDNrUSTl/WS/u6HMzCyTw8LMzDI5LMzMLJPDwszMMjkszMws0057NZSZNWzVqlWsXLmSjRs3NncpVkJt2rShR48edOnSpdHLOizMWplVq1axYsUKKisr6dChA3VvcWE7q4hg3bp1LF2a3FqksYHhbiizVmblypVUVlZSUVHhoGhFJFFRUUFlZSUrV65s9PIOC7NWZuPGjXTo0KG5y7Bm0qFDh+3qfnQ3VHMp1Ts636/E8vARReu1vT97H1mYmVkmh4WZmWVyN5SZATBsWGm39+STpd2e7RgfWZhZizFkyBDGjBnT3GW0Sg4LM9up+IOGxeGwMLMWYeTIkbz88sv87Gc/QxKSmDBhApJ4+umnOfLII2nbti3PPfccY8eO5aCDDqqz/IQJE+jUqVOdtieffJJBgwbRvn179t13X6699lo2bNhQyt1qMXzOwsxahNtuu4358+ez//77c+ONNwIwe/ZsAK688kpuvvlmBgwYQOfOnQu68dlzzz3HOeecw2233cYXvvAF3n//fUaNGsX69eu56aabirovLZHDwsxahN133522bdtSUVHBXnvtBcA777wDwNixYznhhBMatb5x48ZxxRVXcP755wOw3377MX78eL72ta/x4x//2J9FqcdhYWYtXlVVVaOXmTFjBtOmTWP8+PG1bVu2bGHdunV8+OGH9OrVqylLbPEcFmbW4nXs2LHO61122YWoN3pB/RPfW7Zs4frrr+ess87aan3du3dv+iJbOIeFmbUYbdu2ZfPmzZnzde/enRUrVhARtd1Jb775Zp15Dj/8cN555x0GDBhQjFJ3Og4LM2sx+vXrx7Rp03j33Xfp1KkTW7ZsyTvfkCFD+Pjjj7nxxhsZPnw4L730Eg8//HCdea677jqGDh1K3759Ofvss9ltt92YNWsW06ZN40c/+lEpdqdliYid8jFo0KAoa8kQf8V/mNUzZ86c5i5hu82bNy8GDx4cHTp0CCDuvffeAKK6unqree+8887YZ599oqKiIr7yla/ErbfeGh07dqwzz3PPPRfHHntsdOjQITp37hyDBg2Kn/70p6XanWazrd8BYHrk+Z+q2ElHJa2qqopCLp9rNh511prJ3LlzOeCAA5q7DGtG2/odkDQjIra6YsAfyjMzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMRQsLSXtLelHSXEmzJV2Wto+VtFTSm+njlJxlrpa0UNI8SSfmtA+S9HY67XZ50BYzs5Iq5ofyNgHfiYg3JHUGZkh6Pp32k4ioM6yjpAOB4cDngN7A7yX9Q0RsBn4BXAxMAZ4GTgKeKWLtZmaWo2hHFhGxPCLeSJ+vBuYCldtY5DTgwYhYHxGLgYXAkZJ6AV0i4vX0AyP3AacXq24zM9taSc5ZSOoHHAZMTZvGSHpL0j2SuqZtlcAHOYstSdsq0+f12/Nt52JJ0yVNr66ubspdMDNr1YoeFpI6AY8Al0fEKpIupf2AQ4HlwM01s+ZZPLbRvnVjxF0RURURVR410qyRpNI+ytjQoUMZOXJk7eumuPf3yJEjGTp06A5W1nyKOpCgpDYkQfEfEfEoQESsyJl+N/BU+nIJsHfO4n2AZWl7nzztZmYl8eijj9KmTZuC5n3ppZc4/vjjqa6uplu3brXtt91221bDprckxbwaSsCvgLkRcUtOe+4dRc4AZqXPnwCGS2onaV9gIDAtIpYDqyUNTtd5HvB4seo2s51DU95Le88996Rz5847tI7dd9+dPfbYo2kKagbF7IY6BjgX+GK9y2R/lF4G+xZwPPBtgIiYDUwC5gDPApekV0IBjAb+neSk9yJ8JZRZqzNkyBBGjRrFZZddRteuXenatStXXHFF7TDl/fr1Y+zYsVxwwQXssccenHPOOQC89tprHHfccVRUVFBZWcno0aNZtWpV7Xo//fRTRo4cSadOnejZs2ft/b3rbzu3G2rDhg1cc8019O3bl3bt2tG/f39uv/123n33XY4//ngguaeGpNrurPrdUOvXr+fyyy+nZ8+etG/fnsGDB/Pqq6/WTn/ppZeQxOTJkznqqKOoqKigqqqKN954o3aev/3tb5x77rn06NGD9u3b079/f2699dYd/2bnk28o2p3h4SHKPUS55dfg8NSl+p3czt/N4447Ljp16hRjxoyJuXPnxkMPPRRdunSJm2++OSIi+vbtG507d47x48fHggULYv78+fHWW29Fx44d46abbor58+fHlClTYvDgwXHmmWfWrnf06NHRu3fvePbZZ+Ptt9+OL3/5y9G5c+cYMWJEnW1fcsklta+HDx8elZWV8fDDD8eiRYvihRdeiIkTJ8amTZvikUceCSBmz54dy5cvj08++SQiIkaMGBGnnnpq7TouvfTS2GuvveKpp56KOXPmxEUXXRQdO3aMZcuWRUTEiy++GEAcccQR8cILL8TcuXPjhBNOiP333z+2bNkSERFjxoyJQw45JKZOnRqLFy+OF198MSZNmpT5vdyeIcqb/Z96sR4OC4eF5deSw2LgwIG1/ygjIm644YaorKyMiCQshg4dWmeZc889Ny644II6bTNnzgwgVqxYEatXr462bdvGAw88UDt99erVsfvuuzcYFvPnzw8gnnnmmbx11vyTr3+PjdywWLNmTbRp0yYmTpxYO33Tpk3Rv3//uPbaa+us59lnn62d59VXXw0gPvjgg4iIGDZsWIwcOXIb37X8ticsPNyHmbUYgwcPJncAh6OPPpqlS5fWditVVdW9DcOMGTN44IEH6NSpU+3jmGOOAWDRokUsWrSIDRs2cPTRR9cu06lTJ/7xH/+xwRpmzpzJLrvsUtvdtD0WLVrExo0ba2sB2HXXXTn66KOZM2dOnXkPPvjg2ue9e/cGYOXKlQCMHj2aSZMmccghh/Dd736Xl19+ebtryuLbquYxbFjxt/Fk8Tdh1up07NixzustW7Zw0UUX8e1vf3ureSsrK5k3b16jt5G8+d4xNevIN3JR/bbcq7BqptWcpzn55JN57733eOaZZ5g8eTKnnnoqZ511Fvfee+8O11ifjyzMrMWYOnVqnX/WU6ZMoXfv3nTp0iXv/IcffjizZ89mwIABWz06dOjAgAEDaNOmDVOmTKldZu3atcyaNSvv+mrWuWXLFl588cW809u2bQvA5s2b804HGDBgAG3btq1zQnvz5s28/vrrHHjggQ0ul0+3bt0499xzmTBhAr/61a+YOHEi69evb9Q6CuGwMLMWY9myZVx++eXMmzePhx9+mB//+Md5jxpqXHnllUybNo1Ro0Yxc+ZMFi5cyFNPPcU3vvENIOlyuvDCC7nyyit5/vnnmT17NhdccME2/9EPHDiQs88+m4suuohHHnmExYsX84c//IH7778fgL59+yKJ3/3ud1RXV7NmzZqt1tGxY0dGjx7NVVddxdNPP83cuXMZPXo0K1as4Jvf/GbB34/rrruOxx57jAULFjB37lweffRR+vfvT7t27QpeR6HcDWVmiSboXim2c845h82bN3PUUUchiQsvvHCbYXHwwQfzyiuv8L3vfY/jjjuOzZs3079/f84444zaeW666SbWrl3LGWecQUVFBd/61rdYu3btNuu47777+P73v8+ll17KRx99RJ8+fWrrqKys5Ac/+AHXXnstF110Eeeddx4TJkzYah3jx48H4Pzzz+eTTz7hsMMO49lnn6VXr15bzduQdu3ace2117J48eLay2+ffLI4ndxqiv63clRVVRXTp0/frmVLcs7iqRINd7CT/nxt+82dO5cDDjiguctotCFDhnDQQQdxxx13NHcpLd62fgckzYiIqvrt7oYyM7NMDgszM8vkcxZm1iK89NJLzV1Cq+YjCzMzy+SwMGuFdtYLWyzb9v7sHRZmrUybNm1Yt25dc5dhzWTdunUF35sjl8PCrJXp0aMHS5cu5dNPP/URRisSEXz66acsXbqUHj16NHp5n+A2a2VqhsZYtmwZGzdubOZqrJTatGlDz549GxweZVscFmatUJcuXbbrH4a1Xu6GMjOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy1S0sJC0t6QXJc2VNFvSZWn7npKel7Qg/do1Z5mrJS2UNE/SiTntgyS9nU67XZKKVbeZmW2tmEcWm4DvRMQBwGDgEkkHAlcBkyNiIDA5fU06bTjwOeAk4OeSdk3X9QvgYmBg+jipiHWbmVk9RQuLiFgeEW+kz1cDc4FK4DRgYjrbROD09PlpwIMRsT4iFgMLgSMl9QK6RMTrkdyp5b6cZczMrARKcs5CUj/gMGAq0DMilkMSKEDNLZsqgQ9yFluStlWmz+u359vOxZKmS5peXV3dpPtgZtaaFT0sJHUCHgEuj4hV25o1T1tso33rxoi7IqIqIqq6d+/e+GLNzCyvooaFpDYkQfEfEfFo2rwi7Voi/boybV8C7J2zeB9gWdreJ0+7mZmVSDGvhhLwK2BuRNySM+kJYET6fATweE77cEntJO1LciJ7WtpVtVrS4HSd5+UsY2ZmJVDMe3AfA5wLvC3pzbTtGuCHwCRJFwLvA2cBRMRsSZOAOSRXUl0SEZvT5UYDE4AOwDPpw8zMSqRoYRERr5L/fAPAlxpYZhwwLk/7dOCgpqvOzMwaw5/gNjOzTA4LMzPL5LAwM7NMDgszM8vksDAzs0wOCzMzy+SwMDOzTA4LMzPL5LAwM7NMBYWFJH962sysFSv0yOJOSdMkfVPSHsUsyMzMyk9BYRERxwLnkAwhPl3SryX9n6JWZmZmZaPgcxYRsQD4HnAlcBxwu6R3JP1zsYozM7PyUOg5i4Ml/YTkPtpfBIZFxAHp858UsT4zMysDhQ5RfgdwN3BNRKyraYyIZZK+V5TKzMysbBQaFqcA62puRiRpF6B9RHwaEfcXrTozMysLhZ6z+D3JXepqVKRtZmbWChQaFu0jYk3Ni/R5RXFKMjOzclNoWKyVdHjNC0mDgHXbmN/MzHYihZ6zuBz4raRl6etewFeKUpGZmZWdgsIiIv4kaX/gs4CAdyJiY1ErMzOzslHokQXAEUC/dJnDJBER9xWlKjMzKysFhYWk+4H9gDeBzWlzAA4LM7NWoNAjiyrgwIiIYhZjZmblqdCwmAXsBSwvYi1mZs1m2LDmrqBpPPlkcdZbaFh0A+ZImgasr2mMiH8qSlVmZlZWCg2LscUswszMyluhl86+LKkvMDAifi+pAti1uKWZmVm5KHSI8q8DDwO/TJsqgccylrlH0kpJs3LaxkpaKunN9HFKzrSrJS2UNE/SiTntgyS9nU67XZIasX9mZtYECh3u4xLgGGAV1N4IqUfGMhOAk/K0/yQiDk0fTwNIOhAYDnwuXebnkmqOXH4BXAwMTB/51mlmZkVUaFisj4gNNS8k7UbyOYsGRcQrwMcFrv804MGIWB8Ri4GFwJGSegFdIuL19LLd+4DTC1ynmZk1kULD4mVJ1wAd0ntv/xbY3gu0xkh6K+2m6pq2VQIf5MyzJG2rTJ/Xb89L0sWSpkuaXl1dvZ3lmZlZfYWGxVVANfA28A3gaZL7cTfWL0g+CX4oyWc2bk7b852HiG205xURd0VEVURUde/efTvKMzOzfAq9GmoLyW1V796RjUXEiprnku4GnkpfLgH2zpm1D7Asbe+Tp93MzEqo0KuhFkv6c/1HYzeWnoOocQbJJ8MBngCGS2onaV+SE9nTImI5sFrS4PQqqPOAxxu7XTMz2zGNGRuqRnvgLGDPbS0g6TfAEKCbpCXA9cAQSYeSdCW9S9KlRUTMljQJmANsAi6pud83MJrkyqoOwDPpw8zMSqjQbqi/1Gu6VdKrwHXbWOareZp/tY35xwHj8rRPBw4qpE4zMyuOQocoPzzn5S4kRxqdi1KRmZmVnUK7oW7Oeb6JpAvp7CavxszMylKh3VDHF7sQMzMrX4V2Q/3LtqZHxC1NU46ZmZWjxlwNdQTJJa4Aw4BXqPupazMz20k15uZHh0fEakhGjwV+GxEXFaswMzMrH4UO97EPsCHn9QagX5NXY2ZmZanQI4v7gWmS/pPkA3VnkIwAa2ZmrUChV0ONk/QM8Pm06fyImFm8sszMrJwU2g0FUAGsiojbgCXpGE5mZtYKFDqQ4PXAlcDVaVMb4IFiFWVmZuWl0COLM4B/AtYCRMQyPNyHmVmrUWhYbEhvaxoAkjoWryQzMys3hYbFJEm/BPaQ9HXg9+zgjZDMzKzlyLwaKr3p0EPA/sAq4LPAdRHxfJFrMzOzMpEZFhERkh6LiEGAA8LMrBUqtBtqiqQjilqJmZmVrUI/wX08MErSuyRXRInkoOPgYhVmZmblY5thIWmfiHgfOLlE9ZiZNYsnn1LRtzFsaBR9G8WSdWTxGMlos+9JeiQizixBTWZmVmayzlnkRm3/YhZiZmblKyssooHnZmbWimR1Qx0iaRXJEUaH9Dn8zwnuLkWtzszMysI2wyIidi1VIWZmVr4aM0S5mZm1Ug4LMzPL5LAwM7NMRQsLSfdIWilpVk7bnpKel7Qg/do1Z9rVkhZKmifpxJz2QZLeTqfdng5saGZmJVTMI4sJwEn12q4CJkfEQGBy+hpJBwLDgc+ly/xcUs3J9V8AFwMD00f9dZqZWZEVLSwi4hXg43rNpwET0+cTgdNz2h+MiPURsRhYCBwpqRfQJSJeT2++dF/OMmZmViKlPmfRMyKWA6Rfe6TtlcAHOfMtSdsq0+f12/OSdLGk6ZKmV1dXN2nhZmatWbmc4M53HiK20Z5XRNwVEVURUdW9e/cmK87MrLUrdVisSLuWSL+uTNuXAHvnzNcHWJa298nTbmZmJVTqsHgCGJE+HwE8ntM+XFI7SfuSnMielnZVrZY0OL0K6rycZczMrEQKvflRo0n6DTAE6CZpCXA98ENgkqQLgfeBswAiYrakScAcYBNwSURsTlc1muTKqg7AM+nDzMxKqGhhERFfbWDSlxqYfxwwLk/7dOCgJizNzMwaqVxOcJuZWRlzWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZpmYJC0nvSnpb0puSpqdte0p6XtKC9GvXnPmvlrRQ0jxJJzZHzWZmrVlzHlkcHxGHRkRV+voqYHJEDAQmp6+RdCAwHPgccBLwc0m7NkfBZmatVTl1Q50GTEyfTwROz2l/MCLWR8RiYCFwZOnLMzNrvZorLAL4L0kzJF2ctvWMiOUA6dceaXsl8EHOskvStq1IuljSdEnTq6uri1S6mVnrs1szbfeYiFgmqQfwvKR3tjGv8rRFvhkj4i7gLoCqqqq885iZWeM1y5FFRCxLv64E/pOkW2mFpF4A6deV6exLgL1zFu8DLCtdtWZmVvKwkNRRUuea58AJwCzgCWBEOtsI4PH0+RPAcEntJO0LDASmlbZqM7PWrTm6oXoC/ympZvu/johnJf0JmCTpQuB94CyAiJgtaRIwB9gEXBIRm5uhbjOzVqvkYRERfwYOydP+F+BLDSwzDhhX5NLMzKwB5XTprJmZlSmHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhZmZZXJYmJlZJoeFmZllajFhIekkSfMkLZR0VXPXY2bWmrSIsJC0K/Az4GTgQOCrkg5s3qrMzFqPFhEWwJHAwoj4c0RsAB4ETmvmmszMWo3dmruAAlUCH+S8XgIcVX8mSRcDF6cv10iaV4LatougG/BR8Tekom+CUu1Laews+7Kz7AfsTPvylIq+L03wJ983X2NLCYt8ux9bNUTcBdxV/HJ2nKTpEVHV3HU0Be9L+dlZ9gO8L+WipXRDLQH2znndB1jWTLWYmbU6LSUs/gQMlLSvpLbAcOCJZq7JzKzVaBHdUBGxSdIY4DlgV+CeiJjdzGXtqBbRXVYg70v52Vn2A7wvZUERW3X9m5mZ1dFSuqHMzKwZOSzMzCyTw6LIsoYpUeL2dPpbkg5vjjoLUcC+nJPuw1uSXpN0SHPUmaXQoWMkHSFps6Qvl7K+xihkXyQNkfSmpNmSXi51jYUq4Pdrd0lPSvrvdF/Ob446s0i6R9JKSbMamN5i/ubriAg/ivQgORm/COgPtAX+Gziw3jynAM+QfJZkMDC1uevegX35X0DX9PnJ5bgvhexHznwvAE8DX27uunfgZ7IHMAfYJ33do7nr3oF9uQYYnz7vDnwMtG3u2vPsyxeAw4FZDUxvEX/z9R8+siiuQoYpOQ24LxJTgD0k9Sp1oQXI3JeIeC0i/pq+nELyeZhyU+jQMd8CHgFWlrK4RipkX/4v8GhEvA8QEeW6P4XsSwCdJQnoRBIWm0pbZraIeIWktoa0lL/5OhwWxZVvmJLK7ZinHDS2zgtJ3j2Vm8z9kFQJnAHcWcK6tkchP5N/ALpKeknSDEnnlay6xilkX+4ADiD5QO7bwGURsaU05TWplvI3X0eL+JxFC1bIMCUFDWVSBgquU9LxJGFxbFEr2j6F7MetwJURsVmlGVtrexWyL7sBg4AvAR2A1yVNiYj5xS6ukQrZlxOBN4EvAvsBz0v6Q0SsKnJtTa2l/M3X4bAorkKGKWkpQ5kUVKekg4F/B06OiL+UqLbGKGQ/qoAH06DoBpwiaVNEPFaSCgtX6O/XRxGxFlgr6RXgEKDcwqKQfTkf+GEkHf8LJS0G9gemlabEJtNS/ubrcDdUcRUyTMkTwHnpFRKDgb9FxPJSF1qAzH2RtA/wKHBuGb5zrZG5HxGxb0T0i4h+wMPAN8swKKCw36/Hgc9L2k1SBclozXNLXGchCtmX90mOkJDUE/gs8OeSVtk0WsrffB0+siiiaGCYEkmj0ul3klxtcwqwEPiU5N1T2SlwX64DPgP8PH1XvinKbITNAvejRShkXyJirqRngbeALcC/R0TeSzqbU4E/lxuACZLeJunKuTIiym7ockm/AYYA3SQtAa4H2kDL+puvz8N9mJlZJndDmZlZJoeFmZllcliYmVkmh4WZmWVyWJiZWSaHhdkOkrSmEfOOlfTdYq3frFgcFmZmlslhYVYEkoZJmipppqTfp584rnGIpBckLZD09ZxlrpD0p/QeBz9ohrLNGuSwMCuOV4HBEXEYyXDb/5oz7WDgVOBo4DpJvSWdAAwkGar7UGCQpC+UtmSzhnm4D7Pi6AM8lN6noC2wOGfa4xGxDlgn6UWSgDgWOAGYmc7TiSQ8XildyWYNc1iYFcdPgVsi4glJQ4CxOdPqj7ETJGMd/VtE/LIk1Zk1kruhzIpjd2Bp+nxEvWmnSWov6TMkA879iWQAvQskdYLkBkySepSqWLMsPrIw23EV6eiiNW4hOZL4raSlJLeY3Tdn+jTgd8A+wA0RsQxYJukAkpsTAawBvkZ539bVWhGPOmtmZpncDWVmZpkcFmZmlslhYWZmmRwWZmaWyWFhZmaZHBZmZpbJYWFmZpn+P4O+TK7sGzTBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_distribution(y_train, soft_train_preds, voting='Soft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "4f3c3800-7c4c-4008-ab45-72ff0ebf50b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkxUlEQVR4nO3de5xVdb3/8ddbuQ6IoFzC4SbCUcm84Kjw0xL1HC8JmuccjX5koJhhWtnJUtSMHh48WubRMo9RGqiVklpeMs04opkXAtGUm2B44SKMliGXw/Vz/lhr1tkDe2b2DLP3noH38/HYj73Wd33X2p/vnpn9mfVda3+/igjMzMwA9ih3AGZm1nI4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFKxkJJ0l6R1JayUdUaYYQtKgOraNk/RsqWMqJklTJf17uvxxSYuaeJzbJX2zeaOzlshJwRpF0nGSnpP0d0l/lfRHSUcVuPuNwCUR0Rn4W/oB3aaRrz9Y0r2SqiWtkbRY0g8k9Wl0Ywp/zQ6SPpB0Yp5t/ynp/p08/puSNqTJcpWkn0rqvDPHzCci/hARBxYQzw7JMSImRMS1zR2TtTxOClYwSV2AR4EfAPsAlcC3gY0FHqI/MG8nXn8Q8CKwAjgiIroAxwJvAMc19bgNiYj/Ae4DPrddPHsCnwGmNcPLjEqT5VDgKODq7Ss0NoGaNYWTgjXGPwBExC8iYmtEbIiI30XEnwEk7SHpaklvSVot6S5Je0tqL2ktsCfwiqQ3gGfSY36Q/oc8vIDXnwT8MSL+LSKWpbGsjoibI+LemkqSPi9pSXom87Ck/fIdTNK+6fY1kmYBB9Tz2tOAf5FUkVN2Csnf0G/T410uabmkDyUtknRSAW2qJSKWp8c7JD1mSLpY0mJgcVo2UtLL6dnLc5IOzWnTEZJeSmO4D+iQs22EpGU5630lPZiedb0v6VZJBwO3A8PTn8sHad2sGypdr/M9TmOekJ7F/U3SDyUp3TZI0tPpmeZ7aYzWgjgpWGO8DmyVNE3SaZK6bbd9XPo4ARgIdAZujYiN6X/BAIdFxAHAJ9L1rhHROSKel9Qv/aDrV8fr/yPwQH0Bpl08/wGcA/QG3gLuraP6D4H/Seudnz7yiojngJXAP+cUnwv8PCK2SDoQuAQ4KiL2IkkYb9YXax3x9wU+CczNKf4UcAwwRNJQ4E7gC8C+wI+Ah9PE2w74NXA3yZncL4F/qeN19iQ563sLGEBy1ndvRCwAJgDPpz+Xrnn2LeQ9HklyxnNYWu+UtPxa4HdAN6APyVmntSQR4YcfBT+Ag4GpwDJgC/Aw0CvdNgP4Yk7dA4HNQJt0PYBB6fKAdL1NI157C3BqzvolwAfAWuDHadkdwHdy6nROYxiQGwPJWctm4KCcutcBz9bz+lcDv0uXuwDrSbqxSI+5miRxtW3ke/pm2oYPSD5gbwM65sR7Yk7d/wKu3W7/RcDxJIl2BaCcbc8B/54ujwCWpcvDgep87z9JYn92u7KpOccp5D0+Lmf7dOCKdPkuYArQp9y/y37kf/hMwRolIhZExLiI6EPSxbEfcHO6eT+SD7UabwFtgF7N9PLvk/xnWhPLrZH8J3sz0DZfDBGxNt2vcrtj9Uhje2e7eOtzF3CCpErgX4ElETE3fZ0lwKUkXVyr04vhebut6vCpiOgaEf0j4osRsSFnW26M/YGvpWdUH6TdO31J2r0fsDzST98G2tQXeCsitjQixhqFvMfv5iyvJ0kcAN8ABMySNE9SnWdnVh5OCtZkEbGQ5D/IQ9KiFSQfWjX6kfx3vyrf7k14yRnU7r7Jp1YMkjqRdLMs365edRpb35yyurqtAIiIt4E/AGNIuo7u2m77zyPiuPT1A7ihgVgLlftevQNMThNIzaMiIn5B0r1VWdN/n6qrTe8A/eq4eN3Qz6bQ93jHA0e8GxGfj4j9SLrAblMdtwhbeTgpWMEkHSTpazW3f6b9358BXkir/AL4qqT901sqrwPuq+O/0WpgG8m1h0JNAj4u6ab0v3UkdSfp0qrxc+A8SYdLap/G8GJEvJl7oIjYCjwITJJUIWkIMLaAGKaRdFsdC/ysplDSgZJOTF/zf4ANwNZGtK1QPwYmSDpGiU6STpe0F/A8SaL7sqQ2kv4ZOLqO48wiSSLXp8foIOnYdNsqoE96jSKfgt7jfCSdrf+7ffhvJAmoGO+TNZGTgjXGhyQXPF+UtI4kGbwGfC3dfifJRc5ngKUkH45fynegiFgPTAb+mHaDDEsvNK+t60JzRLwODCO5QPmKpA+BP5L85/rNtM6MdPkBkg+9A4DRdbTnEpJujXdJznh+WsB7cD/JRdIZEbEyp7w9cD3wXnq8nsCVAJLGSGryrbi5ImI28HngVpIP1SUk1wCIiE0kZ1Lj0m2fJkl8+Y6zFRhFci3kbZJrRJ9ON/83ya3D70p6L8++jXmPt3cUye/PWpLrUV+JiKUF7msloNrdj2ZmtjvzmYKZmWWcFMzMLOOkYGZmGScFMzPLtOoBtrp37x4DBgwodxhmZq3KnDlz3ouIHvm2teqkMGDAAGbPnl3uMMzMWhVJdX57391HZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmmVZ995GZ1W/NmjWsXr2azZs3lzsUK5G2bdvSs2dPunTp0qT9i5YU0ukJc+dfHQhcQzIG/X0kM2+9CZwTEX9L95kIjCcZSvfLEfFEseIz29WtWbOGVatWUVlZSceOHak9zYLtiiKCDRs2sHx5MrVFUxJD0bqPImJRRBweEYcDR5LMvvQr4AqSYYcHk0yacgVAOp79aOCjwKkkk2/sWaz4zHZ1q1evprKykoqKCieE3YQkKioqqKysZPXq1U06RqmuKZwEvBERbwFnkkxUQvr8qXT5TJKJwzem46svoe4JQsysAZs3b6Zjx47lDsPKoGPHjk3uMixVUhhNMisXJJO8rwRIn3um5ZXUnot2GTvOq4ukCyXNljS7urq6iCE3E6n4D7M6+Axh97QzP/eiJ4V0Sr8zgF82VDVP2Q4zAEXElIioioiqHj3yDt1hZmZNVIozhdOAlyKiZvL2VZJ6A6TPNR1fy6g9iXofkmkWzcysREpxS+pn+L+uI0jmZR1LMp/tWOChnPKfS7oJ2A8YTDK5uJk1o1GjSvt6jzxS2teznVPUMwVJFcA/UXvy8OuBf5K0ON12PUBEzAOmA/OBx4GL08nFzWw3MmLECC655JJyh7HbKuqZQkSsB/bdrux9kruR8tWfDEwuZkxm1vpt3ryZtm3bljuMXZKHuTCzFmPcuHE8/fTT/PCHP0QSkpg6dSqSeOyxxzj66KNp164dTzzxBJMmTeKQQw6ptf/UqVPp3LlzrbJHHnmEI488kg4dOrD//vtz1VVXsWnTplI2q1XxMBdm1mLccsstvP766xx00EFcd911AMybNw+Ayy+/nO9973sMGjSIvfbaq6AJtp544gnGjBnDLbfcwic+8QnefvttJkyYwMaNG7nxxhuL2pbWyknBzFqMvffem3bt2lFRUcFHPvIRABYuXAjApEmTOPnkkxt1vMmTJ/P1r3+d8847D4ADDjiAG264gc9+9rN897vf9fc48nBSMLNWoaqqqtH7zJkzh1mzZnHDDTdkZdu2bWPDhg28++679O7duzlD3CU4KZhZq9CpU6da63vssQcRtb/fuv3QDtu2beNb3/oWZ5999g7H85df83NSMLMWpV27dmzd2vDd6D169GDVqlVERNYN9PLLL9eqM3ToUBYuXMigQYOKEeouyUnBzFqUAQMGMGvWLN588006d+7Mtm3b8tYbMWIEf/3rX7nuuusYPXo0M2fO5P77769V55prrmHkyJH079+fc845hzZt2vDaa68xa9YsvvOd75SiOa1PRLTax5FHHhktHhT/YZbH/Pnzyx1CkyxatCiGDRsWHTt2DCB++tOfBhDV1dU71L399tujX79+UVFREZ/+9Kfj5ptvjk6dOtWq88QTT8Rxxx0XHTt2jL322iuOPPLI+MEPflCq5pRNfT9/YHbU8bmqiB3GnGs1qqqqopDb0sqqFHc3tOKfoRXPggULOPjgg8sdhpVJfT9/SXMiIu+Ve395zczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCma7G6m0jxZs5MiRjBs3Lltvjvmhx40bx8iRI3cysvLxgHhmZqkHH3yw4LmfZ86cyQknnEB1dTXdu3fPym+55ZYdhvRuTZwUzKxV27RpE+3atWuWY+2zzz47fYy99967GSIpn6J2H0nqKul+SQslLZA0XNI+kp6UtDh97pZTf6KkJZIWSTqlmLGZWcs0YsQIJkyYwFe+8hW6detGt27d+PrXv54NoT1gwAAmTZrE+eefT9euXRkzZgwAzz33HMcffzwVFRVUVlZy0UUXsWbNmuy469evZ9y4cXTu3JlevXplc0Bv/9q53UebNm3iyiuvpH///rRv356BAwfy/e9/nzfffJMTTjgBSOZ1kJR1Q23ffbRx40YuvfRSevXqRYcOHRg2bBjPPvtstn3mzJlIYsaMGRxzzDFUVFRQVVXFSy+9lNX5+9//zrnnnkvPnj3p0KEDAwcO5Oabb975NzuPYl9TuAV4PCIOAg4DFgBXADMiYjAwI11H0hBgNPBR4FTgNkl7Fjk+M2uBfvazn7Ft2zaef/55fvSjHzFlypRaH4I33XQTBx10ELNnz+a6667j1Vdf5eSTT+aMM87glVde4cEHH+Tll1/m/PPPz/a57LLLePLJJ3nggQeYMWMGc+fO5Zlnnqk3jrFjx3LXXXdx0003sWDBAu644w66du1K3759eeCBBwCYN28eK1eu5JZbbsl7jG984xvcd9993HnnncydO5ePfexjnHrqqaxcubJWvYkTJ3L99dfz0ksvse+++zJmzJisG+rqq6/m1Vdf5dFHH2XhwoXceeedVFZWNuWtbVhdY2rv7APoAiyFZHjunPJFQO90uTewKF2eCEzMqfcEMLy+1/B8Cp5PwepW53j6pfid3Infz+OPPz4GDx4c27Zty8quvfbaqKysjIiI/v37x8iRI2vtc+6558b5559fq2zu3LkBxKpVq+LDDz+Mdu3axT333JNt//DDD2PvvfeOsWPH1nrtiy++OCIiXn/99QDit7/9bd44n3rqqbzzPIwdOzZOP/30iIhYu3ZttG3bNqZNm5Zt37JlSwwcODCuuuqqWsd5/PHHszrPPvtsAPHOO+9ERMSoUaNi3Lhx9bxrO2rqfArFPFMYCFQDP5U0V9JPJHUCekXEyjQhrQR6pvUrgXdy9l+WltUi6UJJsyXNrq6uLmL4ZlYuw4YNy6bYBBg+fDjLly/PuoOqqmpPBTBnzhzuueceOnfunD2OPfZYAN544w3eeOMNNm3axPDhw7N9OnfuzMc+9rE6Y5g7dy577LFH1k3UFG+88QabN2/OYgHYc889GT58OPPnz69V99BDD82W99tvPwBWr14NwEUXXcT06dM57LDDuOyyy3j66aebHFNDipkU2gBDgf+KiCOAdaRdRXXId+/aDpfwI2JKRFRFRJUn3jbbPXXq1KnW+rZt27jgggt4+eWXs8crr7zC4sWLOfzww5t0N1BT9qnrGMpza+72Zbl3PdVsq7mOctppp/HWW29x2WWX8d5773H66adz3nnn7XR8+RQzKSwDlkXEi+n6/SRJYpWk3gDp8+qc+n1z9u8DrChifGbWQr344ou1PpRfeOEF9ttvP7p06ZK3/tChQ5k3bx6DBg3a4dGxY0cGDRpE27ZteeGFF7J91q1bx2uvvVZnDEOHDmXbtm089dRTebfX3PG0devWOo8xaNAg2rVrV+vC8tatW3n++ecZMmRInfvl0717d84991ymTp3KHXfcwbRp09i4cWOjjlGIoiWFiHgXeEfSgWnRScB84GFgbFo2FngoXX4YGC2pvaT9gcHArGLFZ2Yt14oVK7j00ktZtGgR999/P9/97nf56le/Wmf9yy+/nFmzZjFhwgTmzp3LkiVLePTRR/nCF74AJF1F48eP5/LLL+fJJ59k3rx5nH/++fV+oA8ePJhzzjmHCy64gAceeIClS5fyhz/8gbvvvhuA/v37I4nf/OY3VFdXs3bt2h2O0alTJy666CKuuOIKHnvsMRYsWMBFF13EqlWr+OIXv1jw+3HNNdfw61//msWLF7NgwQIefPBBBg4cSPv27Qs+RqGK/T2FLwE/k9QO+AtwHkkimi5pPPA2cDZARMyTNJ0kcWwBLo6Iun9iZtY0reCLVWPGjGHr1q0cc8wxSGL8+PH1JoVDDz2UZ555hquvvprjjz+erVu3MnDgQM4666yszo033si6des466yzqKio4Etf+hLr1q2rN4677rqLb37zm3z5y1/mvffeo0+fPlkclZWVfPvb3+aqq67iggsu4HOf+xxTp07d4Rg33HADAOeddx4ffPABRxxxBI8//ji9e/cu+P1o3749V111FUuXLs1ua33kkUcK3r8x1Bz9ZuVSVVUVs2fPbvL+o0Y1YzB1eOTREnzNvxX/DK146pu4vSUbMWIEhxxyCLfeemu5Q2nV6vv5S5oTEVX5tnnsIzMzyzgpmJlZxmMfmVmLMnPmzHKHsFvzmYKZmWWcFMx2Ya35RhJrup35uTspmO2i2rZty4YNG8odhpXBhg0bCp4XYntOCma7qJ49e7J8+XLWr1/vM4bdRESwfv16li9fTs+ePRveIQ9faDbbRdUMCbFixQo2b95c5misVNq2bUuvXr3qHBKkIU4KZruwLl26NPnDwXZP7j4yM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCxT1KQg6U1Jr0p6WdLstGwfSU9KWpw+d8upP1HSEkmLJJ1SzNjMzGxHpThTOCEiDo+IqnT9CmBGRAwGZqTrSBoCjAY+CpwK3CZpzxLEZ2ZmqXJ0H50JTEuXpwGfyim/NyI2RsRSYAlwdOnDMzPbfRU7KQTwO0lzJF2YlvWKiJUA6XPNTBCVwDs5+y5Ly2qRdKGk2ZJmV1dXFzF0M7PdT7HnUzg2IlZI6gk8KWlhPXWVp2yH6aIiYgowBaCqqsrTSZmZNaOinilExIr0eTXwK5LuoFWSegOkz6vT6suAvjm79wFWFDM+MzOrrWhJQVInSXvVLAMnA68BDwNj02pjgYfS5YeB0ZLaS9ofGAzMKlZ8Zma2o2J2H/UCfiWp5nV+HhGPS/oTMF3SeOBt4GyAiJgnaTowH9gCXBwRW4sYn5mZbadoSSEi/gIclqf8feCkOvaZDEwuVkxmZlY/f6PZzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWcZJwczMMk4KZmaWKSgpSDqk2IGYmVn5FXqmcLukWZK+KKlrMQMyM7PyKSgpRMRxwBiS+Q5mS/q5pH8qamRmZlZyBV9TiIjFwNXA5cDxwPclLZT0z8UKzszMSqvQawqHSvpPYAFwIjAqIg5Ol/+ziPGZmVkJFTqfwq3Aj4ErI2JDTWE6//LVRYnMzMxKrtCk8ElgQ81MaJL2ADpExPqIuLto0ZmZWUkVek3h90DHnPWKtMzMzHYhhSaFDhGxtmYlXa4oTkhmZlYuhXYfrZM0NCJeApB0JLChgX3MzFqUUaPKHUHzeeSR4hy30KRwKfBLSSvS9d7Ap4sSkZmZlU1BSSEi/iTpIOBAQMDCiNhcyL6S9gRmA8sjYqSkfYD7gAHAm8A5EfG3tO5EYDywFfhyRDzRuOaYmdnOaMyAeEcBhwJHAJ+R9LkC9/sKyfcbalwBzIiIwcCMdB1JQ4DRwEeBU4Hb0oRiZmYlUuiX1+4GbgSOI0kORwFVBezXBzgd+ElO8ZnAtHR5GvCpnPJ7I2JjRCwFlgBHFxKfmZk1j0KvKVQBQyIiGnn8m4FvAHvllPWKiJUAEbFSUs+0vBJ4IafesrSsFkkXAhcC9OvXr5HhmJlZfQrtPnoN+EhjDixpJLA6IuYUukuesh2SUERMiYiqiKjq0aNHY0IyM7MGFHqm0B2YL2kWsLGmMCLOqGefY4EzJH0S6AB0kXQPsEpS7/QsoTewOq2/jGQU1hp9gBWYmVnJFJoUJjX2wBExEZgIIGkEcFlEfFbSd4GxwPXp80PpLg8DP5d0E7AfMBiY1djXNTOzpiv0ltSnJfUHBkfE7yVVAE29M+h6YLqk8cDbwNnpa8yTNB2YD2wBLq4Za8nMzEqjoKQg6fMkF3f3AQ4guQB8O3BSIftHxExgZrr8fl37RcRkYHIhxzQzs+ZX6IXmi0muEayBbMKdnvXuYWZmrU6hSWFjRGyqWZHUhjx3BpmZWetWaFJ4WtKVQMd0buZfAkUajsnMzMql0KRwBVANvAp8AXiMZL5mMzPbhRR699E2kuk4f1zccMzMrJwKvftoKfm/XTyw2SMyM7OyaczYRzU6kHy3YJ/mD8fMzMqpoGsKEfF+zmN5RNwMnFjc0MzMrNQK7T4amrO6B8mZw151VDczs1aq0O6j7+UsbyGdMa3ZozEzs7Iq9O6jE4odiJmZlV+h3Uf/Vt/2iLipecIxM7NyaszdR0eRDG8NMAp4BninGEGZmVl5NGaSnaER8SGApEnALyPigmIFZmZmpVfoMBf9gE0565uAAc0ejZmZlVWhZwp3A7Mk/Yrkm81nAXcVLSozMyuLQu8+mizpt8DH06LzImJu8cIyM7NyKLT7CKACWBMRtwDLJO1fpJjMzKxMCkoKkr4FXA5MTIvaAvcUKygzMyuPQs8UzgLOANYBRMQKPMyFmdkup9CksCkignT4bEmdGtpBUgdJsyS9ImmepG+n5ftIelLS4vS5W84+EyUtkbRI0ilNaZCZmTVdoUlhuqQfAV0lfR74PQ1PuLMRODEiDgMOB06VNIxkFrcZETEYmJGuI2kIMBr4KHAqcJukPRvZHjMz2wkN3n0kScB9wEHAGuBA4JqIeLK+/dIzi7Xpatv0EcCZwIi0fBowk+R6xZnAvRGxEVgqaQlwNPB8o1pkZmZN1mBSiIiQ9OuIOBKoNxFsL/1Pfw4wCPhhRLwoqVdErEyPvVJSz7R6JfBCzu7L0rLtj3khcCFAv379GhOOmZk1oNAvr70g6aiI+FNjDh4RW4HDJXUFfiXpkHqqK98h8hxzCjAFoKqqaoftZmZ1eeTRfB8zzW/UyNb70VRoUjgBmCDpTZI7kERyEnFoITtHxAeSZpJcK1glqXd6ltAbWJ1WWwb0zdmtD7CiwPjMzKwZ1HuhWVJN/8xpwECSKThHASPT5/r27ZGeISCpI/CPwEKSkVbHptXGAg+lyw8DoyW1T78YNxiY1cj2mJnZTmjoTOHXJKOjviXpgYj4l0YcuzcwLb2usAcwPSIelfQ8yd1M44G3gbMBImKepOnAfJLZ3S5Ou5/MzKxEGkoKuR1wAxtz4Ij4M3BEnvL3gZPq2GcyMLkxr2NmZs2noe8pRB3LZma2C2roTOEwSWtIzhg6psvwfxeauxQ1OjMzK6l6k0JE+BvFZma7kcYMnW1mZrs4JwUzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZhknBTMzyxQtKUjqK+kpSQskzZP0lbR8H0lPSlqcPnfL2WeipCWSFkk6pVixmZlZfsU8U9gCfC0iDgaGARdLGgJcAcyIiMHAjHSddNto4KPAqcBtkjxHtJlZCRUtKUTEyoh4KV3+EFgAVAJnAtPSatOAT6XLZwL3RsTGiFgKLAGOLlZ8Zma2o5JcU5A0ADgCeBHoFRErIUkcQM+0WiXwTs5uy9Ky7Y91oaTZkmZXV1cXNW4zs91N0ZOCpM7AA8ClEbGmvqp5ymKHgogpEVEVEVU9evRorjDNzIwiJwVJbUkSws8i4sG0eJWk3un23sDqtHwZ0Ddn9z7AimLGZ2ZmtRXz7iMBdwALIuKmnE0PA2PT5bHAQznloyW1l7Q/MBiYVaz4zMxsR22KeOxjgXOBVyW9nJZdCVwPTJc0HngbOBsgIuZJmg7MJ7lz6eKI2FrE+MzMbDtFSwoR8Sz5rxMAnFTHPpOBycWKyczM6udvNJuZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs0zRkoKkOyWtlvRaTtk+kp6UtDh97pazbaKkJZIWSTqlWHGZmVndinmmMBU4dbuyK4AZETEYmJGuI2kIMBr4aLrPbZL2LGJsZmaWR9GSQkQ8A/x1u+IzgWnp8jTgUznl90bExohYCiwBji5WbGZmll+pryn0ioiVAOlzz7S8Engnp96ytGwHki6UNFvS7Orq6qIGa2a2u2kpF5qVpyzyVYyIKRFRFRFVPXr0KHJYZma7l1InhVWSegOkz6vT8mVA35x6fYAVJY7NzGy3V+qk8DAwNl0eCzyUUz5aUntJ+wODgVkljs3MbLfXplgHlvQLYATQXdIy4FvA9cB0SeOBt4GzASJinqTpwHxgC3BxRGwtVmxmZpZf0ZJCRHymjk0n1VF/MjC5WPGYmVnDWsqFZjMzawGcFMzMLOOkYGZmGScFMzPLOCmYmVnGScHMzDJOCmZmlnFSMDOzjJOCmZllnBTMzCzjpGBmZhknBTMzyzgpmJlZxknBzMwyTgpmZpZxUjAzs4yTgpmZZZwUzMws46RgZmYZJwUzM8s4KZiZWabFJQVJp0paJGmJpCvKHY+Z2e6kRSUFSXsCPwROA4YAn5E0pLxRmZntPlpUUgCOBpZExF8iYhNwL3BmmWMyM9tttCl3ANupBN7JWV8GHJNbQdKFwIXp6lpJi0oUW5MIugPvFfdFVNTD5yh+W0pjV2kHuC0t06Mqelt28s++f10bWlpSyNfMqLUSMQWYUppwdp6k2RFRVe44msOu0pZdpR3gtrRUrbktLa37aBnQN2e9D7CiTLGYme12WlpS+BMwWNL+ktoBo4GHyxyTmdluo0V1H0XEFkmXAE8AewJ3RsS8Moe1s1pNV1cBdpW27CrtALelpWq1bVFENFzLzMx2Cy2t+8jMzMrIScHMzDJOCs2koeE5lPh+uv3PkoaWI86GFNCOMWn8f5b0nKTDyhFnIQodMkXSUZK2SvrXUsbXGIW0RdIISS9Lmifp6VLHWIgCfr/2lvSIpFfSdpxXjjgLIelOSaslvVbH9lbxN7+DiPBjJx8kF8XfAAYC7YBXgCHb1fkk8FuS72IMA14sd9xNbMf/A7qly6e1xHYU2pacev8NPAb8a7nj3omfS1dgPtAvXe9Z7rib2I4rgRvS5R7AX4F25Y69jvZ8AhgKvFbH9hb/N5/v4TOF5lHI8BxnAndF4gWgq6TepQ60AQ22IyKei4i/pasvkHyXpCUqdMiULwEPAKtLGVwjFdKW/w88GBFvA0RES2xPIe0IYC9JAjqTJIUtpQ2zMBHxDEl8dWkNf/M7cFJoHvmG56hsQp1ya2yM40n+E2qJGmyLpErgLOD2EsbVFIX8XP4B6CZppqQ5kj5XsugKV0g7bgUOJvnS6qvAVyJiW2nCa3at4W9+By3qewqtWIPDcxRYp9wKjlHSCSRJ4biiRtR0hbTlZuDyiNiq0o0f1RSFtKUNcCRwEtAReF7SCxHxerGDa4RC2nEK8DJwInAA8KSkP0TEmiLHVgyt4W9+B04KzaOQ4TlawxAeBcUo6VDgJ8BpEfF+iWJrrELaUgXcmyaE7sAnJW2JiF+XJMLCFfr79V5ErAPWSXoGOAxoSUmhkHacB1wfSaf8EklLgYOAWaUJsVm1hr/5Hbj7qHkUMjzHw8Dn0jsShgF/j4iVpQ60AQ22Q1I/4EHg3Bb2X+j2GmxLROwfEQMiYgBwP/DFFpgQoLDfr4eAj0tqI6mCZHThBSWOsyGFtONtkrMdJPUCDgT+UtIom09r+Jvfgc8UmkHUMTyHpAnp9ttJ7m75JLAEWE/yH1GLUmA7rgH2BW5L/8PeEi1wNMgC29IqFNKWiFgg6XHgz8A24CcRkfdWyXIp8GdyLTBV0qsk3S+XR0SLHE5b0i+AEUB3ScuAbwFtofX8zefjYS7MzCzj7iMzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4JZASStbUTdSZIuK9bxzYrJScHMzDJOCmZNJGmUpBclzZX0+/QbuDUOk/TfkhZL+nzOPl+X9Kd0fP1vlyFss3o5KZg13bPAsIg4gmQY6G/kbDsUOB0YDlwjaT9JJwODSYaQPhw4UtInShuyWf08zIVZ0/UB7kvHyG8HLM3Z9lBEbAA2SHqKJBEcB5wMzE3rdCZJEs+ULmSz+jkpmDXdD4CbIuJhSSOASTnbth8/JkjG8vmPiPhRSaIzawJ3H5k13d7A8nR57HbbzpTUQdK+JIOm/YlkILjzJXWGZJIfST1LFaxZIXymYFaYinQkzBo3kZwZ/FLScpKpSffP2T4L+A3QD7g2IlYAKyQdTDIBDsBa4LO07KlAbTfjUVLNzCzj7iMzM8s4KZiZWcZJwczMMk4KZmaWcVIwM7OMk4KZmWWcFMzMLPO/VRv/QfEg5A4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_distribution(y_test, soft_test_preds, voting='Soft')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e11478f-32db-44a6-9a50-209717ce4837",
   "metadata": {},
   "source": [
    "## Kaggle Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "13896c6b-efc6-4db7-b4e6-5535822d28c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('data/test.csv', sep='\\t')\n",
    "test['text'] = test['text'].apply(lambda x: clean_text(x))\n",
    "test_embeddings = get_w2v_embeddings(test['text'])\n",
    "X_train, X_test, y_train, y_test = w2v_data.get_data(scaled=False)\n",
    "test_m = RandomForestClassifier()\n",
    "test_m.fit(X_train, y_train)\n",
    "sub_preds = test_m.predict(test_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "b672c390-d46c-4457-9626-de127c0135b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({'id' : test['id'], 'label' : sub_preds})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "9c34e7ca-5f18-4640-a31c-5e324e7c2f95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1242</th>\n",
       "      <td>1244</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1243</th>\n",
       "      <td>1245</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1244</th>\n",
       "      <td>1246</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1245</th>\n",
       "      <td>1247</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1246</th>\n",
       "      <td>1248</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1247 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  label\n",
       "0        2      0\n",
       "1        3      0\n",
       "2        4      1\n",
       "3        5      1\n",
       "4        6      0\n",
       "...    ...    ...\n",
       "1242  1244      0\n",
       "1243  1245      0\n",
       "1244  1246      0\n",
       "1245  1247      0\n",
       "1246  1248      0\n",
       "\n",
       "[1247 rows x 2 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "b16280a7-d318-4446-9303-fcab28994aad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# submission.to_csv(\"submission.csv\", index=False) # save to csv to submit"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}